├── .dockerignore ├── .editorconfig ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── issue.md │ ├── rfe.md │ ├── task.md │ └── user-stories.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── _docker-build.yml │ ├── _e2e-test.yml │ ├── _gocilint.yml │ ├── _gosecscan.yml │ ├── _trivy.yml │ ├── codeql-analysis.yml │ ├── e2e-awskms-on-pr.yml │ ├── e2e-azurekms-on-pr.yml │ ├── e2e-debug-on-pr.yml │ ├── e2e-proxy-on-pr.yml │ ├── e2e-trousseau-on-pr.yml │ ├── e2e-vault-on-pr.yml │ ├── generator-test-on-pr.yml │ └── scorecards-analysis.yml ├── .gitignore ├── .golangci.yaml ├── .husky └── hooks │ ├── pre-commit │ └── pre-push ├── .task ├── cluster.yml ├── docker.yml ├── fetch.yml ├── go.yml └── prod.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MAINTAINERS.md ├── Makefile ├── README.md ├── ROADMAP.md ├── SECURITY.md ├── Taskfile.yml ├── assets ├── Ondat Diagram-w-all.png ├── logo-color-horizontal.png ├── logo-horizontal-mono.png ├── logo-horizontal.png └── logo-vertical.png ├── deployment ├── docker-compose │ ├── docker-compose.override.awskms.yaml │ ├── docker-compose.override.azurekms.yaml │ ├── docker-compose.override.vault.yaml │ └── docker-compose.yaml ├── helm │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ │ ├── configmap-awskms.yaml │ │ ├── configmap-azurekms.yaml │ │ ├── configmap-vault.yaml │ │ ├── daemonset.yaml │ │ └── rbac.yaml │ └── values.yaml ├── kustomize │ ├── configmap-awskms.yaml │ ├── configmap-azurekms.yaml │ ├── configmap-vault-generator.yaml │ ├── configmap-vault.yaml │ ├── daemonset.yaml │ ├── kustomization.yaml │ ├── rbac.yaml │ ├── sidecar-awskms.yaml │ ├── sidecar-azurekms.yaml │ └── sidecar-vault.yaml └── systemd │ ├── tousseau-awskms.service │ ├── tousseau-azurekms.service │ ├── tousseau-proxy.service │ ├── tousseau-trousseau.service │ └── tousseau-vault.service ├── go.mod ├── go.sum ├── go.work ├── go.work.sum ├── issue-31 ├── localdev.md ├── pkg ├── logger │ └── logger.go ├── metrics │ ├── exporter.go │ ├── prometheus_exporter.go │ └── stats_reporter.go ├── providers │ └── providers.go ├── utils │ ├── config.go │ ├── grpc.go │ ├── grpc_test.go │ ├── utils.go │ └── utils_test.go └── version │ └── version.go ├── providers ├── awskms │ ├── go.mod │ ├── go.sum │ ├── localdev.md │ ├── main.go │ └── pkg │ │ └── awskms │ │ ├── awskms.go │ │ ├── awskms_test.go │ │ └── config.go ├── azurekms │ ├── go.mod │ ├── go.sum │ ├── localdev.md │ ├── main.go │ └── pkg │ │ └── azurekms │ │ └── config.go ├── debug │ ├── go.mod │ ├── go.sum │ ├── localdev.md │ └── main.go └── vault │ ├── go.mod │ ├── go.sum │ ├── localdev.md │ ├── main.go │ └── pkg │ └── vault │ ├── config.go │ ├── errors.go │ └── vault.go ├── proxy ├── go.mod ├── go.sum ├── main.go └── pkg │ └── server │ └── grpc.go ├── scripts ├── generic │ └── vault-kms-provider.yaml └── hcvault │ ├── archives │ ├── k8s │ │ ├── encryption-config.yaml │ │ ├── kube-apiserver.yaml │ │ ├── kubelet-config.yaml │ │ ├── trousseau-hcvault.yaml │ │ └── vault.yaml │ ├── monitoring │ │ ├── grafana-dashboard.yaml │ │ └── prometheus.yaml │ ├── rke2 │ │ ├── config.yaml │ │ └── trousseau-hcvault.yaml │ └── testing │ │ ├── encryption-config.yaml │ │ ├── kind-cluster.yaml │ │ ├── kms.yaml │ │ ├── secret.yaml │ │ ├── secret2.yaml │ │ └── test.bash │ └── hcvault-agent │ ├── trousseau-hcvault-configmap.yaml │ ├── trousseau-hcvault-daemonset.yaml │ ├── trousseau-hcvault-kube-apiserver.yaml │ └── trousseau-hcvault-serviceaccount.yaml ├── tests └── e2e │ └── kuttl │ ├── kube-v1.22 │ ├── aws-credentials.ini │ ├── awskms.yaml │ ├── azurekms.json │ ├── azurekms.yaml │ ├── encryption-config.yaml │ ├── kind.yaml │ ├── kuttl.yaml │ └── vault.yaml │ ├── kube-v1.23 │ ├── aws-credentials.ini │ ├── awskms.yaml │ ├── azurekms.json │ ├── azurekms.yaml │ ├── encryption-config.yaml │ ├── kind.yaml │ ├── kuttl.yaml │ └── vault.yaml │ ├── kube-v1.24 │ ├── aws-credentials.ini │ ├── awskms.yaml │ ├── azurekms.json │ ├── azurekms.yaml │ ├── encryption-config.yaml │ ├── kind.yaml │ ├── kuttl.yaml │ └── vault.yaml │ └── tests │ ├── 00-assert.yaml │ ├── 00-create-secret.yaml │ ├── 01-assert.yaml │ └── 01-restart-components.yaml └── trousseau ├── go.mod ├── go.sum ├── main.go └── pkg ├── health ├── health.go └── health_test.go └── server ├── grpc.go ├── grpc_test.go ├── sort.go └── sort_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | Taskfile.yml 2 | skaffold.yaml 3 | README.md 4 | .git 5 | .gitignore 6 | .gitattributes 7 | scripts 8 | bin 9 | .vscode 10 | .envrc 11 | deploy 12 | tests -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | [*] 3 | end_of_line = lf 4 | insert_final_newline = true 5 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.go text eol=lf -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Defect 3 | about: Defect or Support issues 4 | title: "[DEFECT] " 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Issue template is used for reporting defects or support issues. 11 | 12 | 13 | 14 | ## Detailed Description 15 | 16 | 17 | ## Expected Behavior 18 | 19 | 20 | ## Current Behavior 21 | 22 | 23 | ## Steps to Reproduce 24 | 25 | 26 | 1. 27 | 2. 28 | 3. 29 | 4. 30 | 31 | ## Context (Environment) 32 | 33 | 34 | 35 | ## Possible Solution/Implementation 36 | 37 | 38 | ## Possible PR 39 | 40 | 41 | --- 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/rfe.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: RFE 3 | about: Request for Enhancement description 4 | title: "[RFE] " 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | 12 | Is it linked to a user story, issue, defect? (use the "#" to tag the relevant entry) 13 | 14 | 15 | What do we want to build? 16 | 17 | 18 | Why do we want to build it? 19 | 20 | 21 | How do we want to design it? 22 | 23 | 24 | 25 | --- 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/task.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Task 3 | about: Task description based on an existing user story or not 4 | title: "[TASK] " 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Is it linked to a user story? (use the "#" to tag the user story) 11 | 12 | What do we want to build? 13 | 14 | Why do we want to build it? 15 | 16 | How do we want to design it? 17 | 18 | *Example*: 19 | 20 | Is it linked to a user story? 21 | #42 22 | 23 | What do we want to build? 24 | 25 | 26 | --- 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/user-stories.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: User Stories 3 | about: Feature description from an user perspective 4 | title: "[FEAT] " 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | User story: 11 | As a [persona], I [want to], [so that]. 12 | 13 | 14 | Acceptance criteria: 15 | Given [how things begin], when [action taken], then [outcome of taking action]. 16 | 17 | --- 18 | User story: 19 | Who are we building this feature for? Describe what you expect the feature to achieve, the end goal, what are the benefits of the feature and what is the challenge that needs solving? 20 | 21 | Acceptance criteria: 22 | Defines when the user story is considered completed/done. 23 | 24 | *Example*: 25 | 26 | User story: 27 | As a mobile user with no prior experience of the application, I want to login using my social network credentials either Github or Google, so that I don't know to manage additional password and security parameters and can login from any of my device passwordless. 28 | 29 | Acceptance criteria: 30 | Given that a new user, when I click register, the form will provide me with the option to use social network credentials, a Github and Google button will be displaye, then I can use my existing account on one of these platforms to seamlessly login without the need of creating an account with an email. 31 | --- 32 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Please include: 4 | - [ ] a summary of the change and which issue is fixed 5 | - [ ] relevant motivation and context 6 | - [ ] any dependencies that are required for this change 7 | 8 | Fixes # (issue) 9 | 10 | ## Type of change 11 | 12 | Please check options that are relevant. 13 | 14 | - [ ] Bug fix (non-breaking change which fixes an issue) 15 | - [ ] New feature (non-breaking change which adds functionality) 16 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 17 | - [ ] This change requires a documentation update 18 | 19 | # How Has This Been Tested? 20 | 21 | Please describe: 22 | - the tests that you ran to verify your changes 23 | - the instructions so we can reproduce 24 | - the list any relevant details for your test configuration 25 | 26 | **Test Configuration**: 27 | * Kubernetes distribution 28 | * Kubernetes version: 29 | * How many control plane node: 30 | * KMS: 31 | * KMS version: 32 | 33 | # Checklist: 34 | 35 | - [ ] My code follows the style guidelines of this project 36 | - [ ] I have performed a self-review of my own code 37 | - [ ] I have commented my code, particularly in hard-to-understand areas 38 | - [ ] I have made corresponding changes to the documentation 39 | - [ ] My changes generate no new warnings 40 | - [ ] I have added tests that prove my fix is effective or that my feature works 41 | - [ ] New and existing unit tests pass locally with my changes 42 | - [ ] Any dependent changes have been merged and published in downstream modules 43 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" 9 | directory: "/proxy" 10 | schedule: 11 | interval: "daily" 12 | open-pull-requests-limit: 10 13 | - package-ecosystem: "gomod" 14 | directory: "/providers/vault" 15 | schedule: 16 | interval: "daily" 17 | open-pull-requests-limit: 10 18 | - package-ecosystem: "gomod" 19 | directory: "/trousseau" 20 | schedule: 21 | interval: "daily" 22 | open-pull-requests-limit: 10 23 | -------------------------------------------------------------------------------- /.github/workflows/_docker-build.yml: -------------------------------------------------------------------------------- 1 | name: docker build 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | registry: 7 | required: true 8 | type: string 9 | imageName: 10 | required: true 11 | type: string 12 | imageTagPrefix: 13 | required: true 14 | type: string 15 | project: 16 | required: true 17 | type: string 18 | 19 | jobs: 20 | build-and-push: 21 | runs-on: ubuntu-latest 22 | permissions: 23 | contents: read 24 | packages: write 25 | 26 | steps: 27 | - name: Harden Runner 28 | uses: step-security/harden-runner@9b0655f430fba8c7001d4e38f8d4306db5c6e0ab 29 | with: 30 | egress-policy: audit 31 | 32 | - name: checkout repository 33 | uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 34 | 35 | - name: log in to ghrc.io 36 | uses: docker/login-action@1edf6180e07d2ffb423fc48a1a552855c0a1f508 37 | with: 38 | registry: ${{ inputs.registry }} 39 | username: ${{ github.actor }} 40 | password: ${{ secrets.GITHUB_TOKEN }} 41 | 42 | - name: build and container image 43 | uses: docker/build-push-action@9472e9021074a3cb3279ba431598b8836d40433f 44 | with: 45 | context: . 46 | push: true 47 | build-args: PROJECT=${{ inputs.project }} 48 | tags: ${{ inputs.registry }}/${{ inputs.imageName }}:${{ inputs.imageTagPrefix }}-${{ github.sha }} 49 | -------------------------------------------------------------------------------- /.github/workflows/_e2e-test.yml: -------------------------------------------------------------------------------- 1 | name: e2e test on pr 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | provider: 7 | required: true 8 | default: "debug" 9 | type: string 10 | kubever: 11 | required: true 12 | default: "1.24" 13 | type: string 14 | 15 | env: 16 | DOCKER_REGISTRY: local 17 | IMAGE_NAME: trousseau 18 | IMAGE_VERSION: e2e 19 | 20 | jobs: 21 | e2e: 22 | name: kuttl e2e 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v2 26 | - name: e2e test 27 | run: make go:e2e-tests:${{ inputs.provider }} KIND_CLUSTER_VERSION=${{ inputs.kubever }} 28 | 29 | -------------------------------------------------------------------------------- /.github/workflows/_gocilint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | project: 7 | required: true 8 | type: string 9 | 10 | permissions: 11 | contents: read 12 | pull-requests: read 13 | 14 | jobs: 15 | golangci: 16 | name: lint 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/setup-go@4a4352b33067e47da692b40ea6e19467075219ac 20 | with: 21 | go-version: '1.18' 22 | - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 23 | - name: golangci-lint 24 | uses: golangci/golangci-lint-action@c3ef0c370269e2a25b67c7f8e03d37e6cb106cb9 25 | with: 26 | version: latest 27 | working-directory: ${{ inputs.project }} 28 | -------------------------------------------------------------------------------- /.github/workflows/_gosecscan.yml: -------------------------------------------------------------------------------- 1 | name: "gosec" 2 | 3 | on: workflow_call 4 | 5 | permissions: 6 | contents: read 7 | pull-requests: read 8 | 9 | jobs: 10 | build: 11 | name: scan 12 | runs-on: ubuntu-latest 13 | env: 14 | GO111MODULE: on 15 | steps: 16 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 17 | - name: checkout repo 18 | uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 19 | 20 | - name: run gosec scan 21 | uses: securego/gosec@a64cde55a4499d951566243783f204e94b9197ed 22 | with: 23 | args: "./..." 24 | -------------------------------------------------------------------------------- /.github/workflows/_trivy.yml: -------------------------------------------------------------------------------- 1 | name: Vulnerability Scanning 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | registry: 7 | required: true 8 | type: string 9 | imageName: 10 | required: true 11 | type: string 12 | imageTagPrefix: 13 | required: true 14 | type: string 15 | 16 | jobs: 17 | build: 18 | name: Build 19 | runs-on: "ubuntu-latest" 20 | steps: 21 | - name: Run Trivy vulnerability scanner 22 | uses: aquasecurity/trivy-action@f39d29766a1eb7432c47f6bb7b64ed70b2241524 23 | with: 24 | image-ref: ${{ inputs.registry }}/${{ inputs.imageName }}:${{ inputs.imageTagPrefix }}-${{ github.sha }} 25 | format: 'template' 26 | template: '@/contrib/sarif.tpl' 27 | output: 'trivy-results.sarif' 28 | severity: 'CRITICAL,HIGH' 29 | 30 | - name: Upload Trivy scan results to GitHub Security tab 31 | uses: github/codeql-action/upload-sarif@1fc1008278d05ba9455caf083444e6c5a1a3cfd8 32 | with: 33 | sarif_file: 'trivy-results.sarif' 34 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | schedule: 16 | - cron: '26 16 * * 5' 17 | 18 | permissions: read-all 19 | 20 | jobs: 21 | analyze: 22 | name: Analyze 23 | runs-on: ubuntu-latest 24 | permissions: 25 | actions: read 26 | contents: read 27 | security-events: write 28 | 29 | strategy: 30 | fail-fast: false 31 | matrix: 32 | language: [ 'go' ] 33 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 34 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@1fc1008278d05ba9455caf083444e6c5a1a3cfd8 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@1fc1008278d05ba9455caf083444e6c5a1a3cfd8 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@1fc1008278d05ba9455caf083444e6c5a1a3cfd8 68 | -------------------------------------------------------------------------------- /.github/workflows/e2e-awskms-on-pr.yml: -------------------------------------------------------------------------------- 1 | name: AWS KMS e2e on pr 2 | 3 | on: 4 | pull_request: 5 | branches: [ main, v2* ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | pull-requests: read 11 | actions: read 12 | security-events: write 13 | packages: write 14 | 15 | concurrency: 16 | group: ci-e2e-awskms-${{ github.ref }}-1 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | golangci-lint: 21 | uses: ./.github/workflows/_gocilint.yml 22 | with: 23 | project: providers/awskms 24 | 25 | gosec-scanning: 26 | uses: ./.github/workflows/_gosecscan.yml 27 | needs: golangci-lint 28 | 29 | image-build: 30 | uses: ./.github/workflows/_docker-build.yml 31 | with: 32 | registry: ghcr.io 33 | imageName: ${{ github.repository }} 34 | imageTagPrefix: awskms 35 | project: providers/awskms 36 | needs: gosec-scanning 37 | 38 | e2e-1_22: 39 | uses: ./.github/workflows/_e2e-test.yml 40 | with: 41 | provider: awskms 42 | kubever: "1.22" 43 | needs: image-build 44 | 45 | e2e-1_23: 46 | uses: ./.github/workflows/_e2e-test.yml 47 | with: 48 | provider: awskms 49 | kubever: "1.23" 50 | needs: image-build 51 | 52 | e2e-1_24: 53 | uses: ./.github/workflows/_e2e-test.yml 54 | with: 55 | provider: awskms 56 | kubever: "1.24" 57 | needs: image-build 58 | 59 | image-vulnerability-scan: 60 | uses: ./.github/workflows/_trivy.yml 61 | with: 62 | registry: ghcr.io 63 | imageName: ${{ github.repository }} 64 | imageTagPrefix: awskms 65 | needs: image-build -------------------------------------------------------------------------------- /.github/workflows/e2e-azurekms-on-pr.yml: -------------------------------------------------------------------------------- 1 | name: Azure KMS e2e on pr 2 | 3 | on: 4 | pull_request: 5 | branches: [ main, v2* ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | pull-requests: read 11 | actions: read 12 | security-events: write 13 | packages: write 14 | 15 | concurrency: 16 | group: ci-e2e-azurekms-${{ github.ref }}-1 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | golangci-lint: 21 | uses: ./.github/workflows/_gocilint.yml 22 | with: 23 | project: providers/azurekms 24 | 25 | gosec-scanning: 26 | uses: ./.github/workflows/_gosecscan.yml 27 | needs: golangci-lint 28 | 29 | image-build: 30 | uses: ./.github/workflows/_docker-build.yml 31 | with: 32 | registry: ghcr.io 33 | imageName: ${{ github.repository }} 34 | imageTagPrefix: azurekms 35 | project: providers/azurekms 36 | needs: gosec-scanning 37 | 38 | # e2e-1_22: 39 | # uses: ./.github/workflows/_e2e-test.yml 40 | # with: 41 | # provider: azurekms 42 | # kubever: "1.22" 43 | # needs: image-build 44 | 45 | # e2e-1_23: 46 | # uses: ./.github/workflows/_e2e-test.yml 47 | # with: 48 | # provider: azurekms 49 | # kubever: "1.23" 50 | # needs: image-build 51 | 52 | # e2e-1_24: 53 | # uses: ./.github/workflows/_e2e-test.yml 54 | # with: 55 | # provider: azurekms 56 | # kubever: "1.24" 57 | # needs: image-build 58 | 59 | image-vulnerability-scan: 60 | uses: ./.github/workflows/_trivy.yml 61 | with: 62 | registry: ghcr.io 63 | imageName: ${{ github.repository }} 64 | imageTagPrefix: azurekms 65 | needs: image-build -------------------------------------------------------------------------------- /.github/workflows/e2e-debug-on-pr.yml: -------------------------------------------------------------------------------- 1 | name: Debug e2e on pr 2 | 3 | on: 4 | pull_request: 5 | branches: [ main, v2* ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | pull-requests: read 11 | actions: read 12 | security-events: write 13 | packages: write 14 | 15 | concurrency: 16 | group: ci-e2e-debug-${{ github.ref }}-1 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | golangci-lint: 21 | uses: ./.github/workflows/_gocilint.yml 22 | with: 23 | project: providers/debug 24 | 25 | gosec-scanning: 26 | uses: ./.github/workflows/_gosecscan.yml 27 | needs: golangci-lint 28 | 29 | image-build: 30 | uses: ./.github/workflows/_docker-build.yml 31 | with: 32 | registry: ghcr.io 33 | imageName: ${{ github.repository }} 34 | imageTagPrefix: debug 35 | project: providers/debug 36 | needs: gosec-scanning 37 | 38 | image-vulnerability-scan: 39 | uses: ./.github/workflows/_trivy.yml 40 | with: 41 | registry: ghcr.io 42 | imageName: ${{ github.repository }} 43 | imageTagPrefix: debug 44 | needs: image-build -------------------------------------------------------------------------------- /.github/workflows/e2e-proxy-on-pr.yml: -------------------------------------------------------------------------------- 1 | name: Proxy e2e on pr 2 | 3 | on: 4 | pull_request: 5 | branches: [ main, v2* ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | pull-requests: read 11 | actions: read 12 | security-events: write 13 | packages: write 14 | 15 | concurrency: 16 | group: ci-e2e-proxy-${{ github.ref }}-1 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | golangci-lint: 21 | uses: ./.github/workflows/_gocilint.yml 22 | with: 23 | project: proxy 24 | 25 | gosec-scanning: 26 | uses: ./.github/workflows/_gosecscan.yml 27 | needs: golangci-lint 28 | 29 | image-build: 30 | uses: ./.github/workflows/_docker-build.yml 31 | with: 32 | registry: ghcr.io 33 | imageName: ${{ github.repository }} 34 | imageTagPrefix: proxy 35 | project: proxy 36 | needs: gosec-scanning 37 | 38 | image-vulnerability-scan: 39 | uses: ./.github/workflows/_trivy.yml 40 | with: 41 | registry: ghcr.io 42 | imageName: ${{ github.repository }} 43 | imageTagPrefix: proxy 44 | needs: image-build -------------------------------------------------------------------------------- /.github/workflows/e2e-trousseau-on-pr.yml: -------------------------------------------------------------------------------- 1 | name: Trousseau e2e on pr 2 | 3 | on: 4 | pull_request: 5 | branches: [ main, v2* ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | pull-requests: read 11 | actions: read 12 | security-events: write 13 | packages: write 14 | 15 | concurrency: 16 | group: ci-e2e-trousseau-${{ github.ref }}-1 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | golangci-lint: 21 | uses: ./.github/workflows/_gocilint.yml 22 | with: 23 | project: trousseau 24 | 25 | gosec-scanning: 26 | uses: ./.github/workflows/_gosecscan.yml 27 | needs: golangci-lint 28 | 29 | image-build: 30 | uses: ./.github/workflows/_docker-build.yml 31 | with: 32 | registry: ghcr.io 33 | imageName: ${{ github.repository }} 34 | imageTagPrefix: trousseau 35 | project: trousseau 36 | needs: gosec-scanning 37 | 38 | e2e-1_22: 39 | uses: ./.github/workflows/_e2e-test.yml 40 | with: 41 | provider: debug 42 | kubever: "1.22" 43 | needs: image-build 44 | 45 | e2e-1_23: 46 | uses: ./.github/workflows/_e2e-test.yml 47 | with: 48 | provider: debug 49 | kubever: "1.23" 50 | needs: image-build 51 | 52 | e2e-1_24: 53 | uses: ./.github/workflows/_e2e-test.yml 54 | with: 55 | provider: debug 56 | kubever: "1.24" 57 | needs: image-build 58 | 59 | image-vulnerability-scan: 60 | uses: ./.github/workflows/_trivy.yml 61 | with: 62 | registry: ghcr.io 63 | imageName: ${{ github.repository }} 64 | imageTagPrefix: trousseau 65 | needs: image-build -------------------------------------------------------------------------------- /.github/workflows/e2e-vault-on-pr.yml: -------------------------------------------------------------------------------- 1 | name: Vault e2e on pr 2 | 3 | on: 4 | pull_request: 5 | branches: [ main, v2* ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | pull-requests: read 11 | actions: read 12 | security-events: write 13 | packages: write 14 | 15 | concurrency: 16 | group: ci-e2e-vault-${{ github.ref }}-1 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | golangci-lint: 21 | uses: ./.github/workflows/_gocilint.yml 22 | with: 23 | project: providers/vault 24 | 25 | gosec-scanning: 26 | uses: ./.github/workflows/_gosecscan.yml 27 | needs: golangci-lint 28 | 29 | image-build: 30 | uses: ./.github/workflows/_docker-build.yml 31 | with: 32 | registry: ghcr.io 33 | imageName: ${{ github.repository }} 34 | imageTagPrefix: vault 35 | project: providers/vault 36 | needs: gosec-scanning 37 | 38 | e2e-1_22: 39 | uses: ./.github/workflows/_e2e-test.yml 40 | with: 41 | provider: vault 42 | kubever: "1.22" 43 | needs: image-build 44 | 45 | e2e-1_23: 46 | uses: ./.github/workflows/_e2e-test.yml 47 | with: 48 | provider: vault 49 | kubever: "1.23" 50 | needs: image-build 51 | 52 | e2e-1_24: 53 | uses: ./.github/workflows/_e2e-test.yml 54 | with: 55 | provider: vault 56 | kubever: "1.24" 57 | needs: image-build 58 | 59 | image-vulnerability-scan: 60 | uses: ./.github/workflows/_trivy.yml 61 | with: 62 | registry: ghcr.io 63 | imageName: ${{ github.repository }} 64 | imageTagPrefix: vault 65 | needs: image-build -------------------------------------------------------------------------------- /.github/workflows/generator-test-on-pr.yml: -------------------------------------------------------------------------------- 1 | name: Generator test on pr 2 | 3 | on: 4 | pull_request: 5 | branches: [ main, v2* ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: read 10 | pull-requests: read 11 | actions: read 12 | security-events: write 13 | packages: write 14 | 15 | concurrency: 16 | group: ci-generator-${{ github.ref }}-1 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | docker-compose: 21 | name: docker compose 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v2 25 | - uses: "finnp/create-file-action@master" 26 | env: 27 | FILE_NAME: "trousseau-env" 28 | FILE_DATA: | 29 | TR_VERBOSE_LEVEL=3 30 | TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault" 31 | TR_SOCKET_LOCATION=${PWD}/bin/run 32 | TR_PROXY_IMAGE=ondat/trousseau:proxy-develop 33 | TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop 34 | TR_AZUREKMS_IMAGE=ondat/trousseau:azurekms-develop 35 | TR_VAULT_IMAGE=ondat/trousseau:vault-develop 36 | TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop 37 | TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials 38 | TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml 39 | TR_AZUREKMS_CREDENTIALS=${HOME}/.azure/config.json 40 | TR_AZUREKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/azurekms.yaml 41 | TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml 42 | - name: generate services 43 | run: make prod:generate:docker-compose ENV_LOCATION=trousseau-env 44 | kustomize: 45 | name: kustomize 46 | runs-on: ubuntu-latest 47 | steps: 48 | - uses: actions/checkout@v2 49 | - uses: "finnp/create-file-action@master" 50 | env: 51 | FILE_NAME: "awskms.yaml" 52 | FILE_DATA: | 53 | profile: default 54 | - uses: "finnp/create-file-action@master" 55 | env: 56 | FILE_NAME: "trousseau-env" 57 | FILE_DATA: | 58 | TR_VERBOSE_LEVEL=3 59 | TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault" 60 | TR_SOCKET_LOCATION=${PWD}/bin/run 61 | TR_PROXY_IMAGE=ondat/trousseau:proxy-develop 62 | TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop 63 | TR_AZUREKMS_IMAGE=ondat/trousseau:azurekms-develop 64 | TR_VAULT_IMAGE=ondat/trousseau:vault-develop 65 | TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop 66 | TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials 67 | TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml 68 | TR_AZUREKMS_CREDENTIALS=${HOME}/.azure/config.json 69 | TR_AZUREKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/azurekms.yaml 70 | TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml 71 | TR_VAULT_ADDRESS=http://127.0.0.1:8200 72 | - name: generate services 73 | run: make prod:generate:kustomize ENV_LOCATION=trousseau-env 74 | helm: 75 | name: helm 76 | runs-on: ubuntu-latest 77 | steps: 78 | - uses: actions/checkout@v2 79 | - uses: "finnp/create-file-action@master" 80 | env: 81 | FILE_NAME: "awskms.yaml" 82 | FILE_DATA: | 83 | profile: default 84 | - uses: "finnp/create-file-action@master" 85 | env: 86 | FILE_NAME: "trousseau-env" 87 | FILE_DATA: | 88 | TR_VERBOSE_LEVEL=3 89 | TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault" 90 | TR_SOCKET_LOCATION=${PWD}/bin/run 91 | TR_PROXY_IMAGE=ondat/trousseau:proxy-develop 92 | TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop 93 | TR_AZUREKMS_IMAGE=ondat/trousseau:azurekms-develop 94 | TR_VAULT_IMAGE=ondat/trousseau:vault-develop 95 | TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop 96 | TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials 97 | TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml 98 | TR_AZUREKMS_CREDENTIALS=${HOME}/.azure/config.json 99 | TR_AZUREKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/azurekms.yaml 100 | TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml 101 | TR_VAULT_ADDRESS=http://127.0.0.1:8200 102 | - name: generate services 103 | run: make prod:generate:helm ENV_LOCATION=trousseau-env 104 | -------------------------------------------------------------------------------- /.github/workflows/scorecards-analysis.yml: -------------------------------------------------------------------------------- 1 | name: Scorecards supply-chain security 2 | on: 3 | # Only the default branch is supported. 4 | branch_protection_rule: 5 | schedule: 6 | - cron: '28 1 * * 5' 7 | push: 8 | branches: [ main, v2* ] 9 | 10 | # Declare default permissions as read only. 11 | permissions: read-all 12 | 13 | jobs: 14 | analysis: 15 | name: Scorecards analysis 16 | runs-on: ubuntu-latest 17 | permissions: 18 | # Needed to upload the results to code-scanning dashboard. 19 | security-events: write 20 | actions: read 21 | contents: read 22 | 23 | steps: 24 | - name: "Checkout code" 25 | uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2.4.0 26 | with: 27 | persist-credentials: false 28 | 29 | - name: "Run analysis" 30 | uses: ossf/scorecard-action@c8416b0b2bf627c349ca92fc8e3de51a64b005cf # v1.0.2 31 | with: 32 | results_file: results.sarif 33 | results_format: sarif 34 | # Read-only PAT token. To create it, 35 | # follow the steps in https://github.com/ossf/scorecard-action#pat-token-creation. 36 | repo_token: ${{ secrets.SCORECARD_READ_TOKEN }} 37 | # Publish the results to enable scorecard badges. For more details, see 38 | # https://github.com/ossf/scorecard-action#publishing-results. 39 | # For private repositories, `publish_results` will automatically be set to `false`, 40 | # regardless of the value entered here. 41 | publish_results: true 42 | 43 | # Upload the results as artifacts (optional). 44 | - name: "Upload artifact" 45 | uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 # v2.3.1 46 | with: 47 | name: SARIF file 48 | path: results.sarif 49 | retention-days: 5 50 | 51 | # Upload the results to GitHub's code scanning dashboard. 52 | - name: "Upload to code-scanning" 53 | uses: github/codeql-action/upload-sarif@5f532563584d71fdef14ee64d17bafb34f751ce5 # v1.0.26 54 | with: 55 | sarif_file: results.sarif 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | *.socket 3 | cover.out 4 | kind-logs-*/ 5 | kubeconfig 6 | generated_manifests/ -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | errcheck: 3 | check-type-assertions: true 4 | goconst: 5 | min-len: 2 6 | min-occurrences: 3 7 | gocritic: 8 | enabled-tags: 9 | - diagnostic 10 | - experimental 11 | - opinionated 12 | - performance 13 | - style 14 | govet: 15 | check-shadowing: true 16 | nolintlint: 17 | require-explanation: true 18 | require-specific: true 19 | 20 | linters: 21 | disable-all: true 22 | enable: 23 | - deadcode 24 | - depguard 25 | - dogsled 26 | - dupl 27 | - errcheck 28 | - exportloopref 29 | - exhaustive 30 | - goconst 31 | - gocritic 32 | - gofmt 33 | - goimports 34 | - gomnd 35 | - gocyclo 36 | - gosec 37 | - gosimple 38 | - govet 39 | - ineffassign 40 | - misspell 41 | - nolintlint 42 | - nakedret 43 | - prealloc 44 | - predeclared 45 | - revive 46 | - staticcheck 47 | - stylecheck 48 | - thelper 49 | - typecheck 50 | - unconvert 51 | - varcheck 52 | - whitespace 53 | - wsl 54 | 55 | run: 56 | issues-exit-code: 0 57 | -------------------------------------------------------------------------------- /.husky/hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | husky install -------------------------------------------------------------------------------- /.husky/hooks/pre-push: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ "$SKIP_GIT_PUSH_HOOK" ]]; then exit 0; fi 4 | 5 | set -e 6 | 7 | if git status --short | grep -qv "??"; then 8 | git stash 9 | function unstash() { 10 | git reset --hard 11 | git stash pop 12 | } 13 | trap unstash EXIT 14 | fi 15 | 16 | task go:fmt go:vet go:gosec go:golangci go:unit-tests 17 | 18 | git diff --exit-code --quiet || (git status && exit 1) 19 | -------------------------------------------------------------------------------- /.task/cluster.yml: -------------------------------------------------------------------------------- 1 | version: 3 2 | tasks: 3 | create: 4 | desc: create kind cluster 5 | deps: 6 | - delete 7 | - :fetch:kind 8 | cmds: 9 | - ./bin/kind create cluster --retain --name "{{.KIND_CLUSTER_NAME}}" --wait 2m --config ./tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/kind.yaml 10 | delete: 11 | desc: destroy kind cluster 12 | deps: 13 | - :fetch:kind 14 | cmds: 15 | - ./bin/kind delete cluster --name "{{.KIND_CLUSTER_NAME}}" -------------------------------------------------------------------------------- /.task/docker.yml: -------------------------------------------------------------------------------- 1 | version: 3 2 | vars: 3 | NOW: 4 | sh: date +'%s' 5 | BASE_IMAGE: 6 | sh: '([ -z "$BASE_IMAGE" ] && head -1 Dockerfile | cut -d= -f2) || echo $BASE_IMAGE' 7 | tasks: 8 | build: 9 | desc: build docker images 10 | cmds: 11 | - task: build:proxy 12 | - task: build:debug 13 | - task: build:vault 14 | - task: build:awskms 15 | - task: build:azurekms 16 | - task: build:trousseau 17 | build:proxy: 18 | cmds: 19 | - docker build --label buildtime={{.NOW}} --build-arg BASE_IMAGE={{.BASE_IMAGE}} --build-arg PROJECT=proxy -t $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION . 20 | status: 21 | - test "{{.NOW}}" == "$(docker inspect $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION --format='{{"{{"}}.Config.Labels.buildtime{{"}}"}}' 2>/dev/null)" 22 | build:debug: 23 | cmds: 24 | - docker build --label buildtime={{.NOW}} --build-arg BASE_IMAGE={{.BASE_IMAGE}} --build-arg PROJECT=providers/debug -t $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION . 25 | status: 26 | - test "{{.NOW}}" == "$(docker inspect $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION --format='{{"{{"}}.Config.Labels.buildtime{{"}}"}}' 2>/dev/null)" 27 | build:vault: 28 | cmds: 29 | - docker build --label buildtime={{.NOW}} --build-arg BASE_IMAGE={{.BASE_IMAGE}} --build-arg PROJECT=providers/vault -t $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION . 30 | status: 31 | - test "{{.NOW}}" == "$(docker inspect $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION --format='{{"{{"}}.Config.Labels.buildtime{{"}}"}}' 2>/dev/null)" 32 | build:awskms: 33 | cmds: 34 | - docker build --label buildtime={{.NOW}} --build-arg BASE_IMAGE={{.BASE_IMAGE}} --build-arg PROJECT=providers/awskms -t $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION . 35 | status: 36 | - test "{{.NOW}}" == "$(docker inspect $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION --format='{{"{{"}}.Config.Labels.buildtime{{"}}"}}' 2>/dev/null)" 37 | build:azurekms: 38 | cmds: 39 | - docker build --label buildtime={{.NOW}} --build-arg BASE_IMAGE={{.BASE_IMAGE}} --build-arg PROJECT=providers/azurekms -t $DOCKER_REGISTRY/$IMAGE_NAME:azurekms-$IMAGE_VERSION . 40 | status: 41 | - test "{{.NOW}}" == "$(docker inspect $DOCKER_REGISTRY/$IMAGE_NAME:azurekms-$IMAGE_VERSION --format='{{"{{"}}.Config.Labels.buildtime{{"}}"}}' 2>/dev/null)" 42 | build:trousseau: 43 | cmds: 44 | - docker build --label buildtime={{.NOW}} --build-arg BASE_IMAGE={{.BASE_IMAGE}} --build-arg PROJECT=trousseau -t $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION . 45 | status: 46 | - test "{{.NOW}}" == "$(docker inspect $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION --format='{{"{{"}}.Config.Labels.buildtime{{"}}"}}' 2>/dev/null)" 47 | push: 48 | desc: push docker image 49 | cmds: 50 | - task: push:proxy 51 | - task: push:debug 52 | - task: push:vault 53 | - task: push:awskms 54 | - task: push:azurekms 55 | - task: push:trousseau 56 | push:proxy: 57 | cmds: 58 | - docker push $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION 59 | push:debug: 60 | cmds: 61 | - docker push $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION 62 | push:vault: 63 | cmds: 64 | - docker push $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION 65 | push:awskms: 66 | cmds: 67 | - docker push $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION 68 | push:azurekms: 69 | cmds: 70 | - docker push $DOCKER_REGISTRY/$IMAGE_NAME:azurekms-$IMAGE_VERSION 71 | push:trousseau: 72 | cmds: 73 | - docker push $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION 74 | run: 75 | desc: run docker image 76 | cmds: 77 | - task: run:proxy 78 | - task: run:debug 79 | - task: run:vault 80 | - task: run:awskms 81 | - task: run:azurekms 82 | - task: run:trousseau 83 | run:proxy: 84 | deps: 85 | - :run-dir:init 86 | cmds: 87 | - docker rm -f trousseau-proxy || true 88 | - docker run -d --name trousseau-proxy --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION 89 | run:debug: 90 | deps: 91 | - :run-dir:init 92 | cmds: 93 | - docker rm -f trousseau-debug || true 94 | - docker run -d --name trousseau-debug --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION 95 | run:vault: 96 | deps: 97 | - :run-dir:init 98 | cmds: 99 | - docker rm -f trousseau-local-vault || true 100 | - docker run -d --name=trousseau-local-vault --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=vault-kms-demo' vault 101 | - sleep 5 102 | - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault login vault-kms-demo 103 | - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault secrets enable transit 104 | - docker rm -f trousseau-vault || true 105 | - docker run -d --name trousseau-vault --rm --network=container:trousseau-local-vault -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/vault.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3 106 | run:awskms: 107 | deps: 108 | - :run-dir:init 109 | cmds: 110 | - docker rm -f trousseau-local-aws || true 111 | - docker run --name trousseau-local-aws --rm --hostname localhost.localstack.cloud -d -e SERVICES=kms -e HOSTNAME=localhost.localstack.cloud -e HOSTNAME_EXTERNAL=localhost.localstack.cloud -e DEFAULT_REGION=eu-west-1 -e KMS_PROVIDER=kms-local -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:0.14.4 112 | - sleep 5 113 | - 'printf %"s\n" "endpoint: https://localhost.localstack.cloud:4566" "profile: trousseau-local-aws" "keyArn: $(docker exec trousseau-local-aws awslocal kms create-key | grep Arn | cut -d''"'' -f4)" > tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml' 114 | - docker rm -f trousseau-awskms || true 115 | - docker run -d --name trousseau-awskms --rm --network=container:trousseau-local-aws -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/aws-credentials.ini:/.aws/credentials -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3 116 | run:azurekms: 117 | deps: 118 | - :run-dir:init 119 | cmds: 120 | - docker rm -f trousseau-azurekms || true 121 | - docker run -d --name trousseau-azurekms --rm -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/azurekms.json:$PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/azurekms.json -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/azurekms.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:azurekms-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3 122 | run:trousseau: 123 | deps: 124 | - :run-dir:init 125 | cmds: 126 | - docker rm -f trousseau-core || true 127 | - docker run -d --name trousseau-core --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION {{.ENABLED_PROVIDERS}} -v=3 128 | -------------------------------------------------------------------------------- /.task/fetch.yml: -------------------------------------------------------------------------------- 1 | version: 3 2 | vars: 3 | KIND_VERSION: 0.14.0 4 | GOSEC_VERSION: 2.11.0 5 | GOLANGCI_VERSION: 1.47.2 6 | HELM_VERSION: 3.6.3 7 | VAULT_VERSION: 1.8.1 8 | KUBECTL_VERSION: 1.21.1 9 | KUTTL_VERSION: 0.12.1 10 | ENVSUBST_VERSION: 1.2.0 11 | HUSKY_VERSION: 0.2.14 12 | CAP_ARCH: 13 | sh: uname 14 | tasks: 15 | golangci: 16 | deps: 17 | - :bin-dir:init 18 | desc: install golanci 19 | cmds: 20 | - curl -L https://github.com/golangci/golangci-lint/releases/download/v{{.GOLANGCI_VERSION}}/golangci-lint-{{.GOLANGCI_VERSION}}-{{OS}}-{{ARCH}}.tar.gz | tar xvz --one-top-level=golangcitmp 21 | - mv golangcitmp/golangci-lint-{{.GOLANGCI_VERSION}}-{{OS}}-{{ARCH}}/golangci-lint ./bin/golangci-lint 22 | - chmod 755 bin/golangci-lint 23 | - rm -rf golangcitmp 24 | status: 25 | - test -f ./bin/golangci-lint 26 | gosec: 27 | deps: 28 | - :bin-dir:init 29 | desc: install gosec 30 | cmds: 31 | - curl -L https://github.com/securego/gosec/releases/download/v{{.GOSEC_VERSION}}/gosec_{{.GOSEC_VERSION}}_{{OS}}_{{ARCH}}.tar.gz | tar xvz --one-top-level=gosectmp 32 | - mv gosectmp/gosec ./bin/gosec 33 | - chmod 755 bin/gosec 34 | - rm -rf gosectmp 35 | status: 36 | - test -f ./bin/gosec 37 | kind: 38 | deps: 39 | - :bin-dir:init 40 | desc: install kind 41 | cmds: 42 | - curl -L https://github.com/kubernetes-sigs/kind/releases/download/v{{.KIND_VERSION}}/kind-{{OS}}-{{ARCH}} --output ./bin/kind && chmod +x ./bin/kind 43 | status: 44 | - test -f ./bin/kind 45 | helm: 46 | deps: 47 | - :bin-dir:init 48 | desc: install helm 49 | cmds: 50 | - curl -L https://get.helm.sh/helm-v{{.HELM_VERSION}}-{{OS}}-{{ARCH}}.tar.gz | tar xvz -C ./ 51 | - mv {{OS}}-{{ARCH}}/helm ./bin/helm 52 | - chmod 755 bin/helm 53 | - rm -rf {{OS}}-{{ARCH}} 54 | status: 55 | - test -f ./bin/helm 56 | vault: 57 | desc: install vault 58 | cmds: 59 | - curl "https://releases.hashicorp.com/vault/{{.VAULT_VERSION}}/vault_{{.VAULT_VERSION}}_{{OS}}_{{ARCH}}.zip" -o "vault.zip" 60 | - unzip vault.zip 61 | - mv vault bin/vault 62 | - chmod 755 bin/vault 63 | - rm vault.zip 64 | status: 65 | - test -f bin/vault 66 | kubectl: 67 | deps: 68 | - :bin-dir:init 69 | desc: install kubectl 70 | cmds: 71 | - curl -Lo ./bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v{{.KUBECTL_VERSION}}/bin/{{OS}}/{{ARCH}}/kubectl 72 | - chmod +x ./bin/kubectl 73 | status: 74 | - test -f ./bin/kubectl 75 | kuttl: 76 | deps: 77 | - :bin-dir:init 78 | desc: install kuttl 79 | cmds: 80 | - cd bin ; curl -L https://github.com/kudobuilder/kuttl/releases/download/v{{.KUTTL_VERSION}}/kuttl_{{.KUTTL_VERSION}}_{{OS}}_x86_64.tar.gz | tar -xz kubectl-kuttl 81 | status: 82 | - test -f ./bin/kuttl 83 | envsubst: 84 | deps: 85 | - :bin-dir:init 86 | desc: install envsubst 87 | cmds: 88 | - cd bin ; curl -o envsubst -L https://github.com/a8m/envsubst/releases/download/v{{.ENVSUBST_VERSION}}/envsubst-{{.CAP_ARCH}}-x86_64 89 | - chmod +x ./bin/envsubst 90 | status: 91 | - test -f ./bin/envsubst 92 | husky: 93 | deps: 94 | - :bin-dir:init 95 | desc: install husky 96 | cmds: 97 | - cd bin ; curl -L https://github.com/automation-co/husky/releases/download/v{{.HUSKY_VERSION}}/husky_{{.HUSKY_VERSION}}_{{.CAP_ARCH}}_x86_64.tar.gz | tar -xz husky 98 | - chmod +x ./bin/husky 99 | - ./bin/husky install 100 | status: 101 | - test -f ./bin/husky 102 | -------------------------------------------------------------------------------- /.task/prod.yml: -------------------------------------------------------------------------------- 1 | version: 3 2 | silent: true 3 | vars: 4 | SCRIPT: scripts/hcvault/archives/testing 5 | ENV_LOCATION: '{{.ENV_LOCATION | default "/please/set/ENV_LOCATION"}}' 6 | tasks: 7 | gen-dir:init: 8 | desc: create bin directory 9 | cmds: 10 | - mkdir -p generated_manifests 11 | status: 12 | - test -d generated_manifests 13 | prometheus:deploy: 14 | deps: 15 | - task: :fetch:helm 16 | - task: :fetch:kubectl 17 | desc: install prometheus and grafana on cluster 18 | cmds: 19 | - ./bin/helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 20 | - ./bin/helm upgrade --install prometheus prometheus-community/kube-prometheus-stack --wait 21 | - ./bin/kubectl apply -f {{.SCRIPT}}/prometheus.yaml --wait 22 | - ./bin/kubectl apply -f {{.SCRIPT}}/grafana-dashboard.yaml --wait 23 | prometheus:port-forward: 24 | deps: 25 | - task: :fetch:kubectl 26 | desc: port forwarding for prometheus 27 | cmds: 28 | - ./bin/kubectl port-forward svc/prometheus-kube-prometheus-prometheus 9090 29 | grafana:port-forward: 30 | deps: 31 | - task: :fetch:kubectl 32 | desc: open grafana (admin/prom-operator) 33 | cmds: 34 | - ./bin/kubectl port-forward svc/prometheus-grafana 8300:80 35 | generate:systemd: 36 | desc: generate systemd services 37 | deps: 38 | - gen-dir:init 39 | cmds: 40 | - rm -rf generated_manifests/systemd/* 41 | - cp -rf deployment/systemd generated_manifests 42 | - cp -f {{.ENV_LOCATION}} generated_manifests/systemd/trousseau-env 43 | generate:docker-compose: 44 | desc: generate docker compose services 45 | deps: 46 | - gen-dir:init 47 | - :fetch:envsubst 48 | cmds: 49 | - rm -rf generated_manifests/docker-compose/* ; mkdir -p generated_manifests/docker-compose 50 | - source {{.ENV_LOCATION}} ; 51 | export $(echo "${!TR_*}") ; 52 | for f in `cd deployment ; find docker-compose -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done ; 53 | (cd generated_manifests/docker-compose ; docker compose -f docker-compose.yaml -f docker-compose.override.awskms.yaml -f docker-compose.override.azurekms.yaml -f docker-compose.override.vault.yaml config 1>/dev/null) 54 | generate:kustomize: 55 | desc: generate kustomize manifests 56 | deps: 57 | - gen-dir:init 58 | - :fetch:envsubst 59 | cmds: 60 | - rm -rf generated_manifests/kustomize/* ; mkdir -p generated_manifests/kustomize 61 | - source {{.ENV_LOCATION}} ; 62 | TR_ENABLED_PROVIDERS=$(echo ${TR_ENABLED_PROVIDERS} | sed "s/ --/\n - --/") ; 63 | test -n "${TR_AWSKMS_CONFIG}" && TR_AWSKMS_CONFIG=$(cat ${TR_AWSKMS_CONFIG} 2>/dev/null | sed 's/^/ /') ; 64 | test -n "${TR_AZUREKMS_CONFIG}" && TR_AZUREKMS_CONFIG=$(cat ${TR_AZUREKMS_CONFIG} 2>/dev/null | sed 's/^/ /') ; 65 | export $(echo "${!TR_*}") ; 66 | for f in `cd deployment ; find kustomize -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done ; 67 | docker run --rm -v $PWD/generated_manifests/kustomize:/work -w /work nixery.dev/shell/kustomize/kubeval sh -c 'kustomize build | kubeval' 68 | generate:helm: 69 | desc: generate helm manifests 70 | deps: 71 | - gen-dir:init 72 | - :fetch:envsubst 73 | cmds: 74 | - rm -rf generated_manifests/helm/* 75 | - cp -rf deployment/helm generated_manifests 76 | - source {{.ENV_LOCATION}} ; 77 | test -n "${TR_AWSKMS_CONFIG}" && cat ${TR_AWSKMS_CONFIG} | sed 's/^/ /' > generated_manifests/helm/awsconfig.yaml ; 78 | TR_AWSKMS_CONFIG=awsconfig.yaml; 79 | test -n "${TR_AZUREKMS_CONFIG}" && cat ${TR_AZUREKMS_CONFIG} | sed 's/^/ /' > generated_manifests/helm/azureconfig.yaml ; 80 | TR_AZUREKMS_CONFIG=azureconfig.yaml; 81 | TR_ENABLED_PROVIDERS=$(echo ${TR_ENABLED_PROVIDERS} | sed "s/ --/\n - --/") ; 82 | export $(echo "${!TR_*}") ; 83 | ./bin/envsubst -no-empty -i deployment/helm/values.yaml -o generated_manifests/helm/values.yaml ; 84 | docker run --rm -v $PWD/generated_manifests/helm:/work -w /work nixery.dev/shell/kubernetes-helm sh -c 'helm lint && helm template ../work 1>/dev/null' 85 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | We follow the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | 5 | Please contact git@ondat.io to report an issue. 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=gcr.io/distroless/static@sha256:957bbd91e4bfe8186bd218c08b2bbc5c852e6ebe6a7b2dcc42a86b22ea2b6bb6 2 | 3 | FROM golang@sha256:5b75b529da0f2196ee8561a90e5b99aceee56e125c6ef09a3da4e32cf3cc6c20 AS build 4 | ARG PROJECT=required 5 | ADD . /work 6 | WORKDIR /work/${PROJECT} 7 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -o endpoint main.go 8 | 9 | FROM alpine:3.16.0 as certs 10 | RUN apk --update add ca-certificates 11 | 12 | FROM ${BASE_IMAGE} 13 | ARG PROJECT=required 14 | LABEL org.opencontainers.image.title "Trousseau - ${PROJECT}" 15 | LABEL org.opencontainers.image.vendor "Trousseau.io" 16 | LABEL org.opencontainers.image.licenses "Apache-2.0 License" 17 | LABEL org.opencontainers.image.source "https://github.com/ondat/trousseau" 18 | LABEL org.opencontainers.image.description "Trousseau, an open-source project leveraging the Kubernetes KMS provider framework to connect any Key Management Service the Kubernetes native way" 19 | LABEL org.opencontainers.image.documentation "https://github.com/ondat/trousseau/wiki" 20 | 21 | COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 22 | 23 | COPY --from=build /work/${PROJECT}/endpoint /bin/ 24 | 25 | USER 10123:10123 26 | 27 | ENTRYPOINT ["/bin/endpoint"] 28 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Maintainers 2 | 3 | * Romuald Vandepoel [@rovandep](https://github.com/rovandep) 4 | * Richard Kovacs [mhmxs](https://github.com/mhmxs) 5 | * Cannis Chan [@cannischan](https://github.com/cannischan) 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | %: 2 | mkdir -p bin && test -f ./bin/task || (cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task) 3 | 4 | PATH=./bin:$(PATH) ./bin/task $@ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 5 | 6 |

7 |
8 | Trousseau 9 |
10 |

11 | 12 |

A multi KMS solution supporting the Kubernetes Provider Plugin data encryption for Secrets and ConfigMap in etcd.

13 | 14 | 15 |

16 | 17 | 18 | 19 | Total alerts 20 | 21 | 22 | 23 | 24 | 25 | 26 |

27 | 28 |

29 | Key Features • 30 | Why • 31 | Documentation • 32 | Press • 33 | Hands-on Lab • 34 | How to test • 35 | Roadmap • 36 | Contributing • 37 | License • 38 | Security 39 |

40 | 41 |

42 | 43 |

44 | 45 | ## Key Features 46 | 47 | * Kubernetes native - no additional CLI tooling, respectful of the concern APIs (like Secrets, ConfigMap, ...) 48 | * Encryption of sensitive data payload on the fly and store in *etcd* 49 | * Multi KMS support - one KMS or two KMS at the same time[1] 50 | * HashiCorp Vault (Community and Enterprise editions) 51 | * AWS Key Vault 52 | * Azure KeyVault 53 | * Redesign to full micro-service architecture decloupling the core components for maximum resiliency and distributed handling 54 | * proxy socket to address the Kubernetes API request for encryption/decryption 55 | * trousseau to handle the proxy requests and KMS interaction 56 | * KMS socket to address the connection towards the KMS providers 57 | * Prometheus endpoint 58 | 59 | Notes: 60 | 61 | 1. Trousseau will use each KMS provider to encrypt the data and combine both payload within the same secret data section. 62 | This design is provide more resiliency in case of a KMS failure by introducing reduancy, and add a fast decryption appraoch with first to response decryption approach along with roundrobin. 63 | At the current stade, there is no option to have multi KMS configured and targeting one specific entry for scenario like multi-tenancy and/or multi-staging environment. This is due to a missing annotation extension within the Kubernetes API that we have address to the Kubernetes project.(see issue [#146](https://github.com/ondat/trousseau/issues/146)) 64 | 65 | ## How to test 66 | 67 | ⚠️ for production deployment, consult the [Documentation](https://github.com/ondat/trousseau/wiki) 68 | 69 | Clone the repo and create your environment file: 70 | ```bash 71 | TR_VERSION=d3e4f2569b2eddeea992e47dae29a931182379dd 72 | TR_VERBOSE_LEVEL=1 73 | TR_SOCKET_LOCATION=/opt/trousseau-kms 74 | TR_PROXY_IMAGE=ghcr.io/ondat/trousseau:proxy-${TR_VERSION} 75 | TR_TROUSSEAU_IMAGE=ghcr.io/ondat/trousseau:trousseau-${TR_VERSION} 76 | # Please configure your KMS plugins, maximum 2 77 | TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=azurekms --enabled-providers=vault" 78 | TR_AWSKMS_IMAGE=ghcr.io/ondat/trousseau:awskms-${TR_VERSION} 79 | TR_AWSKMS_CONFIG=awskms.yaml # For Kubernetes, file must exists only for generation 80 | TR_AWSKMS_CREDENTIALS=.aws/credentials 81 | TR_AZUREKMS_IMAGE=ghcr.io/ondat/trousseau:azurekms-${TR_VERSION} 82 | TR_AZUREKMS_CONFIG=azurekms.yaml # For Kubernetes, file must exists only for generation 83 | TR_AZUREKMS_CREDENTIALS=config.json 84 | TR_VAULT_IMAGE=ghcr.io/ondat/trousseau:vault-${TR_VERSION} 85 | TR_VAULT_ADDRESS=https://127.0.0.1:8200 86 | TR_VAULT_CONFIG=vault.yaml 87 | ``` 88 | 89 | Create shared items on target host: 90 | ```bash 91 | mkdir -p $TR_SOCKET_LOCATION 92 | sudo chown 10123:10123 $TR_SOCKET_LOCATION 93 | sudo chown 10123:10123 $TR_AWSKMS_CREDENTIALS 94 | # On case you haven't enable Vault agen config generation 95 | sudo chown 10123:10123 $TR_VAULT_CONFIG 96 | ``` 97 | 98 | Create your config files: 99 | ```yaml 100 | # awskms.yaml 101 | profile: profile 102 | keyArn: keyArn 103 | # Optional fields 104 | roleArn: roleArn 105 | encryptionContext: 106 | foo: bar 107 | ``` 108 | ```yaml 109 | # azurekms.yaml 110 | configFilePath: configFilePath 111 | keyVaultName: keyVaultName 112 | keyName: keyName 113 | keyVersion: keyVersion 114 | ``` 115 | ```yaml 116 | # vault.yaml 117 | keyNames: 118 | - keyNames 119 | address: address 120 | token: token 121 | ``` 122 | 123 | Generate service files or manifests: 124 | ```bash 125 | make prod:generate:systemd ENV_LOCATION=./bin/trousseau-env 126 | make prod:generate:docker-compose ENV_LOCATION=./bin/trousseau-env 127 | make prod:generate:kustomize ENV_LOCATION=./bin/trousseau-env 128 | make prod:generate:helm ENV_LOCATION=./bin/trousseau-env 129 | ``` 130 | 131 | Verify output: 132 | ```bash 133 | ls -l generated_manifests/systemd 134 | ls -l generated_manifests/docker-compose 135 | ls -l generated_manifests/kustomize 136 | ls -l generated_manifests/helm 137 | ``` 138 | 139 | Deploy the application and configure encryption: 140 | ```yaml 141 | kind: EncryptionConfiguration 142 | apiVersion: apiserver.config.k8s.io/v1 143 | resources: 144 | - resources: 145 | - secrets 146 | providers: 147 | - kms: 148 | name: vaultprovider 149 | endpoint: unix:///opt/trousseau-kms/proxy.socket 150 | cachesize: 1000 151 | - identity: {} 152 | ``` 153 | 154 | Reconfigure Kubernetes API server: 155 | ```yaml 156 | kind: ClusterConfiguration 157 | apiServer: 158 | extraArgs: 159 | encryption-provider-config: "/etc/kubernetes/encryption-config.yaml" 160 | extraVolumes: 161 | - name: encryption-config 162 | hostPath: "/etc/kubernetes/encryption-config.yaml" 163 | mountPath: "/etc/kubernetes/encryption-config.yaml" 164 | readOnly: true 165 | pathType: File 166 | - name: sock-path 167 | hostPath: "/opt/trousseau-kms" 168 | mountPath: "/opt/trousseau-kms" 169 | ``` 170 | 171 | Finally restart Kubernetes API server. 172 | -------------------------------------------------------------------------------- /ROADMAP.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | To see items we are working on, visit our [RoadMap GitHub project](https://github.com/orgs/ondat/projects/1/views/4). 4 | 5 | You can submit an idea [here](https://github.com/ondat/trousseau/issues/new?assignees=&labels=enhancement&template=rfe.md&title=%5BRFE%5D+). 6 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security 2 | 3 | ## Reporting Potential Security Issues 4 | 5 | If you have encountered a potential security vulnerability in this project, 6 | please **report it to us at security@trousseau.io and not via an GitHub issue**. 7 | 8 | We will work with you to verify the vulnerability, build a patch, validate 9 | the fix, and finally issue a public report. 10 | 11 | When reporting issues, please provide the following information: 12 | - Component(s) affected 13 | - A description indicating how to reproduce the issue 14 | - A summary of the security vulnerability and impact 15 | 16 | We request that you contact us via the email address above and give the 17 | project contributors a chance to resolve the vulnerability and issue a new 18 | release prior to any public exposure; this helps protect the project's 19 | users, and provides them with a chance to upgrade and/or update in order to 20 | protect their applications. 21 | 22 | ## Policy 23 | 24 | If we verify a reported security vulnerability, our policy is: 25 | 26 | - We will patch the current release branch, as well as the immediate prior minor 27 | release branch. 28 | 29 | - After patching the release branches, we will immediately issue new security 30 | fix releases for each patched release branch. 31 | 32 | - A security advisory will be released on the project website detailing the 33 | vulnerability, as well as recommendations for end-users to protect themselves. 34 | Security advisories will be listed on the project wiki. 35 | -------------------------------------------------------------------------------- /Taskfile.yml: -------------------------------------------------------------------------------- 1 | version: 3 2 | vars: 3 | KIND_CLUSTER_NAME: kms-vault 4 | KIND_CLUSTER_VERSION: 1.24 5 | ENABLED_PROVIDERS: 6 | sh: '([ -z "$ENABLED_PROVIDERS" ] && echo --enabled-providers=debug) || echo $ENABLED_PROVIDERS' 7 | silent: true 8 | includes: 9 | cluster: .task/cluster.yml 10 | docker: .task/docker.yml 11 | fetch: .task/fetch.yml 12 | go: .task/go.yml 13 | prod: .task/prod.yml 14 | tasks: 15 | default: 16 | cmds: 17 | - task -l 18 | bin-dir:init: 19 | desc: create bin directory 20 | cmds: 21 | - mkdir -p ./bin 22 | status: 23 | - test -d ./bin 24 | run-dir:init: 25 | desc: create bin directory 26 | cmds: 27 | - mkdir -pm 777 bin/run 28 | - mkdir -pm 777 bin/run/debug 29 | - mkdir -pm 777 bin/run/vault 30 | - mkdir -pm 777 bin/run/awskms 31 | - mkdir -pm 777 bin/run/azurekms 32 | status: 33 | - test -d ./bin/run 34 | - test -d ./bin/run/debug 35 | - test -d ./bin/run/vault 36 | - test -d ./bin/run/awskms 37 | - test -d ./bin/run/azurekms 38 | example:load: 39 | desc: load demo data 40 | cmds: 41 | - sh {{.SCRIPT}}/test.bash 42 | example:before-key-rotate: 43 | desc: data before key rotate 44 | cmds: 45 | - echo "-------- secret etcd data --------" 46 | - ./bin/kubectl -n kube-system exec -t etcd-kms-vault-control-plane -- etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key get /registry/secrets/default/data-test 47 | - echo "-------- secret fetch data --------" 48 | - ./bin/kubectl get secret data-test -o yaml 49 | example:after-key-rotate: 50 | desc: data after key rotate 51 | cmds: 52 | - ./bin/kubectl apply -f scripts/secret2.yaml 53 | - echo "-------- old secret etcd --------" 54 | - ./bin/kubectl -n kube-system exec -t etcd-kms-vault-control-plane -- etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key get /registry/secrets/default/data-test 55 | - echo "-------- new secret etcd --------" 56 | - ./bin/kubectl -n kube-system exec -t etcd-kms-vault-control-plane -- etcdctl --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key get /registry/secrets/default/data-test3 57 | - echo "-------- old secret fetch data --------" 58 | - ./bin/kubectl get secret data-test -o yaml 59 | - echo "-------- new secret fetch data --------" 60 | - ./bin/kubectl get secret data-test3 -o yaml 61 | -------------------------------------------------------------------------------- /assets/Ondat Diagram-w-all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ondat/trousseau/83a8c6eb1518ad13e3e5de7428233b33c29db6de/assets/Ondat Diagram-w-all.png -------------------------------------------------------------------------------- /assets/logo-color-horizontal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ondat/trousseau/83a8c6eb1518ad13e3e5de7428233b33c29db6de/assets/logo-color-horizontal.png -------------------------------------------------------------------------------- /assets/logo-horizontal-mono.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ondat/trousseau/83a8c6eb1518ad13e3e5de7428233b33c29db6de/assets/logo-horizontal-mono.png -------------------------------------------------------------------------------- /assets/logo-horizontal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ondat/trousseau/83a8c6eb1518ad13e3e5de7428233b33c29db6de/assets/logo-horizontal.png -------------------------------------------------------------------------------- /assets/logo-vertical.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ondat/trousseau/83a8c6eb1518ad13e3e5de7428233b33c29db6de/assets/logo-vertical.png -------------------------------------------------------------------------------- /deployment/docker-compose/docker-compose.override.awskms.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | awskms: 3 | image: ${TR_AWSKMS_IMAGE} 4 | command: --listen-addr=unix:///sockets/awskms/awskms.socket --config-file-path=/etc/config.yaml -v=${TR_VERBOSE_LEVEL} 5 | volumes: 6 | - sockets:/sockets:rw 7 | - ${TR_AWSKMS_CREDENTIALS}:/.aws/credentials:r 8 | - ${TR_AWSKMS_CONFIG}:/etc/config.yaml:r 9 | restart: always 10 | depends_on: 11 | - init 12 | -------------------------------------------------------------------------------- /deployment/docker-compose/docker-compose.override.azurekms.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | azurekms: 3 | image: ${TR_AZUREKMS_IMAGE} 4 | command: --listen-addr=unix:///sockets/azurekms/azurekms.socket --config-file-path=/etc/config.yaml -v=${TR_VERBOSE_LEVEL} 5 | volumes: 6 | - sockets:/sockets:rw 7 | - ${TR_AZUREKMS_CREDENTIALS}:${TR_AZUREKMS_CREDENTIALS}:r 8 | - ${TR_AZUREKMS_CONFIG}:/etc/config.yaml:r 9 | restart: always 10 | depends_on: 11 | - init 12 | -------------------------------------------------------------------------------- /deployment/docker-compose/docker-compose.override.vault.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | vault: 3 | image: ${TR_VAULT_IMAGE} 4 | command: --listen-addr=unix:///sockets/vault/vault.socket --config-file-path=/etc/config.yaml -v=${TR_VERBOSE_LEVEL} 5 | volumes: 6 | - sockets:/sockets:rw 7 | - ${TR_VAULT_CONFIG}:/etc/config.yaml:r 8 | restart: always 9 | depends_on: 10 | - init 11 | -------------------------------------------------------------------------------- /deployment/docker-compose/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | init: 3 | image: busybox:stable-glibc 4 | command: sh -c 'mkdir -p /sockets/awskms /sockets/azurekms /sockets/vault /sockets/trousseau ; chown -R 10123:10123 /sockets/*' 5 | volumes: 6 | - sockets:/sockets:rw 7 | proxy: 8 | image: ${TR_PROXY_IMAGE} 9 | command: --listen-addr=unix:///opt/trousseau-kms/proxy.socket --trousseau-addr=/sockets/trousseau/trousseau.socket 10 | volumes: 11 | - sockets:/sockets:rw 12 | - ${TR_SOCKET_LOCATION}:/opt/trousseau-kms:rw 13 | restart: always 14 | depends_on: 15 | - trousseau 16 | trousseau: 17 | image: ${TR_TROUSSEAU_IMAGE} 18 | command: --listen-addr=unix:///sockets/trousseau/trousseau.socket --socket-location=/sockets ${TR_ENABLED_PROVIDERS} -v=${TR_VERBOSE_LEVEL} 19 | volumes: 20 | - sockets:/sockets:rw 21 | restart: always 22 | depends_on: 23 | - init 24 | - awskms 25 | - vault 26 | volumes: 27 | sockets: {} 28 | -------------------------------------------------------------------------------- /deployment/helm/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deployment/helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: Trousseau 3 | description: A Helm chart for Trousseau, an open-source project leveraging the Kubernetes KMS provider framework to connect with Key Management Services the Kubernetes native way! 4 | type: application 5 | version: 0.0.1 6 | appVersion: "2.0.0" 7 | keywords: 8 | - secret 9 | - kms 10 | - envelop encryption 11 | - encryption at rest 12 | home: https://trousseau.io 13 | icon: https://www.ondat.io/hs-fs/hubfs/signature-mono.png 14 | sources: 15 | - https://github.com/ondat/trousseau 16 | -------------------------------------------------------------------------------- /deployment/helm/templates/configmap-awskms.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: trousseau-awskms-config 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | {{- toYaml .Values.commonLabels | nindent 4 }} 8 | data: 9 | config.yaml: | 10 | {{ .Files.Get .Values.awskms.configPath | indent 2 }} 11 | -------------------------------------------------------------------------------- /deployment/helm/templates/configmap-azurekms.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: trousseau-azurekms-config 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | {{- toYaml .Values.commonLabels | nindent 4 }} 8 | data: 9 | config.yaml: | 10 | {{ .Files.Get .Values.azurekms.configPath | indent 2 }} 11 | -------------------------------------------------------------------------------- /deployment/helm/templates/configmap-vault.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.vault.withConfigGenerator -}} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: trousseau-vault-agent-config 6 | namespace: {{ .Values.namespace }} 7 | labels: 8 | {{- toYaml .Values.commonLabels | nindent 4 }} 9 | data: 10 | vault-agent-config.hcl: | 11 | exit_after_auth = true 12 | pid_file = "/home/vault/pidfile" 13 | auto_auth { 14 | method "kubernetes" { 15 | mount_path = "auth/kubernetes" 16 | config = { 17 | role = "trousseau" 18 | } 19 | } 20 | sink "file" { 21 | config = { 22 | path = "/home/vault/.vault-token" 23 | } 24 | } 25 | } 26 | 27 | template { 28 | destination = "{{ .Values.vault.configPath }}" 29 | contents = < go.opentelemetry.io/otel/sdk v1.7.0 6 | 7 | require ( 8 | github.com/go-logr/logr v1.2.3 9 | github.com/go-logr/zapr v1.2.3 10 | github.com/stretchr/testify v1.8.0 11 | go.opentelemetry.io/otel v1.7.0 12 | go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0 13 | go.opentelemetry.io/otel/metric v0.21.0 14 | go.uber.org/zap v1.21.0 15 | google.golang.org/grpc v1.47.0 16 | gopkg.in/yaml.v2 v2.4.0 17 | k8s.io/apiserver v0.24.2 18 | k8s.io/klog/v2 v2.70.0 19 | ) 20 | 21 | require ( 22 | github.com/beorn7/perks v1.0.1 // indirect 23 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 24 | github.com/davecgh/go-spew v1.1.1 // indirect 25 | github.com/go-logr/stdr v1.2.2 // indirect 26 | github.com/gogo/protobuf v1.3.2 // indirect 27 | github.com/golang/protobuf v1.5.2 // indirect 28 | github.com/google/go-cmp v0.5.8 // indirect 29 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 30 | github.com/pmezard/go-difflib v1.0.0 // indirect 31 | github.com/prometheus/client_golang v1.12.1 // indirect 32 | github.com/prometheus/client_model v0.2.0 // indirect 33 | github.com/prometheus/common v0.32.1 // indirect 34 | github.com/prometheus/procfs v0.7.3 // indirect 35 | go.opentelemetry.io/otel/internal/metric v0.21.0 // indirect 36 | go.opentelemetry.io/otel/sdk v1.0.0-RC1 // indirect 37 | go.opentelemetry.io/otel/sdk/export/metric v0.21.0 // indirect 38 | go.opentelemetry.io/otel/sdk/metric v0.21.0 // indirect 39 | go.opentelemetry.io/otel/trace v1.7.0 // indirect 40 | go.uber.org/atomic v1.9.0 // indirect 41 | go.uber.org/goleak v1.1.12 // indirect 42 | go.uber.org/multierr v1.6.0 // indirect 43 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 44 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 45 | golang.org/x/text v0.3.7 // indirect 46 | google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect 47 | google.golang.org/protobuf v1.28.0 // indirect 48 | gopkg.in/yaml.v3 v3.0.1 // indirect 49 | ) 50 | -------------------------------------------------------------------------------- /go.work: -------------------------------------------------------------------------------- 1 | go 1.18 2 | 3 | use ( 4 | . 5 | ./providers/awskms 6 | ./providers/azurekms 7 | ./providers/debug 8 | ./providers/vault 9 | ./proxy 10 | ./trousseau 11 | ) 12 | -------------------------------------------------------------------------------- /issue-31: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ondat/trousseau/83a8c6eb1518ad13e3e5de7428233b33c29db6de/issue-31 -------------------------------------------------------------------------------- /localdev.md: -------------------------------------------------------------------------------- 1 | # Local development 2 | 3 | This document describes how to develop Trousseau on your local machine. 4 | 5 | Requirements: 6 | 7 | * install and set up Docker 8 | * install taskfile https://taskfile.dev/#/installation 9 | 10 | ## Run Trousseau components 11 | 12 | ```bash 13 | task go:run:proxy 14 | task go:run:debug 15 | task go:run:trousseau 16 | ``` 17 | 18 | ## Start cluster with encryption support 19 | 20 | For local testing we suggest to use Kind to create a cluster. Everything is configured for you so please run the command below: 21 | 22 | ```bash 23 | task cluster:create 24 | ``` 25 | 26 | You are ready for create secrets! 27 | 28 | ### Verify secret encryption 29 | 30 | To verify encryption please create a secret and check value in ETCD. 31 | 32 | ``` 33 | kubectl create secret -n default generic trousseau-test --from-literal=FOO=bar 34 | kubectl get secret -o yaml 35 | docker exec kms-vault-control-plane bash -c 'apt update && apt install -y etcd-client' # only once 36 | docker exec -it -e ETCDCTL_API=3 -e SSL_OPTS='--cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/apiserver-etcd-client.crt --key=/etc/kubernetes/pki/apiserver-etcd-client.key --endpoints=localhost:2379' kms-vault-control-plane \ 37 | bash -c 'etcdctl $SSL_OPTS get --keys-only=false --prefix /registry/secrets/default' 38 | ``` 39 | 40 | You have to see encrypted data in ETCD dump. 41 | 42 | ### Cleanup cluster 43 | 44 | After you have finished fun on Trousseau you should terminate the cluster with the following command: 45 | 46 | ```bash 47 | task cluster:delete 48 | ``` 49 | 50 | ## Run end to end tests 51 | 52 | Test are found in tests/e2e/kuttl directory. To run full test please execute the command below: 53 | 54 | ```bash 55 | task go:e2e-tests:debug 56 | ``` -------------------------------------------------------------------------------- /pkg/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "strconv" 7 | 8 | "github.com/go-logr/logr" 9 | "github.com/go-logr/zapr" 10 | "go.uber.org/zap" 11 | "go.uber.org/zap/zapcore" 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | const ( 16 | Info1 klog.Level = iota + 1 17 | Info2 18 | Info3 19 | Debug1 20 | Debug2 21 | ) 22 | 23 | func init() { 24 | klog.InitFlags(nil) 25 | } 26 | 27 | // InitializeLogging initializing global logging. 28 | func InitializeLogging(logEncoder string) error { 29 | v := flag.CommandLine.Lookup("v").Value.String() 30 | 31 | logLevel, err := strconv.Atoi(v) 32 | if err != nil { 33 | klog.Error(err, "Invalid verbosity level", "level", v) 34 | return err 35 | } else if logLevel < 0 || logLevel > 5 { 36 | klog.Fatalln("Log level must be between 0 and 5", "level", v) 37 | } 38 | 39 | klog.SetLogger(newLogger(klog.Level(logLevel), logEncoder)) 40 | 41 | return nil 42 | } 43 | 44 | func newLogger(logLevel klog.Level, logEncoder string) logr.Logger { 45 | encConfig := zap.NewProductionEncoderConfig() 46 | encConfig.TimeKey = "timestamp" 47 | encConfig.EncodeTime = zapcore.RFC3339NanoTimeEncoder 48 | 49 | encoder := zapcore.NewConsoleEncoder(encConfig) 50 | if logEncoder == "json" { 51 | encoder = zapcore.NewJSONEncoder(encConfig) 52 | } 53 | 54 | core := zapcore.NewCore(encoder, zapcore.AddSync(os.Stdout), zap.NewAtomicLevelAt(zapcore.Level(-logLevel))) 55 | logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel)) 56 | 57 | return zapr.NewLogger(logger) 58 | } 59 | -------------------------------------------------------------------------------- /pkg/metrics/exporter.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "k8s.io/klog/v2" 8 | ) 9 | 10 | const ( 11 | prometheusExporter = "prometheus" 12 | ) 13 | 14 | // Serve starts new exporter if needed 15 | func Serve(metricsBackend, metricsAddress string) error { 16 | exporter := strings.ToLower(metricsBackend) 17 | klog.InfoS("Metrics backend", "exporter", exporter) 18 | 19 | switch exporter { 20 | // Prometheus is the only exporter supported for now 21 | case prometheusExporter: 22 | return servePrometheusExporter(metricsAddress) 23 | default: 24 | return fmt.Errorf("unsupported metrics backend %v", metricsBackend) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pkg/metrics/prometheus_exporter.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | 8 | "go.opentelemetry.io/otel/exporters/metric/prometheus" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | const ( 13 | metricsEndpoint = "metrics" 14 | ) 15 | 16 | // serve creates the http handler for serving metrics requests 17 | func servePrometheusExporter(metricsAddress string) error { 18 | exporter, err := prometheus.InstallNewPipeline(prometheus.Config{ 19 | DefaultHistogramBoundaries: []float64{ 20 | 0.1, 0.2, 0.3, 0.4, 0.5, 1, 1.5, 2, 2.5, 3.0, 5.0, 10.0, 15.0, 30.0, 21 | }, 22 | }, 23 | ) 24 | if err != nil { 25 | return fmt.Errorf("failed to register prometheus exporter: %w", err) 26 | } 27 | 28 | klog.InfoS("Prometheus metrics server starting", "address", metricsAddress) 29 | 30 | serveMux := http.NewServeMux() 31 | serveMux.HandleFunc(fmt.Sprintf("/%s", metricsEndpoint), exporter.ServeHTTP) 32 | 33 | const timeout = time.Second * 5 34 | 35 | srv := &http.Server{ 36 | ReadTimeout: timeout, 37 | ReadHeaderTimeout: timeout, 38 | WriteTimeout: timeout, 39 | Addr: fmt.Sprintf(":%s", metricsAddress), 40 | Handler: serveMux, 41 | } 42 | 43 | if err := srv.ListenAndServe(); err != nil { 44 | return fmt.Errorf("failed to register prometheus endpoint: %w", err) 45 | } 46 | 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /pkg/metrics/stats_reporter.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ondat/trousseau/pkg/logger" 7 | "go.opentelemetry.io/otel/attribute" 8 | "go.opentelemetry.io/otel/metric" 9 | "go.opentelemetry.io/otel/metric/global" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | const ( 14 | instrumentationName = "vaultprovider" 15 | errorMessageKey = "error_message" 16 | statusTypeKey = "status" 17 | operationTypeKey = "operation" 18 | providerKey = "provider" 19 | kmsRequestMetricName = "kms_request" 20 | // ErrorStatusTypeValue sets status tag to "error" 21 | ErrorStatusTypeValue = "error" 22 | // SuccessStatusTypeValue sets status tag to "success" 23 | SuccessStatusTypeValue = "success" 24 | // EncryptOperationTypeValue sets operation tag to "encrypt" 25 | EncryptOperationTypeValue = "encrypt" 26 | // DecryptOperationTypeValue sets operation tag to "decrypt" 27 | DecryptOperationTypeValue = "decrypt" 28 | // GrpcOperationTypeValue sets operation tag to "grpc" 29 | GrpcOperationTypeValue = "grpc" 30 | ) 31 | 32 | var kmsRequest metric.Float64ValueRecorder 33 | 34 | type reporter struct { 35 | meter metric.Meter 36 | } 37 | 38 | // StatsReporter reports metrics 39 | type StatsReporter interface { 40 | ReportRequest(ctx context.Context, provider, operationType, status string, duration float64, errors ...string) 41 | } 42 | 43 | // NewStatsReporter instantiates otel reporter 44 | func NewStatsReporter() StatsReporter { 45 | klog.V(logger.Debug1).Info("Initialize new stats reporter...") 46 | 47 | meter := global.Meter(instrumentationName) 48 | 49 | kmsRequest = metric.Must(meter).NewFloat64ValueRecorder(kmsRequestMetricName, metric.WithDescription("Distribution of how long it took for an operation")) 50 | 51 | return &reporter{ 52 | meter: meter, 53 | } 54 | } 55 | 56 | func (r *reporter) ReportRequest(ctx context.Context, provider, operationType, status string, duration float64, errors ...string) { 57 | labels := []attribute.KeyValue{ 58 | attribute.String(providerKey, provider), 59 | attribute.String(operationTypeKey, operationType), 60 | attribute.String(statusTypeKey, status), 61 | } 62 | 63 | // Add errors 64 | if (status == ErrorStatusTypeValue) && len(errors) > 0 { 65 | for _, err := range errors { 66 | labels = append(labels, attribute.String(errorMessageKey, err)) 67 | } 68 | } 69 | 70 | r.meter.RecordBatch(ctx, 71 | labels, 72 | kmsRequest.Measurement(duration), 73 | ) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/providers/providers.go: -------------------------------------------------------------------------------- 1 | package providers 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ondat/trousseau/pkg/logger" 8 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | // EncryptionClient is the main interface for provider client. 13 | type EncryptionClient interface { 14 | Decrypt(data []byte) ([]byte, error) 15 | Encrypt(data []byte) ([]byte, error) 16 | Version() *pb.VersionResponse 17 | } 18 | 19 | // KeyManagementService is the main interface for gRPC server. 20 | type KeyManagementService interface { 21 | Decrypt(context.Context, *pb.DecryptRequest) (*pb.DecryptResponse, error) 22 | Encrypt(context.Context, *pb.EncryptRequest) (*pb.EncryptResponse, error) 23 | Version(context.Context, *pb.VersionRequest) (*pb.VersionResponse, error) 24 | } 25 | 26 | // KeyManagementServiceServer base implementation if gRPC server. 27 | type KeyManagementServiceServer struct { 28 | Client EncryptionClient 29 | } 30 | 31 | // Encrypt encryption requet handler. 32 | func (k *KeyManagementServiceServer) Encrypt(ctx context.Context, data *pb.EncryptRequest) (*pb.EncryptResponse, error) { 33 | klog.V(logger.Debug1).Info("Encrypt has been called...") 34 | 35 | response, err := k.Client.Encrypt(data.Plain) 36 | if err != nil { 37 | klog.InfoS("Failed to encrypt", "error", err.Error()) 38 | return nil, fmt.Errorf("failed to encrypt: %w", err) 39 | } 40 | 41 | klog.V(logger.Debug1).Info("Encrypt request complete") 42 | 43 | return &pb.EncryptResponse{Cipher: response}, nil 44 | } 45 | 46 | // Decrypt decryption requet handler. 47 | func (k *KeyManagementServiceServer) Decrypt(ctx context.Context, data *pb.DecryptRequest) (*pb.DecryptResponse, error) { 48 | klog.V(logger.Debug1).Info("Decrypt has been called...") 49 | 50 | response, err := k.Client.Decrypt(data.Cipher) 51 | if err != nil { 52 | klog.InfoS("Failed to decrypt", "error", err.Error()) 53 | return nil, fmt.Errorf("failed to decrypt: %w", err) 54 | } 55 | 56 | klog.V(logger.Debug1).Info("Decrypt request complete") 57 | 58 | return &pb.DecryptResponse{Plain: response}, nil 59 | } 60 | 61 | // Version version of gRPS server. 62 | func (k *KeyManagementServiceServer) Version(context.Context, *pb.VersionRequest) (*pb.VersionResponse, error) { 63 | return k.Client.Version(), nil 64 | } 65 | -------------------------------------------------------------------------------- /pkg/utils/config.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/ondat/trousseau/pkg/logger" 9 | "gopkg.in/yaml.v2" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | // ParseConfig parses config. 14 | func ParseConfig(cfpPath string, cfg interface{}) error { 15 | klog.V(logger.Info2).InfoS("Parsing config...", "path", cfpPath) 16 | 17 | content, err := os.ReadFile(filepath.Clean(cfpPath)) 18 | if err != nil { 19 | klog.ErrorS(err, "Unable to open config file", "path", cfpPath) 20 | return fmt.Errorf("unable to open config file %s: %w", cfpPath, err) 21 | } 22 | 23 | if err = yaml.Unmarshal(content, cfg); err != nil { 24 | klog.ErrorS(err, "Unable to unmarshal config file", "path", cfpPath) 25 | return fmt.Errorf("unable to unmarshal config file %s: %w", cfpPath, err) 26 | } 27 | 28 | klog.V(logger.Debug2).Info("Current config", "config", cfg) 29 | 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/utils/grpc.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "strings" 8 | 9 | "github.com/ondat/trousseau/pkg/logger" 10 | "google.golang.org/grpc" 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | const ( 15 | splitin = 2 16 | ) 17 | 18 | // ParseEndpoint returns unix socket's protocol and address 19 | func ParseEndpoint(ep string) (proto, address string, err error) { 20 | err = fmt.Errorf("invalid endpoint: %s", ep) 21 | 22 | if !strings.HasPrefix(strings.ToLower(ep), "unix://") { 23 | return 24 | } 25 | 26 | parts := strings.SplitN(ep, "://", splitin) 27 | if parts[1] != "" { 28 | proto = parts[0] 29 | address = parts[1] 30 | err = nil 31 | } 32 | 33 | return 34 | } 35 | 36 | // UnaryServerInterceptor provides metrics around Unary RPCs. 37 | func UnaryServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 38 | klog.V(logger.Debug1).InfoS("GRPC call", "method", info.FullMethod) 39 | 40 | resp, err := handler(ctx, req) 41 | if err != nil { 42 | klog.InfoS("GRPC request error", "method", info.FullMethod, "error", err.Error()) 43 | 44 | return nil, fmt.Errorf("GRPC request error: %w", err) 45 | } 46 | 47 | return resp, nil 48 | } 49 | 50 | // DialUnixSocket creates a gRPC connection. 51 | func DialUnixSocket(unixSocketPath string) (*grpc.ClientConn, error) { 52 | return grpc.Dial( 53 | unixSocketPath, 54 | //nolint:staticcheck // We know WithInsecure is deprecated 55 | grpc.WithInsecure(), 56 | grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) { 57 | return (&net.Dialer{}).DialContext(ctx, "unix", target) 58 | }), 59 | ) 60 | } 61 | -------------------------------------------------------------------------------- /pkg/utils/grpc_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestParseEndpoint(t *testing.T) { 11 | testcases := map[string]struct { 12 | endpoint string 13 | wantProtocol string 14 | wantEndpoint string 15 | wantErr error 16 | }{ 17 | "OK": { 18 | endpoint: "unix://foo.bar", 19 | wantProtocol: "unix", 20 | wantEndpoint: "foo.bar", 21 | }, 22 | "Wrong protocol": { 23 | endpoint: "http://foo.bar", 24 | wantErr: errors.New("invalid endpoint: http://foo.bar"), 25 | }, 26 | } 27 | 28 | for name, tc := range testcases { 29 | tc := tc 30 | 31 | t.Run(name, func(t *testing.T) { 32 | t.Parallel() 33 | 34 | for i := 0; i < 3; i++ { 35 | proto, address, err := ParseEndpoint(tc.endpoint) 36 | 37 | assert.Equal(t, err, tc.wantErr, "Invalid error") 38 | assert.Equal(t, tc.wantProtocol, proto, "Invalid protocol") 39 | assert.Equal(t, tc.wantEndpoint, address, "Invalid address") 40 | } 41 | }) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | var ( 11 | // -X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1 12 | SecretLogDivider string 13 | secretLogDivider = 2 14 | ) 15 | 16 | func init() { 17 | if SecretLogDivider != "" { 18 | var err error 19 | 20 | secretLogDivider, err = strconv.Atoi(SecretLogDivider) 21 | if err != nil || secretLogDivider <= 0 { 22 | panic("Invalid github.com/ondat/trousseau/pkg/utils.SecretLogDivider=" + SecretLogDivider) 23 | } 24 | } 25 | } 26 | 27 | // SecretToLog truncates secret to log. 28 | func SecretToLog(s string) string { 29 | b := []byte(s) 30 | 31 | var suffix string 32 | if secretLogDivider > 1 { 33 | suffix = "..." 34 | } 35 | 36 | return string(b[:len(b)/secretLogDivider]) + suffix 37 | } 38 | 39 | // RemoveFile removes given file. 40 | func RemoveFile(path string) error { 41 | path = filepath.Clean(path) 42 | 43 | if _, err := os.Stat(path); err != nil { 44 | return nil 45 | } 46 | 47 | return os.Remove(path) 48 | } 49 | 50 | func WatchFile(path string) <-chan error { 51 | errChan := make(chan error) 52 | ticker := time.NewTicker(time.Second) 53 | 54 | go func() { 55 | for { 56 | <-ticker.C 57 | 58 | if _, err := os.Stat(path); err != nil { 59 | errChan <- err 60 | } 61 | } 62 | }() 63 | 64 | return errChan 65 | } 66 | -------------------------------------------------------------------------------- /pkg/utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSecretToLog(t *testing.T) { 10 | assert.Equal(t, "ab...", SecretToLog("abcd"), "Wrong secret part returned") 11 | } 12 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "runtime" 7 | ) 8 | 9 | var ( 10 | // BuildDate is the date when the binary was built 11 | BuildDate string 12 | // GitCommit is the commit hash when the binary was built 13 | GitCommit string 14 | // BuildVersion is the version of the KMS binary 15 | BuildVersion = "dev" 16 | APIVersion = "v1beta1" 17 | Runtime = "HashiCorp Vault KMS" 18 | ) 19 | 20 | // PrintVersion prints the current KMS plugin version 21 | func PrintVersion() (err error) { 22 | pv := struct { 23 | BuildVersion string 24 | GitCommit string 25 | BuildDate string 26 | }{ 27 | BuildDate: BuildDate, 28 | BuildVersion: BuildVersion, 29 | GitCommit: GitCommit, 30 | } 31 | 32 | res, err := json.Marshal(pv) 33 | if err != nil { 34 | return 35 | } 36 | 37 | fmt.Printf(string(res) + "\n") 38 | 39 | return 40 | } 41 | 42 | // GetUserAgent returns UserAgent string to append to the agent identifier. 43 | func GetUserAgent() string { 44 | return fmt.Sprintf("k8s-kms-vault/%s (%s/%s) %s/%s", BuildVersion, runtime.GOOS, runtime.GOARCH, GitCommit, BuildDate) 45 | } 46 | -------------------------------------------------------------------------------- /providers/awskms/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ondat/trousseau/providers/awskms 2 | 3 | go 1.18 4 | 5 | replace github.com/ondat/trousseau => ../.. 6 | 7 | require ( 8 | github.com/aws/aws-sdk-go v1.44.24 9 | github.com/ondat/trousseau v0.0.0-00010101000000-000000000000 10 | github.com/stretchr/testify v1.8.0 11 | google.golang.org/grpc v1.47.0 12 | k8s.io/apiserver v0.24.2 13 | k8s.io/klog/v2 v2.70.0 14 | ) 15 | 16 | require ( 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/go-logr/logr v1.2.3 // indirect 19 | github.com/go-logr/zapr v1.2.3 // indirect 20 | github.com/gogo/protobuf v1.3.2 // indirect 21 | github.com/golang/protobuf v1.5.2 // indirect 22 | github.com/jmespath/go-jmespath v0.4.0 // indirect 23 | github.com/pmezard/go-difflib v1.0.0 // indirect 24 | go.uber.org/atomic v1.9.0 // indirect 25 | go.uber.org/multierr v1.6.0 // indirect 26 | go.uber.org/zap v1.21.0 // indirect 27 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 28 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 29 | golang.org/x/text v0.3.7 // indirect 30 | google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect 31 | google.golang.org/protobuf v1.28.0 // indirect 32 | gopkg.in/yaml.v2 v2.4.0 // indirect 33 | gopkg.in/yaml.v3 v3.0.1 // indirect 34 | ) 35 | -------------------------------------------------------------------------------- /providers/awskms/localdev.md: -------------------------------------------------------------------------------- 1 | # Local development 2 | 3 | This document describes how to develop Trousseau AWS KMS provider on your local machine. 4 | 5 | Please follow base documentation at [localdev.md](/../../localdev.md) 6 | 7 | ## Login to AWS 8 | 9 | Log in and create profile file at `~/.aws/credentials`. 10 | 11 | Edit config file at [awskms.yaml](/../../tests/e2e/kuttl/kube-v1.24/awskms.yaml): 12 | 13 | ```yaml 14 | profile: profile 15 | keyArn: keyArn 16 | # Optional fields 17 | roleArn: roleArn 18 | encryptionContext: 19 | foo: bar 20 | ``` 21 | 22 | ### Using Localstack 23 | 24 | Aternatively you should spin up a Localstack on your machine to test AWS KMS without using AWS itself. 25 | 26 | Edit profile file at `~/.aws/credentials`: 27 | 28 | ```ini 29 | [trousseau-local-aws] 30 | aws_access_key_id=000000000000 31 | aws_secret_access_key=XXX 32 | ``` 33 | 34 | Start services: 35 | 36 | ```bash 37 | docker run --name trousseau-local-aws --rm -d -e SERVICES=kms -e HOSTNAME=localhost.localstack.cloud -e HOSTNAME_EXTERNAL=localhost.localstack.cloud -e DEFAULT_REGION=eu-west-1 -e KMS_PROVIDER=kms-local -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:0.14.4 38 | docker exec trousseau-local-aws awslocal kms create-key 39 | ``` 40 | 41 | Output: 42 | ```json 43 | { 44 | "KeyMetadata": { 45 | "AWSAccountId": "000000000000", 46 | "KeyId": "...", 47 | "Arn": "arn:aws:kms:eu-west-1:000000000000:key/...", 48 | "CreationDate": 1656405914, 49 | "Enabled": true, 50 | "KeyUsage": "ENCRYPT_DECRYPT", 51 | "KeyState": "Enabled", 52 | "Origin": "AWS_KMS", 53 | "KeyManager": "CUSTOMER", 54 | "CustomerMasterKeySpec": "SYMMETRIC_DEFAULT", 55 | "KeySpec": "SYMMETRIC_DEFAULT", 56 | "EncryptionAlgorithms": [ 57 | "SYMMETRIC_DEFAULT" 58 | ] 59 | } 60 | } 61 | ``` 62 | 63 | Edit config file based on output at [awskms.yaml](/../../tests/e2e/kuttl/kube-v1.24/awskms.yaml): 64 | 65 | ```yaml 66 | endpoint: https://localhost.localstack.cloud:4566 67 | profile: trousseau-local-aws 68 | keyArn: arn:aws:kms:eu-west-1:000000000000:key/c720e1b5-a113-44ed-9f7b-1cf0c1f61ee8 69 | ``` 70 | 71 | ## Run Trousseau components 72 | 73 | Use command line or our favorite IDE to start Trousseau components on your machine: 74 | 75 | ```bash 76 | task go:run:proxy 77 | task go:run:awskms 78 | ENABLED_PROVIDERS="--enabled-providers=awskms" task go:run:trousseau 79 | ``` 80 | -------------------------------------------------------------------------------- /providers/awskms/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "net" 6 | "os" 7 | 8 | "github.com/ondat/trousseau/pkg/logger" 9 | "github.com/ondat/trousseau/pkg/providers" 10 | "github.com/ondat/trousseau/pkg/utils" 11 | "github.com/ondat/trousseau/providers/awskms/pkg/awskms" 12 | "google.golang.org/grpc" 13 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 14 | "k8s.io/klog/v2" 15 | ) 16 | 17 | var ( 18 | listenAddr = flag.String("listen-addr", "unix:///opt/trousseau-kms/awskms/awskms.socket", "gRPC listen address") 19 | configFilePath = flag.String("config-file-path", "/opt/trousseau-kms/awskms/config.yaml", "Path for AWS KMS Provider config file") 20 | logEncoder = flag.String("zap-encoder", "console", "set log encoder [console, json]") 21 | ) 22 | 23 | func main() { 24 | flag.Parse() 25 | 26 | err := logger.InitializeLogging(*logEncoder) 27 | if err != nil { 28 | klog.Errorln(err) 29 | os.Exit(1) 30 | } 31 | 32 | hostname, err := os.Hostname() 33 | if err != nil { 34 | klog.Errorln(err) 35 | os.Exit(1) 36 | } 37 | 38 | cfg := awskms.Config{} 39 | if err = utils.ParseConfig(*configFilePath, &cfg); err != nil { 40 | klog.Errorln(err) 41 | os.Exit(1) 42 | } 43 | 44 | client, err := awskms.New(&cfg, hostname) 45 | if err != nil { 46 | klog.Errorln(err) 47 | os.Exit(1) 48 | } 49 | 50 | opts := []grpc.ServerOption{ 51 | grpc.UnaryInterceptor(utils.UnaryServerInterceptor), 52 | } 53 | 54 | s := grpc.NewServer(opts...) 55 | pb.RegisterKeyManagementServiceServer(s, &providers.KeyManagementServiceServer{ 56 | Client: client, 57 | }) 58 | 59 | proto, addr, err := utils.ParseEndpoint(*listenAddr) 60 | if err != nil { 61 | klog.Errorln(err) 62 | os.Exit(1) 63 | } 64 | 65 | if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { 66 | klog.ErrorS(err, "unable to delete socket file", "file", addr) 67 | } 68 | 69 | listener, err := net.Listen(proto, addr) 70 | if err != nil { 71 | klog.Errorln(err) 72 | os.Exit(1) 73 | } 74 | 75 | klog.InfoS("Listening for connections", "address", listener.Addr()) 76 | 77 | go func() { 78 | klog.Errorln(<-utils.WatchFile(addr)) 79 | os.Exit(1) 80 | }() 81 | 82 | if err := s.Serve(listener); err != nil { 83 | klog.Errorln(err) 84 | os.Exit(1) 85 | } 86 | 87 | klog.Fatalln("GRPC service has stopped gracefully") 88 | } 89 | -------------------------------------------------------------------------------- /providers/awskms/pkg/awskms/awskms.go: -------------------------------------------------------------------------------- 1 | package awskms 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "regexp" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/credentials" 10 | "github.com/aws/aws-sdk-go/aws/endpoints" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/kms" 13 | "github.com/aws/aws-sdk-go/service/sts" 14 | "github.com/ondat/trousseau/pkg/logger" 15 | "github.com/ondat/trousseau/pkg/providers" 16 | "github.com/ondat/trousseau/pkg/utils" 17 | "github.com/ondat/trousseau/pkg/version" 18 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 19 | "k8s.io/klog/v2" 20 | ) 21 | 22 | const maxRoleSessionNameLength = 64 23 | 24 | var ( 25 | keyArnRegexp = regexp.MustCompile(`^arn:aws[\w-]*:kms:(.+):\d+:(key|alias)/.+$`) 26 | hostnameCleanupRegexp = regexp.MustCompile("[^a-zA-Z0-9=,.@-]+") 27 | ) 28 | 29 | // Handle all communication with AWS KMS server. 30 | type awsKmsWrapper struct { 31 | config *Config 32 | sessionOpts session.Options 33 | hostname string 34 | region string 35 | } 36 | 37 | // New creates an instance of the KMS client. 38 | func New(config *Config, hostname string) (providers.EncryptionClient, error) { 39 | matches := keyArnRegexp.FindStringSubmatch(config.KeyArn) 40 | if matches == nil { 41 | klog.Error("No valid ARN found") 42 | return nil, fmt.Errorf("no valid ARN found in %s", config.KeyArn) 43 | } 44 | 45 | opts := session.Options{ 46 | Profile: config.Profile, 47 | Config: aws.Config{ 48 | Region: aws.String(matches[1]), 49 | CredentialsChainVerboseErrors: aws.Bool(true), 50 | }, 51 | SharedConfigState: session.SharedConfigEnable, 52 | } 53 | 54 | if config.Endpoint != "" { 55 | var resolver endpoints.ResolverFunc = func(_, _ string, _ ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) { 56 | return endpoints.ResolvedEndpoint{ 57 | URL: config.Endpoint, 58 | }, nil 59 | } 60 | 61 | opts.Config.Endpoint = aws.String(config.Endpoint) 62 | opts.Config.EndpointResolver = resolver 63 | opts.Config.DisableEndpointHostPrefix = aws.Bool(true) 64 | } 65 | 66 | return &awsKmsWrapper{ 67 | config: config, 68 | sessionOpts: opts, 69 | hostname: hostnameCleanupRegexp.ReplaceAllString(hostname, ""), 70 | region: matches[1], 71 | }, nil 72 | } 73 | 74 | // Encrypt encrypts input. 75 | func (c *awsKmsWrapper) Encrypt(data []byte) ([]byte, error) { 76 | klog.V(logger.Info3).InfoS("Encrypting...") 77 | 78 | sess, err := c.createSession() 79 | if err != nil { 80 | klog.InfoS("Unable to create session", "error", err.Error()) 81 | return nil, fmt.Errorf("unable to create session: %w", err) 82 | } 83 | 84 | klog.V(logger.Debug2).InfoS("Encrypting data", "data", utils.SecretToLog(string(data))) 85 | 86 | response, err := kms.New(sess, &c.sessionOpts.Config).Encrypt(&kms.EncryptInput{Plaintext: data, KeyId: &c.config.KeyArn, EncryptionContext: c.config.EncryptionContext}) 87 | if err != nil { 88 | klog.InfoS("Unable to encrypt data", "error", err.Error()) 89 | return nil, fmt.Errorf("unable to encrypt data: %w", err) 90 | } 91 | 92 | klog.V(logger.Debug2).InfoS("Encrypted data", "data", utils.SecretToLog(string(response.CiphertextBlob))) 93 | 94 | return []byte(base64.StdEncoding.EncodeToString(response.CiphertextBlob)), nil 95 | } 96 | 97 | // Decrypt decrypts input. 98 | func (c *awsKmsWrapper) Decrypt(data []byte) ([]byte, error) { 99 | klog.V(logger.Info3).InfoS("Decrypting...") 100 | 101 | decoded, err := base64.StdEncoding.DecodeString(string(data)) 102 | if err != nil { 103 | klog.InfoS("Failed decode encrypted data", "error", err.Error()) 104 | return nil, fmt.Errorf("failed decode encrypted data: %w", err) 105 | } 106 | 107 | sess, err := c.createSession() 108 | if err != nil { 109 | klog.InfoS("Unable to create session", "error", err.Error()) 110 | return nil, fmt.Errorf("unable to create session: %w", err) 111 | } 112 | 113 | klog.V(logger.Debug2).InfoS("Decrypting data", "data", utils.SecretToLog(string(data))) 114 | 115 | response, err := kms.New(sess, &c.sessionOpts.Config).Decrypt(&kms.DecryptInput{CiphertextBlob: decoded, EncryptionContext: c.config.EncryptionContext}) 116 | if err != nil { 117 | klog.InfoS("Unable to decrypt data", "error", err.Error()) 118 | return nil, fmt.Errorf("unable to decrypt data: %w", err) 119 | } 120 | 121 | klog.V(logger.Debug2).InfoS("Decrypted data", "data", utils.SecretToLog(string(response.Plaintext))) 122 | 123 | return response.Plaintext, nil 124 | } 125 | 126 | func (c *awsKmsWrapper) Version() *pb.VersionResponse { 127 | return &pb.VersionResponse{Version: version.APIVersion, RuntimeName: version.Runtime, RuntimeVersion: version.BuildVersion} 128 | } 129 | 130 | func (c *awsKmsWrapper) createSession() (*session.Session, error) { 131 | klog.V(logger.Info3).InfoS("Creating new session...") 132 | klog.V(logger.Debug1).InfoS("Creating new session", "options", c.sessionOpts) 133 | 134 | sess, err := session.NewSessionWithOptions(c.sessionOpts) 135 | if err != nil { 136 | klog.InfoS("Unable to create new session", "error", err.Error()) 137 | return nil, fmt.Errorf("unable to create new session: %w", err) 138 | } 139 | 140 | if c.config.RoleArn != "" { 141 | return c.createStsSession(sess) 142 | } 143 | 144 | return sess, nil 145 | } 146 | 147 | func (c *awsKmsWrapper) createStsSession(sess *session.Session) (*session.Session, error) { 148 | hostname := c.hostname 149 | if len(hostname) >= maxRoleSessionNameLength { 150 | hostname = hostname[:maxRoleSessionNameLength] 151 | } 152 | 153 | klog.V(logger.Info3).InfoS("Creating new STS session...", "hostname", hostname) 154 | 155 | stsService := sts.New(sess) 156 | 157 | out, err := stsService.AssumeRole(&sts.AssumeRoleInput{ 158 | RoleArn: &c.config.RoleArn, 159 | RoleSessionName: &hostname, 160 | }) 161 | if err != nil { 162 | klog.InfoS("Unable to assume role", "error", err.Error()) 163 | return nil, fmt.Errorf("unable to assume role: %w", err) 164 | } 165 | 166 | sess, err = session.NewSession(&c.sessionOpts.Config, &aws.Config{Credentials: credentials.NewStaticCredentials(*out.Credentials.AccessKeyId, *out.Credentials.SecretAccessKey, *out.Credentials.SessionToken)}) 167 | if err != nil { 168 | klog.InfoS("Unable to create new session", "error", err.Error()) 169 | return nil, fmt.Errorf("unable to create new session: %w", err) 170 | } 171 | 172 | return sess, nil 173 | } 174 | -------------------------------------------------------------------------------- /providers/awskms/pkg/awskms/awskms_test.go: -------------------------------------------------------------------------------- 1 | package awskms 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/ondat/trousseau/pkg/providers" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestNew(t *testing.T) { 13 | testcases := map[string]struct { 14 | config *Config 15 | hostname string 16 | validate func(*testing.T, providers.EncryptionClient, error) 17 | }{ 18 | "OK": { 19 | config: &Config{ 20 | KeyArn: "arn:aws:kms:eu-west-2:660555521754:key/0ac016bac031-a0ff-065c95b0-4ac2-91eb", 21 | }, 22 | hostname: "foo.bar", 23 | validate: func(t *testing.T, client providers.EncryptionClient, err error) { 24 | t.Helper() 25 | 26 | require.Nil(t, err) 27 | 28 | awsClient, ok := client.(*awsKmsWrapper) 29 | assert.True(t, ok, "Failed to cast client") 30 | assert.Equal(t, "foo.bar", awsClient.hostname) 31 | assert.Equal(t, "eu-west-2", awsClient.region) 32 | }, 33 | }, 34 | "Hostname cleanup": { 35 | config: &Config{ 36 | KeyArn: "arn:aws:kms:eu-west-2:660555521754:key/0ac016bac031-a0ff-065c95b0-4ac2-91eb", 37 | }, 38 | hostname: "fo&o.bar_bar", 39 | validate: func(t *testing.T, client providers.EncryptionClient, _ error) { 40 | t.Helper() 41 | 42 | awsClient, ok := client.(*awsKmsWrapper) 43 | assert.True(t, ok, "Failed to cast client") 44 | assert.Equal(t, "foo.barbar", awsClient.hostname) 45 | }, 46 | }, 47 | "Bad ARN": { 48 | config: &Config{ 49 | KeyArn: "X_arn:aws:kms:eu-west-2:660598621754:key/0ac016bac031-a0ff-065c95b0-4ac2-91eb", 50 | }, 51 | validate: func(t *testing.T, client providers.EncryptionClient, err error) { 52 | t.Helper() 53 | 54 | assert.Equal(t, err, errors.New("no valid ARN found in X_arn:aws:kms:eu-west-2:660598621754:key/0ac016bac031-a0ff-065c95b0-4ac2-91eb")) 55 | }, 56 | }, 57 | } 58 | 59 | for name, tc := range testcases { 60 | tc := tc 61 | 62 | t.Run(name, func(t *testing.T) { 63 | t.Parallel() 64 | 65 | for i := 0; i < 3; i++ { 66 | client, err := New(tc.config, tc.hostname) 67 | 68 | tc.validate(t, client, err) 69 | } 70 | }) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /providers/awskms/pkg/awskms/config.go: -------------------------------------------------------------------------------- 1 | package awskms 2 | 3 | // Config contains the details of connection. 4 | type Config struct { 5 | // Endpoint service endpoint, optional 6 | Endpoint string `json:"endpoint" yaml:"endpoint"` 7 | // Name of profile 8 | Profile string `json:"profile" yaml:"profile"` 9 | // Arn of key 10 | KeyArn string `json:"keyArn" yaml:"keyArn"` 11 | // Arn of role 12 | RoleArn string `json:"roleArn" yaml:"roleArn"` 13 | // Context of encryption, optional 14 | EncryptionContext map[string]*string `json:"encryptionContext,omitempty" yaml:"encryptionContext,omitempty"` 15 | } 16 | -------------------------------------------------------------------------------- /providers/azurekms/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ondat/trousseau/providers/azurekms 2 | 3 | go 1.18 4 | 5 | replace github.com/ondat/trousseau => ../.. 6 | 7 | require ( 8 | github.com/Azure/kubernetes-kms v0.3.0 9 | github.com/ondat/trousseau v0.0.0-00010101000000-000000000000 10 | google.golang.org/grpc v1.47.0 11 | k8s.io/apiserver v0.24.2 12 | k8s.io/klog/v2 v2.70.0 13 | ) 14 | 15 | require ( 16 | github.com/Azure/azure-sdk-for-go v63.0.0+incompatible // indirect 17 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 18 | github.com/Azure/go-autorest/autorest v0.11.25 // indirect 19 | github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect 20 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect 21 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect 22 | github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect 23 | github.com/Azure/go-autorest/logger v0.2.1 // indirect 24 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect 25 | github.com/beorn7/perks v1.0.1 // indirect 26 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 27 | github.com/go-logr/logr v1.2.3 // indirect 28 | github.com/go-logr/stdr v1.2.2 // indirect 29 | github.com/go-logr/zapr v1.2.3 // indirect 30 | github.com/gogo/protobuf v1.3.2 // indirect 31 | github.com/golang-jwt/jwt/v4 v4.2.0 // indirect 32 | github.com/golang/protobuf v1.5.2 // indirect 33 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 34 | github.com/prometheus/client_golang v1.12.1 // indirect 35 | github.com/prometheus/client_model v0.2.0 // indirect 36 | github.com/prometheus/common v0.32.1 // indirect 37 | github.com/prometheus/procfs v0.7.3 // indirect 38 | go.opentelemetry.io/otel v1.7.0 // indirect 39 | go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0 // indirect 40 | go.opentelemetry.io/otel/internal/metric v0.21.0 // indirect 41 | go.opentelemetry.io/otel/metric v0.21.0 // indirect 42 | go.opentelemetry.io/otel/sdk v1.0.0-RC1 // indirect 43 | go.opentelemetry.io/otel/sdk/export/metric v0.21.0 // indirect 44 | go.opentelemetry.io/otel/sdk/metric v0.21.0 // indirect 45 | go.opentelemetry.io/otel/trace v1.7.0 // indirect 46 | go.uber.org/atomic v1.9.0 // indirect 47 | go.uber.org/multierr v1.6.0 // indirect 48 | go.uber.org/zap v1.21.0 // indirect 49 | golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect 50 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 51 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 52 | golang.org/x/text v0.3.7 // indirect 53 | google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect 54 | google.golang.org/protobuf v1.28.0 // indirect 55 | gopkg.in/yaml.v2 v2.4.0 // indirect 56 | ) 57 | -------------------------------------------------------------------------------- /providers/azurekms/localdev.md: -------------------------------------------------------------------------------- 1 | # Local development 2 | 3 | This document describes how to develop Trousseau Azure KMS provider on your local machine. 4 | 5 | Please follow base documentation at [localdev.md](/../../localdev.md) 6 | 7 | ## Login to Azure 8 | 9 | Log in and create config file at [azurekms.json](/../../tests/e2e/kuttl/kube-v1.24/azurekms.json). 10 | 11 | ```json 12 | { 13 | "cloud":"AzurePublicCloud", 14 | "tenantId": "...", 15 | "aadClientId": "...", 16 | "aadClientSecret": "...", 17 | "subscriptionId": "..." 18 | } 19 | ``` 20 | 21 | Edit config file at [azurekms.yaml](/../../tests/e2e/kuttl/kube-v1.24/azurekms.yaml): 22 | 23 | ```yaml 24 | configFilePath: configFilePath 25 | keyVaultName: keyVaultName 26 | keyName: keyName 27 | keyVersion: keyVersion 28 | ``` 29 | 30 | ## Run Trousseau components 31 | 32 | Use command line or our favorite IDE to start Trousseau components on your machine: 33 | 34 | ```bash 35 | task go:run:proxy 36 | task go:run:azurekms 37 | ENABLED_PROVIDERS="--enabled-providers=azurekms" task go:run:trousseau 38 | ``` -------------------------------------------------------------------------------- /providers/azurekms/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "net" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | 11 | "github.com/Azure/kubernetes-kms/pkg/plugin" 12 | "github.com/Azure/kubernetes-kms/pkg/utils" 13 | "github.com/Azure/kubernetes-kms/pkg/version" 14 | "github.com/ondat/trousseau/pkg/logger" 15 | trutils "github.com/ondat/trousseau/pkg/utils" 16 | "github.com/ondat/trousseau/providers/azurekms/pkg/azurekms" 17 | "google.golang.org/grpc" 18 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 19 | "k8s.io/klog/v2" 20 | ) 21 | 22 | var ( 23 | listenAddr = flag.String("listen-addr", "unix:///opt/trousseau-kms/azurekms/azurekms.socket", "gRPC listen address") 24 | configFilePath = flag.String("config-file-path", "/opt/trousseau-kms/azurekms/config.yaml", "Path for Azure KMS Provider config file") 25 | logEncoder = flag.String("zap-encoder", "console", "set log encoder [console, json]") 26 | ) 27 | 28 | func main() { 29 | flag.Parse() 30 | 31 | err := logger.InitializeLogging(*logEncoder) 32 | if err != nil { 33 | klog.Errorln(err) 34 | os.Exit(1) 35 | } 36 | 37 | cfg := azurekms.Config{} 38 | if err = trutils.ParseConfig(*configFilePath, &cfg); err != nil { 39 | klog.Errorln(err) 40 | os.Exit(1) 41 | } 42 | 43 | ctx := withShutdownSignal(context.Background()) 44 | 45 | klog.InfoS("Starting KeyManagementServiceServer service", "version", version.BuildVersion, "buildDate", version.BuildDate) 46 | 47 | pc := &plugin.Config{ 48 | KeyVaultName: cfg.KeyVaultName, 49 | KeyName: cfg.KeyName, 50 | KeyVersion: cfg.KeyVersion, 51 | ManagedHSM: cfg.ManagedHMS, 52 | ConfigFilePath: cfg.ConfigFilePath, 53 | } 54 | 55 | kmsServer, err := plugin.New(ctx, pc) 56 | if err != nil { 57 | klog.ErrorS(err, "failed to create server") 58 | os.Exit(1) 59 | } 60 | 61 | // Initialize and run the GRPC server 62 | proto, addr, err := utils.ParseEndpoint(*listenAddr) 63 | if err != nil { 64 | klog.ErrorS(err, "failed to parse endpoint") 65 | os.Exit(1) 66 | } 67 | 68 | if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { 69 | klog.ErrorS(err, "failed to remove socket file", "addr", addr) 70 | os.Exit(1) 71 | } 72 | 73 | listener, err := net.Listen(proto, addr) 74 | if err != nil { 75 | klog.ErrorS(err, "failed to listen", "addr", addr, "proto", proto) 76 | os.Exit(1) 77 | } 78 | 79 | opts := []grpc.ServerOption{ 80 | grpc.UnaryInterceptor(utils.UnaryServerInterceptor), 81 | } 82 | 83 | s := grpc.NewServer(opts...) 84 | pb.RegisterKeyManagementServiceServer(s, kmsServer) 85 | 86 | klog.InfoS("Listening for connections", "addr", listener.Addr().String()) 87 | //nolint:errcheck // original implementation in plugin 88 | go s.Serve(listener) 89 | 90 | <-ctx.Done() 91 | // gracefully stop the grpc server 92 | klog.Info("terminating the server") 93 | s.GracefulStop() 94 | 95 | klog.Flush() 96 | // using os.Exit skips running deferred functions 97 | os.Exit(0) 98 | } 99 | 100 | // withShutdownSignal returns a copy of the parent context that will close if 101 | // the process receives termination signals. 102 | func withShutdownSignal(ctx context.Context) context.Context { 103 | signalChan := make(chan os.Signal, 1) 104 | signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, os.Interrupt) 105 | 106 | nctx, cancel := context.WithCancel(ctx) 107 | 108 | go func() { 109 | <-signalChan 110 | klog.Info("received shutdown signal") 111 | cancel() 112 | }() 113 | 114 | return nctx 115 | } 116 | -------------------------------------------------------------------------------- /providers/azurekms/pkg/azurekms/config.go: -------------------------------------------------------------------------------- 1 | package azurekms 2 | 3 | // Config contains the details of connection. 4 | type Config struct { 5 | // ConfigFilePath Path for Azure Cloud Provider config file 6 | ConfigFilePath string `json:"configFilePath" yaml:"configFilePath"` 7 | // KeyVaultName Azure Key Vault name 8 | KeyVaultName string `json:"keyVaultName" yaml:"keyVaultName"` 9 | // KeyName Azure Key Vault KMS key name 10 | KeyName string `json:"keyName" yaml:"keyName"` 11 | // KeyVersion Azure Key Vault KMS key version 12 | KeyVersion string `json:"keyVersion" yaml:"keyVersion"` 13 | // ManagedHMS Azure Key Vault Managed HSM 14 | ManagedHMS bool `json:"managedHSM,omitempty" yaml:"managedHSM,omitempty"` 15 | } 16 | -------------------------------------------------------------------------------- /providers/debug/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ondat/trousseau/providers/debug 2 | 3 | go 1.18 4 | 5 | replace github.com/ondat/trousseau => ../.. 6 | 7 | require ( 8 | github.com/ondat/trousseau v0.0.0-00010101000000-000000000000 9 | google.golang.org/grpc v1.47.0 10 | k8s.io/apiserver v0.24.2 11 | k8s.io/klog/v2 v2.70.0 12 | ) 13 | 14 | require ( 15 | github.com/go-logr/logr v1.2.3 // indirect 16 | github.com/go-logr/zapr v1.2.3 // indirect 17 | github.com/gogo/protobuf v1.3.2 // indirect 18 | github.com/golang/protobuf v1.5.2 // indirect 19 | go.uber.org/atomic v1.9.0 // indirect 20 | go.uber.org/multierr v1.6.0 // indirect 21 | go.uber.org/zap v1.21.0 // indirect 22 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 23 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 24 | golang.org/x/text v0.3.7 // indirect 25 | google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect 26 | google.golang.org/protobuf v1.28.0 // indirect 27 | gopkg.in/yaml.v2 v2.4.0 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /providers/debug/localdev.md: -------------------------------------------------------------------------------- 1 | # Local development 2 | 3 | This document describes how to develop Trousseau Debug provider on your local machine. 4 | 5 | Please follow base documentation at [localdev.md](../localdev.md) 6 | -------------------------------------------------------------------------------- /providers/debug/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "os" 9 | 10 | "github.com/ondat/trousseau/pkg/logger" 11 | "github.com/ondat/trousseau/pkg/providers" 12 | "github.com/ondat/trousseau/pkg/utils" 13 | "google.golang.org/grpc" 14 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 15 | "k8s.io/klog/v2" 16 | ) 17 | 18 | const logEncoder = "console" 19 | 20 | var listenAddr = flag.String("listen-addr", "unix:///opt/trousseau-kms/debug/debug.socket", "gRPC listen address") 21 | 22 | func main() { 23 | flag.Parse() 24 | 25 | err := logger.InitializeLogging(logEncoder) 26 | if err != nil { 27 | klog.Errorln(err) 28 | os.Exit(1) 29 | } 30 | 31 | opts := []grpc.ServerOption{ 32 | grpc.UnaryInterceptor(utils.UnaryServerInterceptor), 33 | } 34 | 35 | s := grpc.NewServer(opts...) 36 | pb.RegisterKeyManagementServiceServer(s, &providers.KeyManagementServiceServer{ 37 | Client: &service{}, 38 | }) 39 | 40 | proto, addr, err := utils.ParseEndpoint(*listenAddr) 41 | if err != nil { 42 | klog.Errorln(err) 43 | os.Exit(1) 44 | } 45 | 46 | if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { 47 | klog.ErrorS(err, "unable to delete socket file", "file", addr) 48 | } 49 | 50 | listener, err := net.Listen(proto, addr) 51 | if err != nil { 52 | klog.Errorln(err) 53 | os.Exit(1) 54 | } 55 | 56 | klog.InfoS("Listening for connections", "address", listener.Addr()) 57 | 58 | go func() { 59 | klog.Errorln(<-utils.WatchFile(addr)) 60 | os.Exit(1) 61 | }() 62 | 63 | if err := s.Serve(listener); err != nil { 64 | klog.Errorln(err) 65 | os.Exit(1) 66 | } 67 | 68 | klog.Fatalln("GRPC service has stopped gracefully") 69 | } 70 | 71 | type service struct{} 72 | 73 | func (s *service) Encrypt(data []byte) ([]byte, error) { 74 | klog.InfoS("Encrypt", "data", string(data)) 75 | 76 | return []byte(base64.StdEncoding.EncodeToString(data)), nil 77 | } 78 | 79 | func (s *service) Decrypt(data []byte) ([]byte, error) { 80 | decoded, err := base64.StdEncoding.DecodeString(string(data)) 81 | if err != nil { 82 | klog.InfoS("Failed decode encrypted data", "error", err.Error()) 83 | return nil, fmt.Errorf("failed decode encrypted data: %w", err) 84 | } 85 | 86 | klog.InfoS("Decrypt", "data", string(decoded)) 87 | 88 | return decoded, nil 89 | } 90 | 91 | func (s *service) Version() *pb.VersionResponse { 92 | return &pb.VersionResponse{ 93 | Version: "debug", 94 | RuntimeName: "debug", 95 | RuntimeVersion: "0.0.0", 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /providers/vault/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ondat/trousseau/providers/vault 2 | 3 | go 1.18 4 | 5 | replace github.com/ondat/trousseau => ../.. 6 | 7 | require ( 8 | github.com/hashicorp/vault/api v1.7.2 9 | github.com/ondat/trousseau v0.0.0-00010101000000-000000000000 10 | google.golang.org/grpc v1.48.0 11 | k8s.io/apiserver v0.24.2 12 | k8s.io/klog/v2 v2.70.0 13 | ) 14 | 15 | require ( 16 | github.com/armon/go-metrics v0.3.9 // indirect 17 | github.com/armon/go-radix v1.0.0 // indirect 18 | github.com/cenkalti/backoff/v3 v3.0.0 // indirect 19 | github.com/fatih/color v1.7.0 // indirect 20 | github.com/go-logr/logr v1.2.3 // indirect 21 | github.com/go-logr/zapr v1.2.3 // indirect 22 | github.com/gogo/protobuf v1.3.2 // indirect 23 | github.com/golang/protobuf v1.5.2 // indirect 24 | github.com/golang/snappy v0.0.4 // indirect 25 | github.com/hashicorp/errwrap v1.1.0 // indirect 26 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 27 | github.com/hashicorp/go-hclog v0.16.2 // indirect 28 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 29 | github.com/hashicorp/go-multierror v1.1.1 // indirect 30 | github.com/hashicorp/go-plugin v1.4.3 // indirect 31 | github.com/hashicorp/go-retryablehttp v0.6.6 // indirect 32 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 33 | github.com/hashicorp/go-secure-stdlib/mlock v0.1.1 // indirect 34 | github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect 35 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 36 | github.com/hashicorp/go-sockaddr v1.0.2 // indirect 37 | github.com/hashicorp/go-uuid v1.0.2 // indirect 38 | github.com/hashicorp/go-version v1.2.0 // indirect 39 | github.com/hashicorp/golang-lru v0.5.4 // indirect 40 | github.com/hashicorp/hcl v1.0.0 // indirect 41 | github.com/hashicorp/vault/sdk v0.5.1 // indirect 42 | github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect 43 | github.com/mattn/go-colorable v0.1.6 // indirect 44 | github.com/mattn/go-isatty v0.0.12 // indirect 45 | github.com/mitchellh/copystructure v1.0.0 // indirect 46 | github.com/mitchellh/go-homedir v1.1.0 // indirect 47 | github.com/mitchellh/go-testing-interface v1.0.0 // indirect 48 | github.com/mitchellh/mapstructure v1.5.0 // indirect 49 | github.com/mitchellh/reflectwalk v1.0.0 // indirect 50 | github.com/oklog/run v1.0.0 // indirect 51 | github.com/pierrec/lz4 v2.5.2+incompatible // indirect 52 | github.com/ryanuber/go-glob v1.0.0 // indirect 53 | github.com/stretchr/objx v0.4.0 // indirect 54 | go.uber.org/atomic v1.9.0 // indirect 55 | go.uber.org/multierr v1.6.0 // indirect 56 | go.uber.org/zap v1.21.0 // indirect 57 | golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect 58 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 59 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 60 | golang.org/x/text v0.3.7 // indirect 61 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect 62 | google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect 63 | google.golang.org/protobuf v1.28.0 // indirect 64 | gopkg.in/square/go-jose.v2 v2.5.1 // indirect 65 | gopkg.in/yaml.v2 v2.4.0 // indirect 66 | ) 67 | -------------------------------------------------------------------------------- /providers/vault/localdev.md: -------------------------------------------------------------------------------- 1 | # Local development 2 | 3 | This document describes how to develop Trousseau Vault provider on your local machine. 4 | 5 | Please follow base documentation at [localdev.md](/../../localdev.md) 6 | 7 | ## Create Vault in developer mode 8 | 9 | To spin up a Vault localy please execute the following command: 10 | 11 | ```bash 12 | docker run --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=vault-kms-demo' -p 8200:8200 -d --name=trousseau-local-vault vault 13 | ``` 14 | 15 | You can validate your Vault instance by performing a login: 16 | 17 | ```bash 18 | docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault login vault-kms-demo 19 | ``` 20 | 21 | Enable transit engine: 22 | ```bash 23 | docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault secrets enable transit 24 | ``` 25 | 26 | ## Run Trousseau components 27 | 28 | Use command line or our favorite IDE to start Trousseau components on your machine: 29 | 30 | ```bash 31 | task go:run:proxy 32 | task go:run:vault 33 | ENABLED_PROVIDERS="--enabled-providers=vault" task go:run:trousseau 34 | ``` 35 | -------------------------------------------------------------------------------- /providers/vault/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "net" 6 | "os" 7 | 8 | "github.com/ondat/trousseau/pkg/logger" 9 | "github.com/ondat/trousseau/pkg/providers" 10 | "github.com/ondat/trousseau/pkg/utils" 11 | "github.com/ondat/trousseau/providers/vault/pkg/vault" 12 | "google.golang.org/grpc" 13 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 14 | "k8s.io/klog/v2" 15 | ) 16 | 17 | var ( 18 | listenAddr = flag.String("listen-addr", "unix:///opt/trousseau-kms/vault/vault.socket", "gRPC listen address") 19 | configFilePath = flag.String("config-file-path", "/opt/trousseau-kms/vault/config.yaml", "Path for Vault Provider config file") 20 | logEncoder = flag.String("zap-encoder", "console", "set log encoder [console, json]") 21 | ) 22 | 23 | func main() { 24 | flag.Parse() 25 | 26 | err := logger.InitializeLogging(*logEncoder) 27 | if err != nil { 28 | klog.Errorln(err) 29 | os.Exit(1) 30 | } 31 | 32 | cfg := vault.Config{} 33 | if err = utils.ParseConfig(*configFilePath, &cfg); err != nil { 34 | klog.Errorln(err) 35 | os.Exit(1) 36 | } 37 | 38 | client, err := vault.New(&cfg) 39 | if err != nil { 40 | klog.Errorln(err) 41 | os.Exit(1) 42 | } 43 | 44 | opts := []grpc.ServerOption{ 45 | grpc.UnaryInterceptor(utils.UnaryServerInterceptor), 46 | } 47 | 48 | s := grpc.NewServer(opts...) 49 | pb.RegisterKeyManagementServiceServer(s, &providers.KeyManagementServiceServer{ 50 | Client: client, 51 | }) 52 | 53 | proto, addr, err := utils.ParseEndpoint(*listenAddr) 54 | if err != nil { 55 | klog.Errorln(err) 56 | os.Exit(1) 57 | } 58 | 59 | if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { 60 | klog.ErrorS(err, "unable to delete socket file", "file", addr) 61 | } 62 | 63 | listener, err := net.Listen(proto, addr) 64 | if err != nil { 65 | klog.Errorln(err) 66 | os.Exit(1) 67 | } 68 | 69 | klog.InfoS("Listening for connections", "address", listener.Addr()) 70 | 71 | go func() { 72 | klog.Errorln(<-utils.WatchFile(addr)) 73 | os.Exit(1) 74 | }() 75 | 76 | if err := s.Serve(listener); err != nil { 77 | klog.Errorln(err) 78 | os.Exit(1) 79 | } 80 | 81 | klog.Fatalln("GRPC service has stopped gracefully") 82 | } 83 | -------------------------------------------------------------------------------- /providers/vault/pkg/vault/config.go: -------------------------------------------------------------------------------- 1 | package vault 2 | 3 | // Config contains the details of connection. 4 | type Config struct { 5 | // The names of encryption key for Vault transit communication 6 | KeyNames []string `json:"keyNames" yaml:"keyNames"` 7 | 8 | // Vault listen address, for example https://localhost:8200 9 | Address string `json:"address" yaml:"address"` 10 | 11 | // Token authentication information 12 | Token string `json:"token" yaml:"token"` 13 | 14 | // TLS certificate authentication information 15 | ClientCert string `json:"clientCert" yaml:"clientCert"` 16 | ClientKey string `json:"clientKey" yaml:"clientKey"` 17 | 18 | // AppRole authentication information 19 | RoleID string `json:"roleID" yaml:"roleID"` 20 | SecretID string `json:"secretID" yaml:"secretID"` 21 | 22 | // CACert is the path to a PEM-encoded CA cert file to use to verify the 23 | // Vault server SSL certificate. 24 | VaultCACert string `json:"vaultCACert" yaml:"vaultCACert"` 25 | 26 | // TLSServerName, if set, is used to set the SNI host when connecting via TLS. 27 | TLSServerName string `json:"tlsServerName" yaml:"tlsServerName"` 28 | 29 | // The path for transit API, default is "transit" 30 | TransitPath string `json:"transitPath" yaml:"transitPath"` 31 | 32 | // The path for auth backend, default is "auth" 33 | AuthPath string `json:"authPath" yaml:"authPath"` 34 | } 35 | -------------------------------------------------------------------------------- /providers/vault/pkg/vault/errors.go: -------------------------------------------------------------------------------- 1 | package vault 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Return this error when get HTTP code 403. 8 | type forbiddenError struct { 9 | error 10 | } 11 | 12 | // Error string representation of the error. 13 | func (e *forbiddenError) Error() string { 14 | return fmt.Sprintf("forbidden error %s", e.error) 15 | } 16 | 17 | func newForbiddenError(err error) error { 18 | return &forbiddenError{error: err} 19 | } 20 | -------------------------------------------------------------------------------- /proxy/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ondat/trousseau/proxy 2 | 3 | go 1.18 4 | 5 | replace github.com/ondat/trousseau => ../ 6 | 7 | require ( 8 | github.com/ondat/trousseau v0.0.0-00010101000000-000000000000 9 | google.golang.org/grpc v1.47.0 10 | k8s.io/apiserver v0.24.2 11 | ) 12 | 13 | require ( 14 | github.com/go-logr/logr v1.2.3 // indirect 15 | github.com/go-logr/zapr v1.2.3 // indirect 16 | github.com/gogo/protobuf v1.3.2 // indirect 17 | github.com/golang/protobuf v1.5.2 // indirect 18 | go.uber.org/atomic v1.9.0 // indirect 19 | go.uber.org/multierr v1.6.0 // indirect 20 | go.uber.org/zap v1.21.0 // indirect 21 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 22 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 23 | golang.org/x/text v0.3.7 // indirect 24 | google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect 25 | google.golang.org/protobuf v1.28.0 // indirect 26 | gopkg.in/yaml.v2 v2.4.0 // indirect 27 | k8s.io/klog/v2 v2.70.0 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "net" 7 | "os" 8 | "time" 9 | 10 | "github.com/ondat/trousseau/pkg/utils" 11 | "github.com/ondat/trousseau/proxy/pkg/server" 12 | "google.golang.org/grpc" 13 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 14 | ) 15 | 16 | const ( 17 | defaultSocketTimeout = 5 * time.Second 18 | ) 19 | 20 | var ( 21 | listenAddr = flag.String("listen-addr", "unix:///opt/trousseau-kms/proxy.socket", "gRPC listen address") 22 | trousseauAddr = flag.String("trousseau-addr", "/opt/trousseau-kms/trousseau.socket", "gRPC listen address") 23 | socketTimeout = flag.Duration("socket-timeout", defaultSocketTimeout, "RPC timeout for Trousseau socket") 24 | ) 25 | 26 | func main() { 27 | flag.Parse() 28 | 29 | proto, addr, err := utils.ParseEndpoint(*listenAddr) 30 | if err != nil { 31 | log.Fatal(err.Error()) 32 | } 33 | 34 | if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { 35 | log.Printf("Unable to delete socket file: %s: %s\n", addr, err.Error()) 36 | } 37 | 38 | listener, err := net.Listen(proto, addr) 39 | if err != nil { 40 | log.Fatal(err.Error()) 41 | } 42 | 43 | opts := []grpc.ServerOption{ 44 | grpc.UnaryInterceptor(utils.UnaryServerInterceptor), 45 | } 46 | s := grpc.NewServer(opts...) 47 | pb.RegisterKeyManagementServiceServer(s, server.New(*trousseauAddr, *socketTimeout)) 48 | 49 | log.Println("Listening for connections on address: " + listener.Addr().String()) 50 | 51 | if err := s.Serve(listener); err != nil { 52 | log.Fatal(err.Error()) 53 | } 54 | 55 | log.Println("GRPC service has stopped gracefully") 56 | } 57 | -------------------------------------------------------------------------------- /proxy/pkg/server/grpc.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/ondat/trousseau/pkg/providers" 10 | "github.com/ondat/trousseau/pkg/utils" 11 | "github.com/ondat/trousseau/pkg/version" 12 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 13 | ) 14 | 15 | type proxyService struct { 16 | trousseauSocket string 17 | timeout time.Duration 18 | } 19 | 20 | // New creates an instance of the Proxy Server. 21 | func New(trousseauSocket string, timeout time.Duration) providers.KeyManagementService { 22 | return &proxyService{ 23 | trousseauSocket: trousseauSocket, 24 | timeout: timeout, 25 | } 26 | } 27 | 28 | // Encrypt encryption requet handler. 29 | func (k *proxyService) Encrypt(ctx context.Context, data *pb.EncryptRequest) (*pb.EncryptResponse, error) { 30 | conn, err := utils.DialUnixSocket(k.trousseauSocket) 31 | if err != nil { 32 | log.Printf("Unable to connect to %s: %s", k.trousseauSocket, err.Error()) 33 | 34 | return nil, err 35 | } 36 | defer conn.Close() 37 | 38 | for i := 0; i < 5; i++ { 39 | ctx, cancel := context.WithTimeout(context.Background(), k.timeout) 40 | 41 | kmsClient := pb.NewKeyManagementServiceClient(conn) 42 | 43 | res, err := kmsClient.Encrypt(ctx, data) 44 | 45 | cancel() 46 | 47 | if err != nil { 48 | log.Printf("Unable to encrypt data: %s", err.Error()) 49 | 50 | continue 51 | } 52 | 53 | return res, err 54 | } 55 | 56 | return nil, fmt.Errorf("connection error on %s", k.trousseauSocket) 57 | } 58 | 59 | // Decrypt decryption requet handler. 60 | func (k *proxyService) Decrypt(ctx context.Context, data *pb.DecryptRequest) (*pb.DecryptResponse, error) { 61 | conn, err := utils.DialUnixSocket(k.trousseauSocket) 62 | if err != nil { 63 | log.Printf("Unable to connect to %s: %s", k.trousseauSocket, err.Error()) 64 | 65 | return nil, err 66 | } 67 | 68 | for i := 0; i < 5; i++ { 69 | ctx, cancel := context.WithTimeout(context.Background(), k.timeout) 70 | 71 | kmsClient := pb.NewKeyManagementServiceClient(conn) 72 | 73 | res, err := kmsClient.Decrypt(ctx, data) 74 | 75 | cancel() 76 | 77 | if err != nil { 78 | log.Printf("Unable to decrypt data: %s", err.Error()) 79 | 80 | continue 81 | } 82 | 83 | return res, err 84 | } 85 | 86 | return nil, fmt.Errorf("connection error on %s", k.trousseauSocket) 87 | } 88 | 89 | // Version version of gRPS server. 90 | func (k *proxyService) Version(context.Context, *pb.VersionRequest) (*pb.VersionResponse, error) { 91 | return &pb.VersionResponse{Version: version.APIVersion, RuntimeName: version.Runtime, RuntimeVersion: version.BuildVersion}, nil 92 | } 93 | -------------------------------------------------------------------------------- /scripts/generic/vault-kms-provider.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: DaemonSet 4 | metadata: 5 | name: vault-kms-provider 6 | namespace: kube-system 7 | labels: 8 | tier: control-plane 9 | app: vault-kms-provider 10 | spec: 11 | selector: 12 | matchLabels: 13 | name: vault-kms-provider 14 | template: 15 | metadata: 16 | labels: 17 | name: vault-kms-provider 18 | spec: 19 | priorityClassName: system-cluster-critical 20 | hostNetwork: true 21 | containers: 22 | - name: vault-kms-provider 23 | image: ghcr.io/ondat/trousseau:v1.1.0 24 | imagePullPolicy: Always 25 | env: # Extra Vault variables 26 | - name: VAULT_NAMESPACE # For Enterprise - set the value for the namespace 27 | value: admin 28 | - name: VAULT_SKIP_VERIFY # For vault deployment with a self-signed certificate 29 | value: "false" 30 | args: 31 | - -v=5 32 | - --enabled-providers=vault 33 | - --socket-location=/opt/trousseau-kms 34 | - --listen-addr=unix:///opt/trousseau-kms/proxy.socket # [REQUIRED] Version of the key to use 35 | - --zap-encoder=json 36 | - --v=3 37 | securityContext: 38 | allowPrivilegeEscalation: false 39 | capabilities: 40 | drop: 41 | - ALL 42 | readOnlyRootFilesystem: true 43 | runAsUser: 10123 44 | ports: 45 | - containerPort: 8787 46 | protocol: TCP 47 | livenessProbe: 48 | httpGet: 49 | path: /healthz 50 | port: 8787 51 | failureThreshold: 3 52 | periodSeconds: 10 53 | resources: 54 | requests: 55 | cpu: 50m 56 | memory: 64Mi 57 | limits: 58 | cpu: 300m 59 | memory: 256Mi 60 | volumeMounts: 61 | - name: vault-kms 62 | mountPath: /opt/trousseau-kms 63 | volumes: 64 | - name: vault-kms 65 | hostPath: 66 | path: /opt/trousseau-kms 67 | affinity: 68 | nodeAffinity: 69 | requiredDuringSchedulingIgnoredDuringExecution: 70 | nodeSelectorTerms: 71 | - matchExpressions: 72 | - key: node-role.kubernetes.io/control-plane 73 | operator: Exists 74 | tolerations: 75 | - key: node-role.kubernetes.io/control-plane 76 | operator: Exists 77 | effect: NoSchedule 78 | - key: node-role.kubernetes.io/etcd 79 | operator: Exists 80 | effect: NoExecute 81 | -------------------------------------------------------------------------------- /scripts/hcvault/archives/k8s/encryption-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: EncryptionConfiguration 3 | apiVersion: apiserver.config.k8s.io/v1 4 | resources: 5 | - resources: 6 | - secrets 7 | providers: 8 | - kms: 9 | name: vaultprovider 10 | endpoint: unix:///opt/trousseau-kms/trousseau.socket 11 | cachesize: 1000 12 | - identity: {} -------------------------------------------------------------------------------- /scripts/hcvault/archives/k8s/kube-apiserver.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Pod 4 | metadata: 5 | annotations: 6 | kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 10.0.10.5:6443 7 | creationTimestamp: null 8 | labels: 9 | component: kube-apiserver 10 | tier: control-plane 11 | name: kube-apiserver 12 | namespace: kube-system 13 | spec: 14 | containers: 15 | - command: 16 | - kube-apiserver 17 | - --advertise-address=10.0.10.5 18 | - --allow-privileged=true 19 | - --anonymous-auth=True 20 | - --apiserver-count=3 21 | - --authorization-mode=Node,RBAC 22 | - --bind-address=0.0.0.0 23 | - --client-ca-file=/etc/kubernetes/ssl/ca.crt 24 | - --default-not-ready-toleration-seconds=300 25 | - --default-unreachable-toleration-seconds=300 26 | - --enable-admission-plugins=NodeRestriction 27 | - --enable-aggregator-routing=False 28 | - --enable-bootstrap-token-auth=true 29 | - --encryption-provider-config=/opt/trousseau-kms/encryption_config.yaml # Add this line 30 | - --endpoint-reconciler-type=lease 31 | - --etcd-cafile=/etc/ssl/etcd/ssl/ca.pem 32 | - --etcd-certfile=/etc/ssl/etcd/ssl/node-development-ha-master-0.pem 33 | - --etcd-keyfile=/etc/ssl/etcd/ssl/node-development-ha-master-0-key.pem 34 | - --etcd-servers=https://10.0.10.5:2379,https://10.0.10.4:2379,https://10.0.10.3:2379 35 | - --event-ttl=1h0m0s 36 | - --insecure-port=0 37 | - --kubelet-client-certificate=/etc/kubernetes/ssl/apiserver-kubelet-client.crt 38 | - --kubelet-client-key=/etc/kubernetes/ssl/apiserver-kubelet-client.key 39 | - --kubelet-preferred-address-types=InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP 40 | - --profiling=False 41 | - --proxy-client-cert-file=/etc/kubernetes/ssl/front-proxy-client.crt 42 | - --proxy-client-key-file=/etc/kubernetes/ssl/front-proxy-client.key 43 | - --request-timeout=1m0s 44 | - --requestheader-allowed-names=front-proxy-client 45 | - --requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.crt 46 | - --requestheader-extra-headers-prefix=X-Remote-Extra- 47 | - --requestheader-group-headers=X-Remote-Group 48 | - --requestheader-username-headers=X-Remote-User 49 | - --secure-port=6443 50 | - --service-account-issuer=https://kubernetes.default.svc.cluster.local 51 | - --service-account-key-file=/etc/kubernetes/ssl/sa.pub 52 | - --service-account-signing-key-file=/etc/kubernetes/ssl/sa.key 53 | - --service-cluster-ip-range=10.233.0.0/18 54 | - --service-node-port-range=30000-32767 55 | - --storage-backend=etcd3 56 | - --tls-cert-file=/etc/kubernetes/ssl/apiserver.crt 57 | - --tls-private-key-file=/etc/kubernetes/ssl/apiserver.key 58 | image: k8s.gcr.io/kube-apiserver:v1.21.6 59 | imagePullPolicy: IfNotPresent 60 | livenessProbe: 61 | failureThreshold: 8 62 | httpGet: 63 | host: 10.0.10.5 64 | path: /livez 65 | port: 6443 66 | scheme: HTTPS 67 | initialDelaySeconds: 10 68 | periodSeconds: 10 69 | timeoutSeconds: 15 70 | name: kube-apiserver 71 | readinessProbe: 72 | failureThreshold: 3 73 | httpGet: 74 | host: 10.0.10.5 75 | path: /readyz 76 | port: 6443 77 | scheme: HTTPS 78 | periodSeconds: 1 79 | timeoutSeconds: 15 80 | resources: 81 | requests: 82 | cpu: 250m 83 | startupProbe: 84 | failureThreshold: 30 85 | httpGet: 86 | host: 10.0.10.5 87 | path: /livez 88 | port: 6443 89 | scheme: HTTPS 90 | initialDelaySeconds: 10 91 | periodSeconds: 10 92 | timeoutSeconds: 15 93 | volumeMounts: 94 | - mountPath: /etc/ssl/certs 95 | name: ca-certs 96 | readOnly: true 97 | - mountPath: /etc/ca-certificates 98 | name: etc-ca-certificates 99 | readOnly: true 100 | - mountPath: /etc/ssl/etcd/ssl 101 | name: etcd-certs-0 102 | readOnly: true 103 | - mountPath: /etc/kubernetes/ssl 104 | name: k8s-certs 105 | readOnly: true 106 | - mountPath: /usr/local/share/ca-certificates 107 | name: usr-local-share-ca-certificates 108 | readOnly: true 109 | - mountPath: /usr/share/ca-certificates 110 | name: usr-share-ca-certificates 111 | readOnly: true 112 | - mountPath: /opt/trousseau-kms # Add this section 113 | name: vaultkms 114 | readOnly: true 115 | hostNetwork: true 116 | priorityClassName: system-node-critical 117 | volumes: 118 | - hostPath: 119 | path: /etc/ssl/certs 120 | type: DirectoryOrCreate 121 | name: ca-certs 122 | - hostPath: 123 | path: /etc/ca-certificates 124 | type: DirectoryOrCreate 125 | name: etc-ca-certificates 126 | - hostPath: 127 | path: /etc/ssl/etcd/ssl 128 | type: DirectoryOrCreate 129 | name: etcd-certs-0 130 | - hostPath: 131 | path: /etc/kubernetes/ssl 132 | type: DirectoryOrCreate 133 | name: k8s-certs 134 | - hostPath: 135 | path: /usr/local/share/ca-certificates 136 | type: DirectoryOrCreate 137 | name: usr-local-share-ca-certificates 138 | - hostPath: 139 | path: /usr/share/ca-certificates 140 | type: "" 141 | name: usr-share-ca-certificates 142 | - hostPath: 143 | path: /opt/trousseau-kms # Add this section 144 | type: "" 145 | name: vaultkms 146 | status: {} -------------------------------------------------------------------------------- /scripts/hcvault/archives/k8s/kubelet-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kubelet.config.k8s.io/v1beta1 3 | kind: KubeletConfiguration 4 | nodeStatusUpdateFrequency: "10s" 5 | failSwapOn: True 6 | authentication: 7 | anonymous: 8 | enabled: false 9 | webhook: 10 | enabled: True 11 | x509: 12 | clientCAFile: /etc/kubernetes/ssl/ca.crt 13 | authorization: 14 | mode: Webhook 15 | staticPodPath: /etc/kubernetes/manifests # Add this line 16 | cgroupDriver: systemd 17 | containerLogMaxFiles: 5 18 | containerLogMaxSize: 10Mi 19 | maxPods: 110 20 | address: 10.0.10.4 21 | readOnlyPort: 0 22 | healthzPort: 10248 23 | healthzBindAddress: 127.0.0.1 24 | kubeletCgroups: /systemd/system.slice 25 | clusterDomain: cluster.local 26 | protectKernelDefaults: true 27 | rotateCertificates: true 28 | clusterDNS: 29 | - 169.254.25.10 30 | kubeReserved: 31 | cpu: 200m 32 | memory: 512Mi 33 | resolvConf: "/run/systemd/resolve/resolv.conf" 34 | eventRecordQPS: 5 35 | shutdownGracePeriod: 60s 36 | shutdownGracePeriodCriticalPods: 20s -------------------------------------------------------------------------------- /scripts/hcvault/archives/k8s/trousseau-hcvault.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: DaemonSet 4 | metadata: 5 | name: trousseau-hcvault 6 | namespace: kube-system 7 | labels: 8 | tier: control-plane 9 | app: trousseau-hcvault 10 | spec: 11 | selector: 12 | matchLabels: 13 | name: trousseau-hcvault 14 | template: 15 | metadata: 16 | labels: 17 | name: trousseau-hcvault 18 | spec: 19 | priorityClassName: system-cluster-critical 20 | hostNetwork: true 21 | containers: 22 | - name: trousseau-hcvault 23 | image: ghcr.io/ondat/trousseau:v1.1.0 24 | imagePullPolicy: Always 25 | env: # Extra Vault variables 26 | - name: VAULT_NAMESPACE # For Enterprise - set the value for the namespace 27 | value: admin 28 | - name: VAULT_SKIP_VERIFY # For vault deployment with a self-signed certificate 29 | value: "false" # 30 | args: 31 | - -v=5 32 | - --enabled-providers=vault 33 | - --socket-location=/opt/trousseau-kms 34 | - --listen-addr=unix:///opt/trousseau-kms/proxy.socket # [REQUIRED] Version of the key to use 35 | - --zap-encoder=json 36 | - --v=3 37 | securityContext: 38 | allowPrivilegeEscalation: false 39 | capabilities: 40 | drop: 41 | - ALL 42 | readOnlyRootFilesystem: true 43 | runAsUser: 10123 44 | ports: 45 | - containerPort: 8787 46 | protocol: TCP 47 | livenessProbe: 48 | httpGet: 49 | path: /healthz 50 | port: 8787 51 | failureThreshold: 3 52 | periodSeconds: 10 53 | resources: 54 | requests: 55 | cpu: 50m 56 | memory: 64Mi 57 | limits: 58 | cpu: 300m 59 | memory: 256Mi 60 | volumeMounts: 61 | - name: vault-kms 62 | mountPath: /opt/trousseau-kms 63 | volumes: 64 | - name: vault-kms 65 | hostPath: 66 | path: /opt/trousseau-kms 67 | affinity: 68 | nodeAffinity: 69 | requiredDuringSchedulingIgnoredDuringExecution: 70 | nodeSelectorTerms: 71 | - matchExpressions: 72 | - key: node-role.kubernetes.io/control-plane 73 | operator: Exists 74 | tolerations: 75 | - key: node-role.kubernetes.io/control-plane 76 | operator: Exists 77 | effect: NoSchedule 78 | - key: node-role.kubernetes.io/etcd 79 | operator: Exists 80 | effect: NoExecute 81 | -------------------------------------------------------------------------------- /scripts/hcvault/archives/k8s/vault.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | keyNames: 3 | - vault-kms-demo 4 | address: https://addresss:8200 5 | token: s.oYpiOmnWL0PFDPS2ImJTdhRf.CxT2N -------------------------------------------------------------------------------- /scripts/hcvault/archives/monitoring/prometheus.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Service 3 | apiVersion: v1 4 | metadata: 5 | name: vault-kms-metrics 6 | namespace: kube-system 7 | labels: 8 | app: vault-kms-provider 9 | spec: 10 | selector: 11 | app: vault-kms-provider 12 | ports: 13 | - name: metrics 14 | port: 8095 15 | --- 16 | 17 | apiVersion: monitoring.coreos.com/v1 18 | kind: ServiceMonitor 19 | metadata: 20 | name: vault-kms-provider 21 | labels: 22 | release: prometheus 23 | spec: 24 | namespaceSelector: 25 | matchNames: 26 | - kube-system 27 | selector: 28 | matchLabels: 29 | app: vault-kms-provider 30 | endpoints: 31 | - port: metrics 32 | path: /metrics -------------------------------------------------------------------------------- /scripts/hcvault/archives/rke2/config.yaml: -------------------------------------------------------------------------------- 1 | # server: https://
:9345 #to edit/uncomment for second and third control plane node 2 | # token: #to edit/uncomment for second and third control plane node 3 | kube-apiserver-arg: 4 | - "--encryption-provider-config=/var/lib/rancher/rke2/server/cred/vault-kms-encryption-config.yaml" 5 | kube-apiserver-extra-mount: 6 | - "/opt/trousseau-kms:/opt/trousseau-kms" 7 | -------------------------------------------------------------------------------- /scripts/hcvault/archives/rke2/trousseau-hcvault.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: DaemonSet 4 | metadata: 5 | name: trousseau-hcvault 6 | namespace: kube-system 7 | labels: 8 | tier: control-plane 9 | app: trousseau-hcvault 10 | spec: 11 | selector: 12 | matchLabels: 13 | name: trousseau-hcvault 14 | template: 15 | metadata: 16 | labels: 17 | name: trousseau-hcvault 18 | spec: 19 | priorityClassName: system-cluster-critical 20 | hostNetwork: true 21 | containers: 22 | - name: trousseau-hcvault 23 | image: ghcr.io/ondat/trousseau:v1.1.0 24 | imagePullPolicy: Always 25 | env: # Extra Vault variables 26 | - name: VAULT_NAMESPACE # For Enterprise - set the value for the namespace 27 | value: admin 28 | - name: VAULT_SKIP_VERIFY # For vault deployment with a self-signed certificate 29 | value: "false" # 30 | args: 31 | - -v=5 32 | - --enabled-providers=vault 33 | - --socket-location=/opt/trousseau-kms 34 | - --listen-addr=unix:///opt/trousseau-kms/proxy.socket # [REQUIRED] Version of the key to use 35 | - --zap-encoder=json 36 | - --v=3 37 | securityContext: 38 | allowPrivilegeEscalation: false 39 | capabilities: 40 | drop: 41 | - ALL 42 | readOnlyRootFilesystem: true 43 | runAsUser: 10123 44 | ports: 45 | - containerPort: 8787 46 | protocol: TCP 47 | livenessProbe: 48 | httpGet: 49 | path: /healthz 50 | port: 8787 51 | failureThreshold: 3 52 | periodSeconds: 10 53 | resources: 54 | requests: 55 | cpu: 50m 56 | memory: 64Mi 57 | limits: 58 | cpu: 300m 59 | memory: 256Mi 60 | volumeMounts: 61 | - name: vault-kms 62 | mountPath: /opt/trousseau-kms 63 | volumes: 64 | - name: vault-kms 65 | hostPath: 66 | path: /opt/trousseau-kms 67 | affinity: 68 | nodeAffinity: 69 | requiredDuringSchedulingIgnoredDuringExecution: 70 | nodeSelectorTerms: 71 | - matchExpressions: 72 | - key: node-role.kubernetes.io/control-plane 73 | operator: Exists 74 | tolerations: 75 | - key: node-role.kubernetes.io/control-plane 76 | operator: Exists 77 | effect: NoSchedule 78 | - key: node-role.kubernetes.io/etcd 79 | operator: Exists 80 | effect: NoExecute 81 | -------------------------------------------------------------------------------- /scripts/hcvault/archives/testing/encryption-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: EncryptionConfiguration 3 | apiVersion: apiserver.config.k8s.io/v1 4 | resources: 5 | - resources: 6 | - secrets 7 | providers: 8 | - kms: 9 | name: vaultprovider 10 | endpoint: unix:///opt/trousseau-kms/trousseau.socket 11 | cachesize: 1000 12 | - identity: {} -------------------------------------------------------------------------------- /scripts/hcvault/archives/testing/kind-cluster.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Cluster 3 | apiVersion: kind.x-k8s.io/v1alpha4 4 | nodes: 5 | - role: control-plane 6 | extraMounts: 7 | - containerPath: /etc/kubernetes/encryption-config.yaml 8 | hostPath: scripts/hcvault/archives/testing/encryption-config.yaml 9 | readOnly: true 10 | propagation: None 11 | - containerPath: /etc/kubernetes/vault-kms-provider.yaml 12 | hostPath: tests/e2e/generated_manifests/config.yaml 13 | readOnly: true 14 | propagation: None 15 | - containerPath: /etc/kubernetes/manifests/kubernetes-kms.yaml 16 | hostPath: tests/e2e/generated_manifests/kms.yaml 17 | readOnly: true 18 | propagation: None 19 | kubeadmConfigPatches: 20 | - | 21 | kind: ClusterConfiguration 22 | apiServer: 23 | extraArgs: 24 | encryption-provider-config: "/etc/kubernetes/encryption-config.yaml" 25 | extraVolumes: 26 | - name: encryption-config 27 | hostPath: "/etc/kubernetes/encryption-config.yaml" 28 | mountPath: "/etc/kubernetes/encryption-config.yaml" 29 | readOnly: true 30 | pathType: File 31 | - name: sock-path 32 | hostPath: "/opt" 33 | mountPath: "/opt" -------------------------------------------------------------------------------- /scripts/hcvault/archives/testing/kms.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Pod 4 | metadata: 5 | name: vault-kms-provider 6 | namespace: kube-system 7 | labels: 8 | tier: control-plane 9 | app: vault-kms-provider 10 | spec: 11 | priorityClassName: system-node-critical 12 | hostNetwork: true 13 | containers: 14 | - name: vault-kms-provider 15 | image: ${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION} 16 | imagePullPolicy: Always 17 | args: 18 | - -v=5 19 | - --enabled-providers=vault 20 | - --socket-location=/opt/trousseau-kms 21 | - --listen-addr=unix:///opt/trousseau-kms/proxy.socket # [REQUIRED] Version of the key to use 22 | - --zap-encoder=json 23 | - --v=3 24 | securityContext: 25 | allowPrivilegeEscalation: false 26 | capabilities: 27 | drop: 28 | - ALL 29 | readOnlyRootFilesystem: true 30 | runAsUser: 10123 31 | ports: 32 | - containerPort: 8787 33 | protocol: TCP 34 | livenessProbe: 35 | httpGet: 36 | path: /healthz 37 | port: 8787 38 | failureThreshold: 2 39 | periodSeconds: 10 40 | resources: 41 | requests: 42 | cpu: 50m 43 | memory: 64Mi 44 | limits: 45 | cpu: 300m 46 | memory: 256Mi 47 | volumeMounts: 48 | - name: etc-kubernetes 49 | mountPath: /etc/kubernetes 50 | - name: etc-ssl 51 | mountPath: /etc/ssl 52 | readOnly: true 53 | - name: sock 54 | mountPath: /opt 55 | volumes: 56 | - name: etc-kubernetes 57 | hostPath: 58 | path: /etc/kubernetes 59 | - name: etc-ssl 60 | hostPath: 61 | path: /etc/ssl 62 | - name: sock 63 | hostPath: 64 | path: /opt 65 | -------------------------------------------------------------------------------- /scripts/hcvault/archives/testing/secret.yaml: -------------------------------------------------------------------------------- 1 | 2 | kind: Secret 3 | apiVersion: v1 4 | type: Opaque 5 | metadata: 6 | name: data-test 7 | namespace: default 8 | stringData: 9 | config.yaml: | 10 | provider: test 11 | --- 12 | 13 | kind: Secret 14 | apiVersion: v1 15 | type: Opaque 16 | metadata: 17 | name: data-test2 18 | namespace: default 19 | stringData: 20 | config.yaml: | 21 | provider: test2 -------------------------------------------------------------------------------- /scripts/hcvault/archives/testing/secret2.yaml: -------------------------------------------------------------------------------- 1 | 2 | kind: Secret 3 | apiVersion: v1 4 | type: Opaque 5 | metadata: 6 | name: data-test3 7 | namespace: default 8 | stringData: 9 | config.yaml: | 10 | provider: test2 -------------------------------------------------------------------------------- /scripts/hcvault/archives/testing/test.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ./bin/kubectl apply -f scripts/secret.yaml 3 | for (( c=1; c<=100; c++ )) 4 | do 5 | ./bin/kubectl get secret data-test -o yaml > /dev/null 6 | done 7 | 8 | for (( c=1; c<=100; c++ )) 9 | do 10 | ./bin/kubectl get secret data-test2 -o yaml > /dev/null 11 | done 12 | -------------------------------------------------------------------------------- /scripts/hcvault/hcvault-agent/trousseau-hcvault-configmap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: trousseau-vault-agent-config 6 | namespace: kube-system 7 | data: 8 | vault-agent-config.hcl: | 9 | exit_after_auth = true 10 | pid_file = "/home/vault/pidfile" 11 | auto_auth { 12 | method "kubernetes" { 13 | mount_path = "auth/kubernetes" 14 | config = { 15 | role = "trousseau" 16 | } 17 | } 18 | sink "file" { 19 | config = { 20 | path = "/home/vault/.vault-token" 21 | } 22 | } 23 | } 24 | 25 | template { 26 | destination = "/etc/secrets/config.yaml" 27 | contents = < ../ 7 | 8 | go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.7.0 9 | ) 10 | 11 | require ( 12 | github.com/ondat/trousseau v0.0.0-00010101000000-000000000000 13 | github.com/stretchr/testify v1.8.0 14 | google.golang.org/grpc v1.47.0 15 | k8s.io/apiserver v0.24.3 16 | k8s.io/klog/v2 v2.70.0 17 | ) 18 | 19 | require ( 20 | github.com/beorn7/perks v1.0.1 // indirect 21 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 22 | github.com/davecgh/go-spew v1.1.1 // indirect 23 | github.com/go-logr/logr v1.2.3 // indirect 24 | github.com/go-logr/stdr v1.2.2 // indirect 25 | github.com/go-logr/zapr v1.2.3 // indirect 26 | github.com/gogo/protobuf v1.3.2 // indirect 27 | github.com/golang/protobuf v1.5.2 // indirect 28 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 29 | github.com/pmezard/go-difflib v1.0.0 // indirect 30 | github.com/prometheus/client_golang v1.12.1 // indirect 31 | github.com/prometheus/client_model v0.2.0 // indirect 32 | github.com/prometheus/common v0.32.1 // indirect 33 | github.com/prometheus/procfs v0.7.3 // indirect 34 | go.opentelemetry.io/otel v1.7.0 // indirect 35 | go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0 // indirect 36 | go.opentelemetry.io/otel/internal/metric v0.21.0 // indirect 37 | go.opentelemetry.io/otel/metric v0.21.0 // indirect 38 | go.opentelemetry.io/otel/sdk v1.0.0-RC1 // indirect 39 | go.opentelemetry.io/otel/sdk/export/metric v0.21.0 // indirect 40 | go.opentelemetry.io/otel/sdk/metric v0.21.0 // indirect 41 | go.opentelemetry.io/otel/trace v1.7.0 // indirect 42 | go.uber.org/atomic v1.9.0 // indirect 43 | go.uber.org/multierr v1.6.0 // indirect 44 | go.uber.org/zap v1.21.0 // indirect 45 | golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect 46 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect 47 | golang.org/x/text v0.3.7 // indirect 48 | google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect 49 | google.golang.org/protobuf v1.28.0 // indirect 50 | gopkg.in/yaml.v2 v2.4.0 // indirect 51 | gopkg.in/yaml.v3 v3.0.1 // indirect 52 | ) 53 | -------------------------------------------------------------------------------- /trousseau/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "net/url" 9 | "os" 10 | "os/signal" 11 | "strconv" 12 | "strings" 13 | "syscall" 14 | "time" 15 | 16 | "github.com/ondat/trousseau/pkg/logger" 17 | "github.com/ondat/trousseau/pkg/metrics" 18 | "github.com/ondat/trousseau/pkg/utils" 19 | "github.com/ondat/trousseau/pkg/version" 20 | "github.com/ondat/trousseau/trousseau/pkg/health" 21 | "github.com/ondat/trousseau/trousseau/pkg/server" 22 | "google.golang.org/grpc" 23 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 24 | 25 | "k8s.io/klog/v2" 26 | ) 27 | 28 | const hostPortFormatBase = 10 29 | 30 | const ( 31 | maxAllowedProviders = 2 32 | defaultHealthzTimeout = 10 * time.Second 33 | defaultSocketTimeout = 20 * time.Second 34 | defaultHealthPort = 8787 35 | defaultMetricsPort = "8095" 36 | defaultDecryptPreference = "roundrobin" 37 | ) 38 | 39 | type arrayFlags []string 40 | 41 | func (i *arrayFlags) String() string { 42 | return strings.Join(*i, ",") 43 | } 44 | 45 | func (i *arrayFlags) Set(value string) error { 46 | *i = append(*i, value) 47 | return nil 48 | } 49 | 50 | func main() { 51 | var enabledProviders arrayFlags 52 | 53 | flag.Var(&enabledProviders, "enabled-providers", "list of enabled providers") 54 | socketLocation := flag.String("socket-location", "/opt/trousseau-kms", "location of provider sockets") 55 | socketTimeout := flag.Duration("socket-timeout", defaultSocketTimeout, "RPC timeout for provider socket") 56 | listenAddr := flag.String("listen-addr", "unix:///opt/trousseau-kms/trousseau.socket", "gRPC listen address") 57 | logEncoder := flag.String("zap-encoder", "console", "set log encoder [console, json]") 58 | healthzPort := flag.Int("healthz-port", defaultHealthPort, "port for health check") 59 | healthzPath := flag.String("healthz-path", "/healthz", "path for health check") 60 | healthzTimeout := flag.Duration("healthz-timeout", defaultHealthzTimeout, "RPC timeout for health check") 61 | metricsBackend := flag.String("metrics-backend", "prometheus", "Backend used for metrics") 62 | metricsAddress := flag.String("metrics-addr", defaultMetricsPort, "The address the metric endpoint binds to") 63 | decryptPreference := flag.String("decrypt-preference", defaultDecryptPreference, "The decrypt provider preference [roundrobin, fastest]") 64 | 65 | flag.Parse() 66 | 67 | err := logger.InitializeLogging(*logEncoder) 68 | if err != nil { 69 | klog.Errorln(err) 70 | os.Exit(1) 71 | } 72 | 73 | if len(enabledProviders) > maxAllowedProviders { 74 | klog.Errorln(fmt.Errorf("max allowed providers: %d", maxAllowedProviders)) 75 | os.Exit(1) 76 | } 77 | 78 | ctx := withShutdownSignal(context.Background()) 79 | 80 | // initialize metrics exporter 81 | go func() { 82 | //nolint:govet // We know err is a shadow 83 | err := metrics.Serve(*metricsBackend, *metricsAddress) 84 | if err != nil { 85 | klog.Errorln(err) 86 | os.Exit(1) 87 | } 88 | 89 | klog.Fatalln("metrics service has stopped gracefully") 90 | }() 91 | 92 | klog.InfoS("Starting VaultEncryptionServiceServer service", "version", version.BuildVersion, "buildDate", version.BuildDate) 93 | 94 | proto, addr, err := utils.ParseEndpoint(*listenAddr) 95 | if err != nil { 96 | klog.Errorln(err) 97 | os.Exit(1) 98 | } 99 | 100 | if err = os.Remove(addr); err != nil && !os.IsNotExist(err) { 101 | klog.ErrorS(err, "unable to delete socket file", "file", addr) 102 | } 103 | 104 | listener, err := net.Listen(proto, addr) 105 | if err != nil { 106 | klog.Errorln(err) 107 | os.Exit(1) 108 | } 109 | 110 | klog.InfoS("Listening for connections", "address", listener.Addr()) 111 | 112 | go func() { 113 | klog.Errorln(<-utils.WatchFile(addr)) 114 | os.Exit(1) 115 | }() 116 | 117 | kmsServer, err := server.New(*decryptPreference, *socketLocation, enabledProviders, *socketTimeout) 118 | if err != nil { 119 | klog.Errorln(err) 120 | os.Exit(1) 121 | } 122 | 123 | opts := []grpc.ServerOption{ 124 | grpc.UnaryInterceptor(utils.UnaryServerInterceptor), 125 | } 126 | 127 | s := grpc.NewServer(opts...) 128 | pb.RegisterKeyManagementServiceServer(s, kmsServer) 129 | 130 | go func() { 131 | if err := s.Serve(listener); err != nil { 132 | klog.Errorln(err) 133 | os.Exit(1) 134 | } 135 | 136 | klog.Fatalln("GRPC service has stopped gracefully") 137 | }() 138 | 139 | healthz := &health.Service{ 140 | Service: kmsServer, 141 | HealthCheckURL: &url.URL{ 142 | Host: net.JoinHostPort("", strconv.FormatUint(uint64(*healthzPort), hostPortFormatBase)), 143 | Path: *healthzPath, 144 | }, 145 | UnixSocketPath: listener.Addr().String(), 146 | Timeout: *healthzTimeout, 147 | } 148 | 149 | go func() { 150 | if err := healthz.Serve(); err != nil { 151 | klog.Errorln(err) 152 | os.Exit(1) 153 | } 154 | 155 | klog.Fatalln("healtz service has stopped gracefully") 156 | }() 157 | 158 | <-ctx.Done() 159 | // gracefully stop the grpc server 160 | klog.Info("Terminating the server") 161 | s.GracefulStop() 162 | klog.Flush() 163 | // using os.Exit skips running deferred functions 164 | os.Exit(0) 165 | } 166 | 167 | // withShutdownSignal returns a copy of the parent context that will close if 168 | // the process receives termination signals. 169 | func withShutdownSignal(ctx context.Context) context.Context { 170 | signalChan := make(chan os.Signal, 1) 171 | signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, os.Interrupt) 172 | 173 | nctx, cancel := context.WithCancel(ctx) 174 | 175 | go func() { 176 | <-signalChan 177 | klog.Info("Received shutdown signal") 178 | cancel() 179 | }() 180 | 181 | return nctx 182 | } 183 | -------------------------------------------------------------------------------- /trousseau/pkg/health/health.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/ondat/trousseau/pkg/logger" 12 | "github.com/ondat/trousseau/pkg/providers" 13 | "github.com/ondat/trousseau/pkg/utils" 14 | "github.com/ondat/trousseau/pkg/version" 15 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 16 | "k8s.io/klog/v2" 17 | ) 18 | 19 | const ( 20 | healthCheckPlainText = "healthcheck" 21 | ) 22 | 23 | // Service Healtz endpoint. 24 | type Service struct { 25 | Service providers.KeyManagementService 26 | HealthCheckURL *url.URL 27 | UnixSocketPath string 28 | Timeout time.Duration 29 | } 30 | 31 | // Serve creates the http handler for serving health requests. 32 | func (h *Service) Serve() error { 33 | klog.V(logger.Info3).Info("Initialize health check") 34 | 35 | serveMux := http.NewServeMux() 36 | serveMux.HandleFunc(h.HealthCheckURL.EscapedPath(), h.ServeHTTP) 37 | 38 | const timeout = time.Second * 5 39 | 40 | srv := &http.Server{ 41 | ReadTimeout: timeout, 42 | ReadHeaderTimeout: timeout, 43 | WriteTimeout: timeout, 44 | Addr: h.HealthCheckURL.Host, 45 | Handler: serveMux, 46 | } 47 | 48 | if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { 49 | klog.Error(err, "Failed to start health check") 50 | return fmt.Errorf("failed to start health check server: %w", err) 51 | } 52 | 53 | return nil 54 | } 55 | 56 | // ServeHTTP is handling healthz requests. 57 | func (h *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { 58 | klog.V(logger.Debug1).Info("Started health check...") 59 | 60 | ctx, cancel := context.WithTimeout(context.Background(), h.Timeout) 61 | defer cancel() 62 | 63 | conn, err := utils.DialUnixSocket(h.UnixSocketPath) 64 | if err != nil { 65 | klog.Error(err, "Failed to call unix socket") 66 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 67 | 68 | return 69 | } 70 | defer conn.Close() 71 | 72 | kmsClient := pb.NewKeyManagementServiceClient(conn) 73 | 74 | if err = h.checkRPC(ctx, kmsClient); err != nil { 75 | klog.Error(err, "Failed to check RPC") 76 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 77 | 78 | return 79 | } 80 | 81 | enc, err := h.Service.Encrypt(ctx, &pb.EncryptRequest{Plain: []byte(healthCheckPlainText)}) 82 | if err != nil { 83 | klog.Error(err, "Failed to encrypt", "data", healthCheckPlainText) 84 | http.Error(w, err.Error(), http.StatusInternalServerError) 85 | 86 | return 87 | } 88 | 89 | dec, err := h.Service.Decrypt(ctx, &pb.DecryptRequest{Cipher: enc.Cipher}) 90 | if err != nil { 91 | klog.Error(err, "Failed to decrypt encrypted data") 92 | http.Error(w, err.Error(), http.StatusInternalServerError) 93 | 94 | return 95 | } else if string(dec.Plain) != healthCheckPlainText { 96 | klog.ErrorS(errors.New("failed to properly decrypt encrypted data"), "Encryption failed", "original", healthCheckPlainText, "decrypted", string(dec.Plain)) 97 | http.Error(w, "plain text mismatch after decryption", http.StatusInternalServerError) 98 | 99 | return 100 | } 101 | 102 | w.WriteHeader(http.StatusOK) 103 | 104 | if _, err := w.Write([]byte("ok")); err != nil { 105 | http.Error(w, err.Error(), http.StatusNotFound) 106 | 107 | return 108 | } 109 | 110 | klog.V(logger.Debug1).Info("Completed health check") 111 | } 112 | 113 | // checkRPC initiates a grpc request to validate the socket is responding 114 | // sends a KMS VersionRequest and checks if the VersionResponse is valid. 115 | func (h *Service) checkRPC(ctx context.Context, client pb.KeyManagementServiceClient) error { 116 | v, err := client.Version(ctx, &pb.VersionRequest{}) 117 | if err != nil { 118 | return fmt.Errorf("unable to get version: %w", err) 119 | } 120 | 121 | if v.Version != version.APIVersion || v.RuntimeName != version.Runtime || v.RuntimeVersion != version.BuildVersion { 122 | return errors.New("failed to get correct version response") 123 | } 124 | 125 | return nil 126 | } 127 | -------------------------------------------------------------------------------- /trousseau/pkg/health/health_test.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net" 7 | "net/http" 8 | "net/http/httptest" 9 | "net/url" 10 | "os" 11 | "path" 12 | "sync" 13 | "testing" 14 | "time" 15 | 16 | "github.com/ondat/trousseau/pkg/providers" 17 | "github.com/ondat/trousseau/pkg/version" 18 | "github.com/stretchr/testify/assert" 19 | "github.com/stretchr/testify/require" 20 | "google.golang.org/grpc" 21 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 22 | ) 23 | 24 | func TestHealthz(t *testing.T) { 25 | tmpDir, err := os.MkdirTemp("", "") 26 | require.Nil(t, err) 27 | 28 | defer os.Remove(tmpDir) 29 | 30 | socket := path.Join(tmpDir, "healthz.socket") 31 | 32 | wg := sync.WaitGroup{} 33 | wg.Add(1) 34 | 35 | go func() { 36 | s := grpc.NewServer() 37 | pb.RegisterKeyManagementServiceServer(s, &providers.KeyManagementServiceServer{ 38 | Client: &mockClient{}, 39 | }) 40 | 41 | listener, listenErr := net.Listen("unix", socket) 42 | require.Nil(t, listenErr) 43 | 44 | wg.Done() 45 | 46 | err = s.Serve(listener) 47 | require.Nil(t, err) 48 | }() 49 | 50 | wg.Wait() 51 | 52 | req, err := http.NewRequest("GET", "", http.NoBody) 53 | require.Nil(t, err) 54 | 55 | writer := httptest.NewRecorder() 56 | 57 | service := Service{ 58 | Service: &mockProvider{}, 59 | HealthCheckURL: &url.URL{ 60 | Host: net.JoinHostPort("", "8787"), 61 | Path: "/healthz", 62 | }, 63 | UnixSocketPath: socket, 64 | Timeout: time.Second, 65 | } 66 | service.ServeHTTP(writer, req) 67 | 68 | content, err := io.ReadAll(writer.Body) 69 | 70 | require.Nil(t, err) 71 | assert.Equal(t, http.StatusOK, writer.Code) 72 | assert.Equal(t, "ok", string(content)) 73 | } 74 | 75 | type mockProvider struct{} 76 | 77 | func (m *mockProvider) Decrypt(_ context.Context, dr *pb.DecryptRequest) (*pb.DecryptResponse, error) { 78 | return &pb.DecryptResponse{ 79 | Plain: dr.Cipher, 80 | }, nil 81 | } 82 | 83 | func (m *mockProvider) Encrypt(_ context.Context, er *pb.EncryptRequest) (*pb.EncryptResponse, error) { 84 | return &pb.EncryptResponse{ 85 | Cipher: er.Plain, 86 | }, nil 87 | } 88 | 89 | func (m *mockProvider) Version(context.Context, *pb.VersionRequest) (*pb.VersionResponse, error) { 90 | return &pb.VersionResponse{ 91 | Version: version.APIVersion, 92 | RuntimeName: version.Runtime, 93 | RuntimeVersion: version.BuildVersion, 94 | }, nil 95 | } 96 | 97 | type mockClient struct{} 98 | 99 | func (s *mockClient) Encrypt(data []byte) ([]byte, error) { 100 | return data, nil 101 | } 102 | 103 | func (s *mockClient) Decrypt(data []byte) ([]byte, error) { 104 | return data, nil 105 | } 106 | 107 | func (s *mockClient) Version() *pb.VersionResponse { 108 | return &pb.VersionResponse{ 109 | Version: version.APIVersion, 110 | RuntimeName: version.Runtime, 111 | RuntimeVersion: version.BuildVersion, 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /trousseau/pkg/server/grpc.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "time" 11 | 12 | "github.com/ondat/trousseau/pkg/logger" 13 | "github.com/ondat/trousseau/pkg/metrics" 14 | "github.com/ondat/trousseau/pkg/providers" 15 | "github.com/ondat/trousseau/pkg/utils" 16 | "github.com/ondat/trousseau/pkg/version" 17 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 18 | "k8s.io/klog/v2" 19 | ) 20 | 21 | const separator = ":-:" 22 | 23 | type registeredProviders map[string]func(*pb.EncryptRequest, *pb.DecryptRequest) ([]byte, error) 24 | 25 | type providersService struct { 26 | providers registeredProviders 27 | sortProviders func() []string 28 | fastestMetricsChan chan<- Metric 29 | metricsReporter metrics.StatsReporter 30 | } 31 | 32 | // New creates an instance of the KMS Service Server. 33 | func New(decryptPreference, socketLocation string, enabledProviders []string, timeout time.Duration) (providers.KeyManagementService, error) { 34 | klog.V(logger.Debug1).Info("Initialize new providers service") 35 | 36 | service := providersService{ 37 | metricsReporter: metrics.NewStatsReporter(), 38 | } 39 | 40 | switch decryptPreference { 41 | case "roundrobin": 42 | service.sortProviders = NewRoundrobin(enabledProviders).Next 43 | case "fastest": 44 | fastest := NewFastest(enabledProviders) 45 | service.sortProviders = fastest.Fastest 46 | service.fastestMetricsChan = fastest.C() 47 | default: 48 | return nil, fmt.Errorf("selected decryption preference isn't supported: %s", decryptPreference) 49 | } 50 | 51 | registered := registeredProviders{} 52 | 53 | for _, provider := range enabledProviders { 54 | provider := provider 55 | 56 | socket := filepath.Clean(filepath.Join(socketLocation, provider, fmt.Sprintf("%s.socket", provider))) 57 | if _, err := os.Stat(socket); err != nil { 58 | klog.ErrorS(err, "Unable to find socket", "name", provider, "socket", socket, "error", err.Error()) 59 | return nil, fmt.Errorf("unable to find socket at %s: %w", socket, err) 60 | } 61 | 62 | klog.InfoS("Registered provider", "provider", provider) 63 | 64 | registered[provider] = func(encReq *pb.EncryptRequest, decReq *pb.DecryptRequest) ([]byte, error) { 65 | klog.V(logger.Debug1).InfoS("Calling provider", "name", provider, "socket", socket, "encryption", encReq != nil, "decryption", decReq != nil) 66 | 67 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 68 | defer cancel() 69 | 70 | conn, err := utils.DialUnixSocket(socket) 71 | if err != nil { 72 | klog.ErrorS(err, "Failed to call unix socket", "name", provider, "socket", socket, "encryption", encReq != nil, "decryption", decReq != nil) 73 | return nil, err 74 | } 75 | defer conn.Close() 76 | 77 | kmsClient := pb.NewKeyManagementServiceClient(conn) 78 | 79 | switch { 80 | case encReq != nil: 81 | res, err := kmsClient.Encrypt(ctx, encReq) 82 | if err != nil { 83 | klog.InfoS("Unable to encrypt data", "name", provider, "socket", socket) 84 | return nil, err 85 | } 86 | 87 | return res.Cipher, err 88 | case decReq != nil: 89 | res, err := kmsClient.Decrypt(ctx, decReq) 90 | if err != nil { 91 | klog.InfoS("Unable to decrypt data", "name", provider, "socket", socket) 92 | return nil, err 93 | } 94 | 95 | return res.Plain, err 96 | default: 97 | return nil, nil 98 | } 99 | } 100 | } 101 | 102 | service.providers = registered 103 | 104 | return &service, nil 105 | } 106 | 107 | // Encrypt encryption requet handler. 108 | func (k *providersService) Encrypt(ctx context.Context, data *pb.EncryptRequest) (*pb.EncryptResponse, error) { 109 | klog.V(logger.Debug1).Info("Encrypt has been called...") 110 | 111 | encrypted := map[string][]byte{} 112 | 113 | for name := range k.providers { 114 | klog.V(logger.Info3).InfoS("Encrypting...", "name", name) 115 | 116 | start := time.Now() 117 | 118 | provider := k.providers[name] 119 | 120 | r, err := provider(&pb.EncryptRequest{ 121 | Version: data.Version, 122 | Plain: data.Plain, 123 | }, nil) 124 | if err != nil { 125 | klog.InfoS("Failed to encrypt", "name", name, "error", err.Error()) 126 | k.metricsReporter.ReportRequest(ctx, name, metrics.EncryptOperationTypeValue, metrics.ErrorStatusTypeValue, time.Since(start).Seconds(), err.Error()) 127 | 128 | return nil, fmt.Errorf("failed to encrypt %s: %w", name, err) 129 | } 130 | 131 | k.metricsReporter.ReportRequest(ctx, name, metrics.EncryptOperationTypeValue, metrics.SuccessStatusTypeValue, time.Since(start).Seconds()) 132 | 133 | encrypted[name] = r 134 | } 135 | 136 | final := strings.Builder{} 137 | for name, enc := range encrypted { 138 | if _, err := final.WriteString(fmt.Sprintf("%s%s%s\n", name, separator, string(enc))); err != nil { 139 | klog.InfoS("Failed to append result", "name", name, "error", err.Error()) 140 | 141 | return nil, fmt.Errorf("failed to append result %s: %w", name, err) 142 | } 143 | } 144 | 145 | klog.V(logger.Debug1).Info("Encrypt request complete") 146 | 147 | return &pb.EncryptResponse{Cipher: []byte(final.String())}, nil 148 | } 149 | 150 | // Decrypt decryption requet handler. 151 | func (k *providersService) Decrypt(ctx context.Context, data *pb.DecryptRequest) (*pb.DecryptResponse, error) { 152 | klog.V(logger.Debug1).Info("Decrypt has been called...") 153 | 154 | const nParts = 2 155 | 156 | secrets := map[string]string{} 157 | 158 | for _, line := range strings.Split(string(data.Cipher), "\n") { 159 | if line == "" { 160 | continue 161 | } 162 | 163 | parts := strings.Split(line, separator) 164 | if len(parts) != nParts { 165 | klog.InfoS("Failed to find proper decryption") 166 | klog.V(logger.Debug2).InfoS("Invalid encrypted input", "line", utils.SecretToLog(line)) 167 | 168 | continue 169 | } 170 | 171 | secrets[parts[0]] = parts[1] 172 | } 173 | 174 | for _, name := range k.sortProviders() { 175 | secret, ok := secrets[name] 176 | if !ok { 177 | klog.InfoS("Failed to find encrypted for provider", "name", name) 178 | 179 | continue 180 | } 181 | 182 | klog.V(logger.Info3).InfoS("Decrypting...", "name", name) 183 | 184 | start := time.Now() 185 | 186 | provider, ok := k.providers[name] 187 | if !ok { 188 | klog.InfoS("Failed to find provider", "name", name) 189 | 190 | continue 191 | } 192 | 193 | response, err := provider(nil, &pb.DecryptRequest{ 194 | Version: data.Version, 195 | Cipher: []byte(secret), 196 | }) 197 | if err != nil { 198 | klog.InfoS("Failed to decrypt", "name", name, "error", err.Error()) 199 | k.metricsReporter.ReportRequest(ctx, name, metrics.EncryptOperationTypeValue, metrics.ErrorStatusTypeValue, time.Since(start).Seconds(), err.Error()) 200 | 201 | if k.fastestMetricsChan != nil { 202 | k.fastestMetricsChan <- Metric{ 203 | Provider: name, 204 | ReponseTime: time.Minute, 205 | } 206 | } 207 | 208 | continue 209 | } 210 | 211 | k.metricsReporter.ReportRequest(ctx, name, metrics.EncryptOperationTypeValue, metrics.SuccessStatusTypeValue, time.Since(start).Seconds()) 212 | 213 | if k.fastestMetricsChan != nil { 214 | k.fastestMetricsChan <- Metric{ 215 | Provider: name, 216 | ReponseTime: time.Since(start), 217 | } 218 | } 219 | 220 | klog.V(logger.Debug1).Info("Decrypt request complete") 221 | 222 | return &pb.DecryptResponse{Plain: response}, nil 223 | } 224 | 225 | klog.InfoS("Failed to decrypt with all providers") 226 | 227 | return nil, errors.New("failed to decrypt with all providers") 228 | } 229 | 230 | // Version version of gRPS server. 231 | func (k *providersService) Version(context.Context, *pb.VersionRequest) (*pb.VersionResponse, error) { 232 | return &pb.VersionResponse{Version: version.APIVersion, RuntimeName: version.Runtime, RuntimeVersion: version.BuildVersion}, nil 233 | } 234 | -------------------------------------------------------------------------------- /trousseau/pkg/server/grpc_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "io/fs" 6 | "net" 7 | "os" 8 | "path" 9 | "sync" 10 | "testing" 11 | "time" 12 | 13 | "github.com/ondat/trousseau/pkg/providers" 14 | "github.com/ondat/trousseau/pkg/version" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | "google.golang.org/grpc" 18 | pb "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1" 19 | ) 20 | 21 | func TestGRPCService(t *testing.T) { 22 | tmpDir, err := os.MkdirTemp("", "") 23 | require.Nil(t, err) 24 | 25 | defer os.Remove(tmpDir) 26 | 27 | socketDir := path.Join(tmpDir, "debug") 28 | err = os.Mkdir(socketDir, fs.ModePerm) 29 | require.Nil(t, err) 30 | 31 | wg := sync.WaitGroup{} 32 | wg.Add(1) 33 | 34 | go func() { 35 | s := grpc.NewServer() 36 | pb.RegisterKeyManagementServiceServer(s, &providers.KeyManagementServiceServer{ 37 | Client: &mockClient{}, 38 | }) 39 | 40 | listener, listenErr := net.Listen("unix", path.Join(socketDir, "debug.socket")) 41 | require.Nil(t, listenErr) 42 | 43 | wg.Done() 44 | 45 | err = s.Serve(listener) 46 | require.Nil(t, err) 47 | }() 48 | 49 | wg.Wait() 50 | 51 | service, err := New("roundrobin", tmpDir, []string{"debug"}, time.Second) 52 | require.Nil(t, err) 53 | 54 | decrypted, err := service.Encrypt(context.Background(), &pb.EncryptRequest{ 55 | Plain: []byte("foobar"), 56 | }) 57 | require.Nil(t, err) 58 | 59 | assert.Equal(t, []byte("debug:-:foobar\n"), decrypted.Cipher, "Invalid encryption") 60 | 61 | encrypted, err := service.Decrypt(context.Background(), &pb.DecryptRequest{ 62 | Cipher: decrypted.Cipher, 63 | }) 64 | require.Nil(t, err) 65 | 66 | assert.Equal(t, []byte("foobar"), encrypted.Plain, "Invalid decryption") 67 | } 68 | 69 | type mockClient struct{} 70 | 71 | func (s *mockClient) Encrypt(data []byte) ([]byte, error) { 72 | return data, nil 73 | } 74 | 75 | func (s *mockClient) Decrypt(data []byte) ([]byte, error) { 76 | return data, nil 77 | } 78 | 79 | func (s *mockClient) Version() *pb.VersionResponse { 80 | return &pb.VersionResponse{ 81 | Version: version.APIVersion, 82 | RuntimeName: version.Runtime, 83 | RuntimeVersion: version.BuildVersion, 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /trousseau/pkg/server/sort.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "sort" 5 | "time" 6 | 7 | "github.com/ondat/trousseau/pkg/logger" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | const ( 12 | fastestMetricsChanBufferSize = 10 13 | fastestAverageDiv = 2 14 | ) 15 | 16 | var fastestAverageMaxAge = 30 * time.Second 17 | 18 | // Roundrobin sorts providers based on Round-robin algorithm. 19 | type Roundrobin struct { 20 | lock chan bool 21 | names []string 22 | index int 23 | } 24 | 25 | // Next returns the next iteration. 26 | func (r *Roundrobin) Next() []string { 27 | if len(r.names) == 0 { 28 | return make([]string, 0) 29 | } 30 | 31 | r.lock <- true 32 | defer func() { 33 | <-r.lock 34 | }() 35 | 36 | r.index++ 37 | if r.index > len(r.names)-1 { 38 | r.index = 0 39 | } 40 | 41 | if r.index == 0 { 42 | klog.V(logger.Debug1).InfoS("Next Round-robin state", "providers", r.names) 43 | 44 | return r.names 45 | } 46 | 47 | //nolint:gocritic // This is a valid append 48 | sorted := append(r.names[r.index:], r.names[:r.index]...) 49 | 50 | klog.V(logger.Debug1).InfoS("Next Round-robin state", "providers", sorted) 51 | 52 | return sorted 53 | } 54 | 55 | // NewRoundrobin creates a new Roundrobin selector. 56 | func NewRoundrobin(providers []string) *Roundrobin { 57 | names := append(make([]string, 0, len(providers)), providers...) 58 | 59 | sort.Strings(names) 60 | 61 | return &Roundrobin{ 62 | lock: make(chan bool, 1), 63 | names: names, 64 | index: -1, 65 | } 66 | } 67 | 68 | type avgMetrics struct { 69 | provider string 70 | lastUpdate time.Time 71 | reponseTime time.Duration 72 | } 73 | 74 | // Metric contains one measurement. 75 | type Metric struct { 76 | Provider string 77 | ReponseTime time.Duration 78 | } 79 | 80 | // Fastest sorts providers by response time. 81 | type Fastest struct { 82 | lock chan bool 83 | metricsChan chan Metric 84 | providers map[string]*avgMetrics 85 | } 86 | 87 | // Fastest returns providers sorted by response time. 88 | func (f *Fastest) Fastest() []string { 89 | responseTimes := []avgMetrics{} 90 | 91 | f.lock <- true 92 | 93 | for provider := range f.providers { 94 | if f.providers[provider] == nil || f.providers[provider].lastUpdate.Add(fastestAverageMaxAge).Before(time.Now()) { 95 | f.providers[provider] = &avgMetrics{ 96 | provider: provider, 97 | } 98 | } 99 | 100 | responseTimes = append(responseTimes, *f.providers[provider]) 101 | } 102 | 103 | <-f.lock 104 | 105 | sort.SliceStable(responseTimes, func(i, j int) bool { 106 | return responseTimes[i].reponseTime.Milliseconds() < responseTimes[j].reponseTime.Milliseconds() 107 | }) 108 | 109 | sorted := []string{} 110 | for _, rt := range responseTimes { 111 | sorted = append(sorted, rt.provider) 112 | } 113 | 114 | klog.V(logger.Debug1).InfoS("Providers names by response time", "providers", sorted) 115 | 116 | return sorted 117 | } 118 | 119 | // C returns channel to send metrics. 120 | func (f *Fastest) C() chan<- Metric { 121 | return f.metricsChan 122 | } 123 | 124 | func (f *Fastest) consumeMetrics() { 125 | klog.Info("Start watching metrics channel...") 126 | 127 | for { 128 | metrics, ok := <-f.metricsChan 129 | if !ok { 130 | klog.Info("Watching metrics channel closed") 131 | 132 | return 133 | } 134 | 135 | now := time.Now() 136 | 137 | f.lock <- true 138 | 139 | existing, ok := f.providers[metrics.Provider] 140 | if !ok || existing == nil || existing.lastUpdate.Add(fastestAverageMaxAge).Before(now) { 141 | existing = &avgMetrics{ 142 | provider: metrics.Provider, 143 | } 144 | } 145 | 146 | existing.lastUpdate = now 147 | 148 | if existing.reponseTime == 0 { 149 | existing.reponseTime = metrics.ReponseTime 150 | } else { 151 | existing.reponseTime = (existing.reponseTime + metrics.ReponseTime) / fastestAverageDiv 152 | } 153 | 154 | f.providers[metrics.Provider] = existing 155 | 156 | <-f.lock 157 | 158 | klog.V(logger.Debug1).InfoS("Current average response time", "provider", metrics.Provider, "ms", existing.reponseTime.Milliseconds()) 159 | } 160 | } 161 | 162 | // NewFastest creates a new Fastest selector. 163 | func NewFastest(providers []string) *Fastest { 164 | fastest := Fastest{ 165 | lock: make(chan bool, 1), 166 | metricsChan: make(chan Metric, fastestMetricsChanBufferSize), 167 | providers: map[string]*avgMetrics{}, 168 | } 169 | 170 | for _, provider := range providers { 171 | fastest.providers[provider] = nil 172 | } 173 | 174 | go fastest.consumeMetrics() 175 | 176 | return &fastest 177 | } 178 | -------------------------------------------------------------------------------- /trousseau/pkg/server/sort_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestRoundrobin(t *testing.T) { 11 | t.Parallel() 12 | 13 | expectedResults := [][]string{ 14 | {"a", "b", "c", "d"}, 15 | {"b", "c", "d", "a"}, 16 | {"c", "d", "a", "b"}, 17 | {"d", "a", "b", "c"}, 18 | } 19 | 20 | rr := NewRoundrobin([]string{"a", "b", "c", "d"}) 21 | 22 | for i := 0; i < 3; i++ { 23 | for _, expected := range expectedResults { 24 | assert.Equal(t, expected, rr.Next(), "Roundrobin failed") 25 | } 26 | } 27 | } 28 | 29 | func TestFastest(t *testing.T) { 30 | testcases := map[string]struct { 31 | producer func(chan<- Metric) 32 | expeted []string 33 | }{ 34 | "Reverse": { 35 | producer: func(m chan<- Metric) { 36 | m <- Metric{ 37 | Provider: "a", 38 | ReponseTime: 3 * time.Second, 39 | } 40 | m <- Metric{ 41 | Provider: "b", 42 | ReponseTime: 2 * time.Second, 43 | } 44 | m <- Metric{ 45 | Provider: "c", 46 | ReponseTime: 1 * time.Second, 47 | } 48 | 49 | time.Sleep(time.Millisecond) 50 | }, 51 | expeted: []string{"c", "b", "a"}, 52 | }, 53 | "Peek in average": { 54 | producer: func(m chan<- Metric) { 55 | m <- Metric{ 56 | Provider: "a", 57 | ReponseTime: 2 * time.Second, 58 | } 59 | m <- Metric{ 60 | Provider: "a", 61 | ReponseTime: 1 * time.Second, 62 | } 63 | m <- Metric{ 64 | Provider: "b", 65 | ReponseTime: 2 * time.Second, 66 | } 67 | m <- Metric{ 68 | Provider: "b", 69 | ReponseTime: 2 * time.Minute, 70 | } 71 | m <- Metric{ 72 | Provider: "c", 73 | ReponseTime: 1 * time.Second, 74 | } 75 | 76 | time.Sleep(time.Millisecond) 77 | }, 78 | expeted: []string{"c", "a", "b"}, 79 | }, 80 | "Reset after max age": { 81 | producer: func(m chan<- Metric) { 82 | m <- Metric{ 83 | Provider: "b", 84 | ReponseTime: time.Minute, 85 | } 86 | 87 | time.Sleep(fastestAverageMaxAge + 1) 88 | 89 | m <- Metric{ 90 | Provider: "a", 91 | ReponseTime: 3 * time.Second, 92 | } 93 | m <- Metric{ 94 | Provider: "c", 95 | ReponseTime: 1 * time.Second, 96 | } 97 | 98 | time.Sleep(time.Millisecond) 99 | }, 100 | expeted: []string{"b", "c", "a"}, 101 | }, 102 | } 103 | 104 | for name, tc := range testcases { 105 | tc := tc 106 | 107 | t.Run(name, func(t *testing.T) { 108 | fastestAverageMaxAge = time.Second 109 | 110 | t.Parallel() 111 | 112 | for i := 0; i < 3; i++ { 113 | fastest := NewFastest([]string{"a", "b", "c"}) 114 | 115 | tc.producer(fastest.C()) 116 | 117 | assert.Equal(t, tc.expeted, fastest.Fastest(), "Fastest failed") 118 | } 119 | }) 120 | } 121 | } 122 | --------------------------------------------------------------------------------