├── .gitignore ├── CHANGELOG ├── LICENSE ├── Makefile ├── README.md ├── briefcase ├── aws_sts.go ├── aws_sts_test.go ├── briefcase.go ├── briefcase_test.go ├── json_secrets.go ├── secrets_cache.go ├── ssh.go ├── ssh_test.go └── template.go ├── config ├── config.go ├── config_test.go └── templates.go ├── docs ├── BUILDING.md ├── CONFIGURATION.md ├── EC2.md ├── KUBERNETES.md └── examples │ ├── absolute-ping.yml │ ├── cleanup.txt │ ├── force-refreshing-credentials.md │ ├── refreshing-leases.txt │ ├── relative-ping.yml │ ├── simple.yml │ └── template-ping │ ├── example.tpl │ └── relative-example.tpl ├── e2e ├── e2e_fixtures.go └── e2e_test.go ├── go.mod ├── go.sum ├── main.go ├── metrics └── metrics.go ├── perform.go ├── secrets ├── json_secrets.go ├── sts.go ├── template.go └── token.go ├── syncer ├── compare.go └── sync.go ├── util ├── clock │ └── clock.go ├── constants.go ├── flags.go ├── locking.go ├── modes.go ├── modes_test.go ├── paths.go ├── touch.go └── wrapped_token.go ├── vaultclient ├── auth.go ├── auth_aws_ami.go ├── auth_aws_iam.go ├── auth_kubernetes.go ├── creds_aws_sts.go ├── creds_sshcert.go ├── mocks │ └── vaultclient.go └── vaultclient.go └── vaulttoken ├── vault_token.go └── vault_token_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | tmp/ 3 | .idea 4 | target/ 5 | vault-ctrl-tool 6 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | v1.3.0: 22-Nov-2021 2 | * Errors during sync loop while running sidecar mode will no longer terminate vault-ctrl-tool. 3 | * Sidecar mode now can run a Prometheus metrics endpoint which emits metrics about sidecar syncs. 4 | Prometheus can be toggled with "--enable-prometheus-metrics" and have its port overridden by "--prometheus-port". 5 | * Added better documentation and some refactoring and cleanup of internal libraries. 6 | * Vault client HTTP timeout and maxRetries are now configurable using "--vault-client-timeout" and "--vault-client-retries" flags. 7 | Note: These now default to 30s and 2, respectively. Compared to previous version of vault-ctrl-tool which where 60s, 2. 8 | 9 | v1.2.0: 26-May-2021 10 | * Added --force-refresh-ttl which temporary credentials will optionally be renewed before their actual expiry. 11 | * Added --sts-ttl flag which lets you specify token ttl for aws tokens 12 | generated using sts. 13 | * Added buildVersion value when running version flag (vault-ctrl-tool --version) 14 | * Embedded version now uses semver that will correspond with tagged versions. 15 | 16 | 10-Nov-2020 17 | * secrets that only output raw fields can now have a lifetime of "version". The output of these fields 18 | will be updated when the version # of the secret changes 19 | * output of raw fields support base64 decoding with "encoding: base64" per-field 20 | * secrets can be pinned to a specific version - this is generally inadvisable, but can be done if needed 21 | for testing with "pinnedVersion:" 22 | * secrets with lifetime of "version" can specify a "touchfile" which is touched when any fields are rewritten (for 23 | external services to trigger refreshes). Touchfiles are only touched after fields are rewritten. 24 | 25 | 13-Oct-2020 26 | * vault-token ConfigMap supports "renewable" which is used to tell v-c-t to not renew the vault token in the ConfigMap 27 | * TOKEN_RENEWABLE environment variable can be used to disable renewing a passed in VAULT_TOKEN 28 | * --token-renewable can be set to "false" to disable renewing the token passed in on the command line 29 | * NOTE: these mechanisms only disable their respective token. For example TOKEN_RENEWABLE=false has no effect on ConfigMap 30 | or tokens passed in as CLI args. 31 | 32 | 23-Sep-2020 33 | * Started CHANGELOG 34 | * Added provisional file locking around sync runs to prevent concurrent modification 35 | * --init in Kubernetes will now function as a --sidecar --oneshot instead of re-authenticating (this is to 36 | work around Kubernetes issues where init containers will re-run) 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018-2020 Hootsuite Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CURRENTOS := $(shell go env GOOS) 2 | CURRENTARCH := $(shell go env GOARCH) 3 | COMMIT := $(shell git rev-parse --short HEAD) 4 | VERSION := v1.3.6 5 | LDFLAGS="-X main.buildVersion=$(VERSION) -X main.commitVersion=$(COMMIT)" 6 | 7 | .DEFAULT_GOAL := build 8 | 9 | help: ## List targets & descriptions 10 | @grep --no-filename -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}' 11 | 12 | build: clean test darwin-binary linux-binary copy-binary ## Build all the binaries 13 | 14 | clean: ## Delete the destination directory. 15 | rm -rf ./bin 16 | 17 | test: mocks ## Run unit tests 18 | go test -v ./... 19 | 20 | darwin-binary: mocks ## Build a macOS binary 21 | GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags $(LDFLAGS) -o bin/vault-ctrl-tool.darwin.amd64 . 22 | GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags $(LDFLAGS) -o bin/vault-ctrl-tool.darwin.arm64 . 23 | 24 | linux-binary: mocks ## Build a Linux (amd64) binary 25 | GOOS=linux GOARCH=amd64 go build -trimpath -ldflags $(LDFLAGS) -o bin/vault-ctrl-tool.linux.amd64 . 26 | GOOS=linux GOARCH=arm64 go build -trimpath -ldflags $(LDFLAGS) -o bin/vault-ctrl-tool.linux.arm64 . 27 | 28 | # Useful when doing development 29 | copy-binary: 30 | cp bin/vault-ctrl-tool.$(CURRENTOS).$(CURRENTARCH) vault-ctrl-tool 31 | 32 | deps: ## Ensure dependencies are present and prune orphaned 33 | go mod download 34 | go mod tidy 35 | 36 | vaultclient/mocks/vaultclient.go: vaultclient/vaultclient.go 37 | mockgen -source=$< -destination=$@ 38 | 39 | mocks: vaultclient/mocks/vaultclient.go 40 | 41 | .PHONY: help mocks deps copy-binary linux-binary darwin-binary test clean build 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Go Report Card](https://goreportcard.com/badge/github.com/hootsuite/vault-ctrl-tool)](https://goreportcard.com/report/github.com/hootsuite/vault-ctrl-tool) 2 | 3 | # Welcome 4 | 5 | Hi! Thanks for taking a look at `vault-ctrl-tool`. This is a little tool that manages, authentication, 6 | applying secrets, and refreshing leases for services. It knows how to authenticate using Kubernetes (ServiceAccounts), 7 | EC2 (registered AMIs and IAM roles), as well as existing Vault tokens for other integration purposes. 8 | 9 | ## Version 10 | 11 | This is version 2 of the Vault Control Tool -- the previous version is on a `v1` branch. 12 | 13 | It is a refactoring, replacing the logging library, and 14 | simplifying the main code logic. It also now supports secrets whose lifetime is scoped to the token being used, 15 | and has a defined failure mode. Lastly, it does away with the "file scrubber". 16 | 17 | Its configuration file is backwards compatible, as are all the command line arguments. 18 | 19 | ### Upgrading 20 | 21 | Confusing, this version of Vault Control Tool supports `version: 3` in your vault-config files. To upgrade your 22 | configuration file, you will need to add a correct `lifetime` to your `secrets` and `templates`. If the secrets 23 | you're fetching from Vault are token-scoped (ie, credentials created dynamically, such as from a database backend), 24 | then set `lifetime: token`. Otherwise if your secrets are "static", then specify `lifetime: static`. Templates follow 25 | a similar pattern. If your template is using token-scoped secrets, specify `lifetime: token`, otherwise `lifetime: static`. 26 | 27 | ## Failure Mode 28 | 29 | Vault Control Tool will now re-authenticate if the token it is using ceases to work (hits a tuned backend limit, 30 | cannot reach the server to renew it for too long, etc). The tool now understands that some secrets will be invalid with the 31 | new token and will fetch new values and rewrite files as needed - these values are called said to have a "token-scoped 32 | lifetime". Note that files with "static" lifetimes are never rewritten. 33 | 34 | Failure is most likely caused by a Vault outage. For consumers, Vault outages have two forms: pre-revocation and 35 | post-revocation. Outages in pre-revocation mean token-lifetime secrets managed by Vault (database credentials, etc) 36 | will remain valid during the outage. Services can continue to happily use these until Vault restarts. This is 37 | the case if Vault is hard-down. 38 | 39 | Outages in post-revocation mean secrets have been removed by Vault, but Vault Control Tool is unable to perform operations 40 | to obtain fresh secrets (database management is unavailable, networking issues, configuration problems, etc). In these 41 | situations the service is unable to function. 42 | 43 | Lastly, in both situations, when a service has credentials that are _externally_ managed (SSH certificates, AWS STS 44 | sessions), they naturally become invalid which will also cause a service to be unable to function. 45 | 46 | Kubernetes services unable to dynamically re-read secrets are encouraged to run with a `restartPolicy: Always`, and delete 47 | token-scoped files after they are consumed. Services should also fail-fast if they're unable to continue to use the credentials (ie, 48 | authentication calls to remote services return errors). Kubernetes will restart the service. This will put the service 49 | into a crashloopbackoff until the Vault Control Tool is able to fetch fresh secrets. 50 | 51 | ## Other Documents 52 | 53 | If you're curious on how to build this in your environment, see [BUILDING.md](docs/BUILDING.md). 54 | 55 | If you're integrating with Kubernetes, see [KUBERNETES.md](docs/KUBERNETES.md). 56 | 57 | If you're integrating with EC2, see [EC2.md](docs/EC2.md). 58 | 59 | To understand how the configuration file works, see [CONFIGURATION.md](docs/CONFIGURATION.md). 60 | 61 | To play with a few examples, see [examples](docs/examples). 62 | 63 | ## Authentication 64 | 65 | | Backend | Supported | 66 | |---|---| 67 | | Kubernetes Service Account Tokens | Yes | 68 | | Passed in Vault tokens | Yes | 69 | | EC2 Metadata | Yes | 70 | | EC2 IAM | Yes | 71 | 72 | ## Secrets 73 | 74 | | Backend | Supported | 75 | |---|--- 76 | | KV | Yes | 77 | | KV v2 | Yes | 78 | | SSH (certificates) | Yes | 79 | | AWS | Yes | 80 | | Token-scoped Secrets | Yes | 81 | -------------------------------------------------------------------------------- /briefcase/aws_sts.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 8 | 9 | "github.com/hashicorp/vault/api" 10 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 11 | ) 12 | 13 | // STS credentials have a maximum lifetime enforced by AWS. The current expiry is kept in the briefcase 14 | // and checked to determine if it needs to be refreshed. Services using STS credentials are expected to handle 15 | // credentials expiring underneath them at any time. 16 | func (b *Briefcase) AWSCredentialExpiresBefore(awsConfig config.AWSType, expiresBefore time.Time) bool { 17 | entry, ok := b.AWSCredentialLeases[awsConfig.OutputPath] 18 | if !ok { 19 | return true 20 | } 21 | 22 | return !entry.Expiry.After(expiresBefore) 23 | } 24 | 25 | // AWSCredentialsShouldRefresh checks if a set of AWS credentials should be force refreshed according to it's refresh_expiry. 26 | func (b *Briefcase) AWSCredentialShouldRefreshBefore(awsConfig config.AWSType, refreshBefore time.Time) bool { 27 | entry, ok := b.AWSCredentialLeases[awsConfig.OutputPath] 28 | if !ok { 29 | return true 30 | } 31 | 32 | return entry.RefreshExpiry != nil && !entry.RefreshExpiry.IsZero() && refreshBefore.After(*entry.RefreshExpiry) 33 | } 34 | 35 | // EnrollAWSCredenntial adds or replaces a managed AWS credential to briefcase. If forceRefreshTTL is not zero then it will associate 36 | // refresh expirty time with the certificate. 37 | func (b *Briefcase) EnrollAWSCredential(ctx context.Context, awsCreds *api.Secret, awsConfig config.AWSType, forceRefreshTTL time.Duration) { 38 | expiry := clock.Now(ctx).Add(time.Second * time.Duration(awsCreds.LeaseDuration)) 39 | 40 | b.log.Debug().Int("forceRefreshTTL", int(forceRefreshTTL)).Msg("attempting to enroll aws_sts creds") 41 | 42 | var refreshExpiry *time.Time 43 | 44 | // we only add a refresh if a ttl value was set. 45 | if forceRefreshTTL > 0 { 46 | exp := clock.Now(ctx).Add(forceRefreshTTL) 47 | refreshExpiry = &exp 48 | if refreshExpiry.After(expiry) { 49 | b.log.Warn().Msgf("forceRefreshTTL is longer than the expiry of aws credentials") 50 | } 51 | b.log.Info().Time("expiry", expiry).Time("refreshTime", exp).Str("outputPath", awsConfig.OutputPath). 52 | Msg("enrolling AWS credential") 53 | } else { 54 | b.log.Info().Time("expiry", expiry).Str("outputPath", awsConfig.OutputPath). 55 | Msg("enrolling AWS credential") 56 | } 57 | 58 | b.AWSCredentialLeases[awsConfig.OutputPath] = leasedAWSCredential{ 59 | AWSCredential: awsConfig, 60 | Expiry: expiry, 61 | RefreshExpiry: refreshExpiry, 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /briefcase/aws_sts_test.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "testing" 7 | "time" 8 | 9 | "github.com/hashicorp/vault/api" 10 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 12 | "github.com/stretchr/testify/assert" 13 | testing2 "k8s.io/utils/clock/testing" 14 | ) 15 | 16 | const stsCreds = ` 17 | { 18 | "request_id": "117d1888-501c-7d83-16fd-9a4dab95268e", 19 | "lease_id": "aws/creds/user-readonly/0c5fUcbvbZ10OW9aa4BEV7UX", 20 | "lease_duration": 3600, 21 | "renewable": false, 22 | "data": { 23 | "access_key": "ASIARUFKCQPWQGZBN2D5", 24 | "secret_key": "rQFWMmrTgKPOC0EzhE095RZ1BTufIH59vx+Iwicc", 25 | "security_token": "FwoGZXIvYXdzEIT//////////wEaDGpHUM2Pj5PZ7cX9CCLeAWWKevneTafapQ3fEj2sVN/g0kamUHQmsBpIOSsI7Ew/D1XnUTa+ufswXhMDcG09LOqea+OSzCJj+w5GBJIVZ81vw/sAHKNhBtX3t+gcITxDttR3RFEhRpqLe2s/yI73/Ral6t644OfQv7Xs/G1A6WWMDT79B5GNVLFmI+uhNLEuDcBlPmC5aLyb6DZps/PvBAu23mUNZXKTGL2PA7gg/rIADdgcMn9P3dwPqOlQQ7Oc87J3HuU9plHE4y8jZPTwh555cTSJIrfk7h+z+4z90ZKcXz07J/zAkgDhcItPvyiw67b6BTItlUahvc5Bg3yZJe1IPufdUk3UKRn7tGaReOaEXCtuEVevkrGc9xTN/ELeSsaf" 26 | }, 27 | "warnings": null 28 | }` 29 | 30 | func mySTSCreds(t *testing.T) api.Secret { 31 | var secret api.Secret 32 | err := json.Unmarshal([]byte(stsCreds), &secret) 33 | assert.NoError(t, err, "could not deserialize example STS lease: %v", err) 34 | return secret 35 | } 36 | 37 | func TestAWSCredentialExpireCheck(t *testing.T) { 38 | assert := assert.New(t) 39 | awsCreds := mySTSCreds(t) 40 | awsConfig := config.AWSType{ 41 | VaultMountPoint: "aws", 42 | VaultRole: "user-readonly", 43 | Profile: "default", 44 | Region: "us-east-1", 45 | OutputPath: "/tmp", 46 | Mode: "0700", 47 | } 48 | 49 | testTime := time.Unix(1443332960, 0) 50 | 51 | ctx := clock.Set(context.Background(), testing2.NewFakeClock(testTime)) 52 | 53 | bc := NewBriefcase(nil) 54 | bc.EnrollAWSCredential(ctx, &awsCreds, awsConfig, 0) 55 | 56 | assert.False(bc.AWSCredentialExpiresBefore(awsConfig, testTime), "freshly enrolled STS token must not need refreshing") 57 | assert.False(bc.AWSCredentialExpiresBefore(awsConfig, testTime.Add(3599*time.Second)), "must not return true before the expiry of the STS token") 58 | assert.True(bc.AWSCredentialExpiresBefore(awsConfig, testTime.Add(time.Hour)), "must return true on the expiry of the token") 59 | assert.True(bc.AWSCredentialExpiresBefore(awsConfig, testTime.Add(3601*time.Second)), "must return true when creds are expired") 60 | 61 | assert.False(bc.AWSCredentialShouldRefreshBefore(awsConfig, testTime.Add(3601*time.Second))) 62 | 63 | bc.EnrollAWSCredential(ctx, &awsCreds, awsConfig, 3600*time.Second) 64 | assert.Equal(bc.AWSCredentialLeases[awsConfig.OutputPath].RefreshExpiry.Format(time.RFC3339Nano), testTime.Add(60*time.Minute).Format(time.RFC3339Nano)) 65 | 66 | assert.True(bc.AWSCredentialShouldRefreshBefore(awsConfig, testTime.Add(3601*time.Second)), "must return true when refreshExpiry is before next update") 67 | assert.False(bc.AWSCredentialExpiresBefore(awsConfig, testTime), "should still check that fresh STS token is not expired") 68 | } 69 | -------------------------------------------------------------------------------- /briefcase/briefcase.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "time" 10 | 11 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 12 | "github.com/hootsuite/vault-ctrl-tool/v2/metrics" 13 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 14 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 15 | "github.com/rs/zerolog" 16 | zlog "github.com/rs/zerolog/log" 17 | ) 18 | 19 | // Briefcase is a serialized file that contains all the information needed for the tool, running in sidecar mode, 20 | // to keep all the associated leases, secrets, etc refreshed. It also keeps a non-serialized copy of secrets that 21 | // are used to populate templates. 22 | type Briefcase struct { 23 | AuthTokenLease LeasedAuthToken `json:"auth"` 24 | SSHCertificates map[string]sshCert `json:"ssh,omitempty"` 25 | AWSCredentialLeases map[string]leasedAWSCredential `json:"aws,omitempty"` 26 | TokenScopedTemplates map[string]bool `json:"tokenscoped_templates,omitempty"` 27 | StaticTemplates map[string]bool `json:"static_templates,omitempty"` 28 | TokenScopedSecrets map[string]bool `json:"tokenscoped_secrets,omitempty"` 29 | StaticScopedSecrets map[string]bool `json:"static_secrets,omitempty"` 30 | VersionScopedSecrets map[string]int64 `json:"versioned_secrets,omitempty"` 31 | TokenScopedComposites map[string]bool `json:"tokenscoped_composites,omitempty"` 32 | StaticScopedComposites map[string]bool `json:"static_composites,omitempty"` 33 | 34 | // cache of secrets, not persisted 35 | secretCache map[util.SecretLifetime][]SimpleSecret 36 | 37 | log zerolog.Logger 38 | metrics *metrics.Metrics 39 | } 40 | 41 | // SimpleSecret is a field in a secret, but also contains some important information about the secret itself. 42 | type SimpleSecret struct { 43 | Key string 44 | Field string 45 | Value interface{} 46 | Version *int64 47 | CreatedTime *time.Time 48 | } 49 | 50 | type sshCert struct { 51 | Expiry time.Time `json:"expiry"` 52 | Cfg config.SSHCertificateType `json:"cfg"` 53 | RefreshExpiry *time.Time `json:"refresh_expiry,omitempty"` 54 | } 55 | 56 | type LeasedAuthToken struct { 57 | Accessor string `json:"accessor"` 58 | Renewable bool `json:"renewable"` 59 | Token string `json:"token"` 60 | ExpiresAt time.Time `json:"expiry"` 61 | NextRefresh time.Time `json:"next_refresh"` 62 | } 63 | 64 | type leasedAWSCredential struct { 65 | AWSCredential config.AWSType `json:"role"` 66 | Expiry time.Time `json:"expiry"` 67 | RefreshExpiry *time.Time `json:"refresh_expiry,omitempty"` 68 | } 69 | 70 | // NewBriefcase creates an empty briefcase. 71 | func NewBriefcase(mtrics *metrics.Metrics) *Briefcase { 72 | return &Briefcase{ 73 | AWSCredentialLeases: make(map[string]leasedAWSCredential), 74 | SSHCertificates: make(map[string]sshCert), 75 | TokenScopedTemplates: make(map[string]bool), 76 | StaticTemplates: make(map[string]bool), 77 | TokenScopedSecrets: make(map[string]bool), 78 | StaticScopedSecrets: make(map[string]bool), 79 | VersionScopedSecrets: make(map[string]int64), 80 | TokenScopedComposites: make(map[string]bool), 81 | StaticScopedComposites: make(map[string]bool), 82 | log: zlog.Logger, 83 | metrics: mtrics, 84 | secretCache: make(map[util.SecretLifetime][]SimpleSecret), 85 | } 86 | } 87 | 88 | // ResetBriefcase is used when a vault token from a briefcase is no longer usable. This means any secrets 89 | // that weren't "static" will likely soon expire and disappear. By resetting the briefcase, it will cause 90 | // all the non-static secrets to be recreated. 91 | func (b *Briefcase) ResetBriefcase() *Briefcase { 92 | 93 | b.metrics.Increment(metrics.BriefcaseReset) 94 | 95 | newBriefcase := NewBriefcase(b.metrics) 96 | // AWS Credentials is done through sts:AssumeRole which currently has no reasonable 97 | // revocation mechanism, so credentials remain valid across tokens. 98 | newBriefcase.AWSCredentialLeases = b.AWSCredentialLeases 99 | 100 | // SSH certificates expire when their TTL says they expire and there is no CRL mode for them, so they 101 | // remain valid across tokens. 102 | newBriefcase.SSHCertificates = b.SSHCertificates 103 | 104 | newBriefcase.StaticScopedSecrets = b.StaticScopedSecrets 105 | newBriefcase.VersionScopedSecrets = b.VersionScopedSecrets 106 | newBriefcase.StaticScopedComposites = b.StaticScopedComposites 107 | newBriefcase.StaticTemplates = b.StaticTemplates 108 | return newBriefcase 109 | } 110 | 111 | func LoadBriefcase(filename string, mtrics *metrics.Metrics) (*Briefcase, error) { 112 | zlog.Info().Str("filename", filename).Msg("reading briefcase") 113 | bytes, err := ioutil.ReadFile(filename) 114 | if err != nil { 115 | return nil, fmt.Errorf("could not read briefcase data: %w", err) 116 | } 117 | 118 | bc := NewBriefcase(mtrics) 119 | err = json.Unmarshal(bytes, bc) 120 | if err != nil { 121 | return nil, fmt.Errorf("could not parse briefcase data: %w", err) 122 | } 123 | 124 | return bc, nil 125 | } 126 | 127 | // EnrollVaultToken adds the specified vault token (from Vault) to the briefcase. It captures some expiry information 128 | // so it knows when it needs to be refreshed. 129 | func (b *Briefcase) EnrollVaultToken(ctx context.Context, token *util.WrappedToken) error { 130 | 131 | if token == nil { 132 | return errors.New("can only enroll non-nil tokens") 133 | } 134 | 135 | tokenID, err := token.TokenID() 136 | if err != nil { 137 | return err 138 | } 139 | 140 | accessor, err := token.TokenAccessor() 141 | if err != nil { 142 | return err 143 | } 144 | 145 | ttl, err := token.TokenTTL() 146 | if err != nil { 147 | return err 148 | } 149 | 150 | now := clock.Now(ctx) 151 | 152 | authToken := LeasedAuthToken{ 153 | Token: tokenID, 154 | Accessor: accessor, 155 | Renewable: token.Renewable, 156 | ExpiresAt: now.Add(ttl), 157 | NextRefresh: now.Add(ttl / 3), 158 | } 159 | 160 | if b.AuthTokenLease.Token != tokenID { 161 | b.log = zlog.With().Str("accessor", accessor).Bool("renewable", authToken.Renewable).Logger() 162 | b.log.Info().Str("ttl", ttl.String()).Str("nextRefresh", authToken.NextRefresh.String()).Msg("enrolling vault token with specified ttl into briefcase") 163 | } else { 164 | b.log.Info().Time("expiresAt", authToken.ExpiresAt).Time("nextRefresh", authToken.NextRefresh).Msg("vault token refreshed") 165 | } 166 | 167 | if authToken.ExpiresAt.Before(now.Add(5 * time.Minute)) { 168 | b.log.Warn().Time("expiresAt", authToken.ExpiresAt).Msg("token expires in less than five minutes, setting next refresh to now") 169 | authToken.NextRefresh = now 170 | } 171 | 172 | b.AuthTokenLease = authToken 173 | 174 | return nil 175 | } 176 | 177 | func (b *Briefcase) SaveAs(filename string) error { 178 | bytes, err := json.Marshal(b) 179 | if err != nil { 180 | return err 181 | } 182 | 183 | b.log.Info().Str("filename", filename).Msg("storing briefcase") 184 | util.MustMkdirAllForFile(filename) 185 | if err := ioutil.WriteFile(filename, bytes, 0600); err != nil { 186 | b.log.Error().Err(err).Str("filename", filename).Msg("failed to write briefcase file") 187 | return err 188 | } 189 | 190 | return nil 191 | } 192 | 193 | // ShouldRefreshVaultToken will return true if it's time to do periodic refresh of the Vault token being 194 | // used by the tool. This time is established when the token is enrolled into the briefcase. It will return 195 | // false if the token is not renewable. If the token is needs a refresh but is non-renewable, then it will 196 | // log (but not throw) an error. 197 | func (b *Briefcase) ShouldRefreshVaultToken(ctx context.Context) bool { 198 | 199 | expiring := clock.Now(ctx).After(b.AuthTokenLease.NextRefresh) 200 | 201 | if expiring && !b.AuthTokenLease.Renewable { 202 | // now >= expiredAt 203 | if !clock.Now(ctx).Before(b.AuthTokenLease.ExpiresAt) { 204 | b.log.Error().Time("expiresAt", b.AuthTokenLease.ExpiresAt). 205 | Time("nextRefresh", b.AuthTokenLease.NextRefresh). 206 | Msg("token has expired and is not renewable - results are unpredictable") 207 | } else { 208 | b.log.Error().Time("expiresAt", b.AuthTokenLease.ExpiresAt). 209 | Time("nextRefresh", b.AuthTokenLease.NextRefresh). 210 | Msg("token is expiring, but is set to be non-renewable - unpredictable results will occur once it expires.") 211 | } 212 | return false 213 | } 214 | 215 | return expiring 216 | } 217 | -------------------------------------------------------------------------------- /briefcase/briefcase_test.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "github.com/hootsuite/vault-ctrl-tool/v2/metrics" 7 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 8 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 9 | testing2 "k8s.io/utils/clock/testing" 10 | "os" 11 | "path" 12 | "testing" 13 | "time" 14 | 15 | "github.com/hashicorp/vault/api" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | const exampleToken = `{ 20 | "request_id": "cd08f32e-f7be-60e9-4af7-d76929bd2a14", 21 | "lease_id": "", 22 | "lease_duration": 0, 23 | "renewable": false, 24 | "data": { 25 | "accessor": "8FvDM61Vc23jht83if5bFWlC", 26 | "creation_time": 1598732682, 27 | "creation_ttl": 32400, 28 | "display_name": "ldap-james.atwill", 29 | "entity_id": "07ef9c85-9f14-1272-eb56-92b7bfd21500", 30 | "expire_time": "2020-07-29T22:24:42.348650363-07:00", 31 | "explicit_max_ttl": 0, 32 | "id": "s.eD8onDKEpvQqNCrSZDwxPLld", 33 | "issue_time": "2020-07-29T13:24:42.348663695-07:00", 34 | "meta": { 35 | "username": "james.atwill" 36 | }, 37 | "num_uses": 0, 38 | "orphan": true, 39 | "path": "auth/ldap/login/james.atwill", 40 | "policies": [ 41 | "a", 42 | "b", 43 | "c", 44 | "default" 45 | ], 46 | "renewable": true, 47 | "ttl": 32382, 48 | "type": "service" 49 | }, 50 | "warnings": null 51 | }` 52 | 53 | func myToken(t *testing.T) api.Secret { 54 | var secret api.Secret 55 | err := json.Unmarshal([]byte(exampleToken), &secret) 56 | assert.NoError(t, err, "could not deserialize example token: %v", err) 57 | return secret 58 | } 59 | 60 | func TestSavePermissions(t *testing.T) { 61 | tempDir := t.TempDir() 62 | defer os.RemoveAll(tempDir) 63 | filename := path.Join(tempDir, "briefcase") 64 | 65 | emptyBriefcase := NewBriefcase(nil) 66 | err := emptyBriefcase.SaveAs(filename) 67 | assert.NoError(t, err, "must be able to save empty briefcase. filename=%q", filename) 68 | 69 | stat, err := os.Stat(filename) 70 | assert.NoError(t, err, "must be able to Stat() briefcase. filename=%q", filename) 71 | 72 | // bitwise 'and' the "group" and "other" modes which should both be zero, so the result should be 0. 73 | assert.Zero(t, stat.Mode()&0077, "briefcase must only be accessible to owner") 74 | } 75 | 76 | func TestSaveAndLoadEmpty(t *testing.T) { 77 | tempDir := t.TempDir() 78 | defer os.RemoveAll(tempDir) 79 | filename := path.Join(tempDir, "briefcase") 80 | 81 | emptyBriefcase := NewBriefcase(nil) 82 | err := emptyBriefcase.SaveAs(filename) 83 | assert.NoError(t, err, "must be able to save empty briefcase. filename=%q", filename) 84 | 85 | loadedBriefcase, err := LoadBriefcase(filename, nil) 86 | assert.NoError(t, err, "must be able to load empty briefcase. filename=%q", filename) 87 | 88 | assert.EqualValues(t, emptyBriefcase, loadedBriefcase, "empty briefcase and loaded empty briefcase must be the same") 89 | } 90 | 91 | func TestSaveAndLoadEnrolledToken(t *testing.T) { 92 | tempDir := t.TempDir() 93 | defer os.RemoveAll(tempDir) 94 | filename := path.Join(tempDir, "briefcase") 95 | 96 | bc := NewBriefcase(nil) 97 | 98 | token := myToken(t) 99 | 100 | assert.NoError(t, bc.EnrollVaultToken(context.Background(), util.NewWrappedToken(&token, true)), "must be able to enroll example token in briefcase") 101 | assert.NoError(t, bc.SaveAs(filename), "must be able to save briefcase") 102 | 103 | loadedBriefcase, err := LoadBriefcase(filename, nil) 104 | assert.NoError(t, err, "must be able to reload briefcase") 105 | 106 | assert.False(t, loadedBriefcase.ShouldRefreshVaultToken(context.TODO()), "must not need to refresh a token with a TTL") 107 | assert.Equal(t, "s.eD8onDKEpvQqNCrSZDwxPLld", loadedBriefcase.AuthTokenLease.Token, "loaded token must equal saved token") 108 | assert.Equal(t, "8FvDM61Vc23jht83if5bFWlC", loadedBriefcase.AuthTokenLease.Accessor, "loaded accessor must equal saved accessor") 109 | } 110 | 111 | func TestExpiringTokenNeedsRefresh(t *testing.T) { 112 | bc := NewBriefcase(nil) 113 | token := myToken(t) 114 | 115 | token.Data["ttl"] = 299 116 | 117 | assert.NoError(t, bc.EnrollVaultToken(context.TODO(), util.NewWrappedToken(&token, true)), "must be able to enroll example token in briefcase") 118 | assert.True(t, bc.ShouldRefreshVaultToken(context.TODO()), "token expiring in less than 5 minutes must require a refresh") 119 | } 120 | 121 | func TestExpiringNonRenewableTokenNeverNeedsRefresh(t *testing.T) { 122 | bc := NewBriefcase(nil) 123 | token := myToken(t) 124 | 125 | token.Data["ttl"] = 1 126 | 127 | assert.NoError(t, bc.EnrollVaultToken(context.TODO(), util.NewWrappedToken(&token, false)), "must be able to enroll example token in briefcase") 128 | assert.False(t, bc.ShouldRefreshVaultToken(context.TODO()), "non renewable token must never need refreshing") 129 | } 130 | 131 | func TestExpiredNonRenewableTokenNeverNeedsRefresh(t *testing.T) { 132 | bc := NewBriefcase(nil) 133 | token := myToken(t) 134 | 135 | fakeClock := testing2.NewFakeClock(time.Now()) 136 | clockContext := clock.Set(context.Background(), fakeClock) 137 | 138 | assert.NoError(t, bc.EnrollVaultToken(clockContext, util.NewWrappedToken(&token, false)), "must be able to enroll example token in briefcase") 139 | 140 | fakeClock.Sleep(100 * time.Hour) 141 | assert.False(t, bc.ShouldRefreshVaultToken(clockContext), "non renewable token must never need refreshing") 142 | } 143 | 144 | func TestNilTokenEnrollment(t *testing.T) { 145 | bc := NewBriefcase(nil) 146 | 147 | assert.NotPanics(t, func() { 148 | assert.Error(t, bc.EnrollVaultToken(context.TODO(), nil), "trying to enroll a nil token must return an error") 149 | }, "trying to enroll a nil token must not panic()") 150 | } 151 | 152 | func TestResetBriefcase(t *testing.T) { 153 | s := map[string]bool{ 154 | "a": true, 155 | "b": true, 156 | } 157 | 158 | sversioned := map[string]int64{ 159 | "a": 3, 160 | "b": 8, 161 | } 162 | 163 | aws := map[string]leasedAWSCredential{ 164 | "foo": {}, 165 | "bar": {}, 166 | } 167 | 168 | ssh := map[string]sshCert{ 169 | "baz": {}, 170 | } 171 | 172 | mtrcs := metrics.NewMetrics() 173 | big := NewBriefcase(mtrcs) 174 | big.AWSCredentialLeases = aws 175 | big.SSHCertificates = ssh 176 | big.TokenScopedTemplates = s 177 | big.StaticTemplates = s 178 | big.TokenScopedSecrets = s 179 | big.StaticScopedSecrets = s 180 | big.TokenScopedComposites = s 181 | big.StaticScopedComposites = s 182 | big.VersionScopedSecrets = sversioned 183 | 184 | small := NewBriefcase(mtrcs) 185 | small.AWSCredentialLeases = aws 186 | small.SSHCertificates = ssh 187 | small.StaticTemplates = s 188 | small.StaticScopedSecrets = s 189 | small.StaticScopedComposites = s 190 | small.VersionScopedSecrets = sversioned 191 | resetBig := big.ResetBriefcase() 192 | 193 | assert.EqualValues(t, small, resetBig, "resetting a briefcase should leave non-token scoped data") 194 | assert.Equal(t, 1, mtrcs.Counter(metrics.BriefcaseReset)) 195 | } 196 | -------------------------------------------------------------------------------- /briefcase/json_secrets.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 6 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 7 | ) 8 | 9 | func (b *Briefcase) ShouldRefreshSecret(secret config.SecretType) bool { 10 | var exists bool 11 | 12 | b.log.Debug().Interface("briefcase", b).Msg("current briefcase") 13 | 14 | switch secret.Lifetime { 15 | case util.LifetimeToken: 16 | _, exists = b.TokenScopedSecrets[secret.Path] 17 | case util.LifetimeStatic: 18 | _, exists = b.StaticScopedSecrets[secret.Path] 19 | default: 20 | panic(fmt.Sprintf("briefcase does not manage refresh of %q lifetime secrets", secret.Lifetime)) 21 | } 22 | 23 | return !exists 24 | } 25 | 26 | func (b *Briefcase) EnrollSecret(secret config.SecretType) { 27 | b.log.Info().Str("vaultPath", secret.Path).Interface("lifetime", secret.Lifetime).Msg("enrolling secret") 28 | 29 | switch secret.Lifetime { 30 | case util.LifetimeToken: 31 | b.TokenScopedSecrets[secret.Path] = true 32 | case util.LifetimeStatic: 33 | b.StaticScopedSecrets[secret.Path] = true 34 | default: 35 | panic(fmt.Sprintf("lifetime of %q cannot be enrolled in briefcase", secret.Lifetime)) 36 | } 37 | } 38 | 39 | func (b *Briefcase) ShouldRefreshComposite(composite config.CompositeSecretFile) bool { 40 | var exists bool 41 | 42 | switch composite.Lifetime { 43 | case util.LifetimeToken: 44 | _, exists = b.TokenScopedComposites[composite.Filename] 45 | case util.LifetimeStatic: 46 | _, exists = b.StaticScopedComposites[composite.Filename] 47 | default: 48 | panic(fmt.Sprintf("composites cannot have a lifetime of %q cannot be enrolled in briefcase", composite.Lifetime)) 49 | 50 | } 51 | 52 | return !exists 53 | } 54 | 55 | func (b *Briefcase) EnrollComposite(composite config.CompositeSecretFile) { 56 | b.log.Info().Str("filename", composite.Filename).Interface("lifetime", composite.Lifetime).Msg("enrolling composite secret") 57 | 58 | switch composite.Lifetime { 59 | case util.LifetimeToken: 60 | b.TokenScopedComposites[composite.Filename] = true 61 | case util.LifetimeStatic: 62 | b.StaticScopedComposites[composite.Filename] = true 63 | default: 64 | panic(fmt.Sprintf("enrolling composites of lifetime %q is not supported", composite.Lifetime)) 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /briefcase/secrets_cache.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import "github.com/hootsuite/vault-ctrl-tool/v2/util" 4 | 5 | // SecretsCache is the interface to the non-persisted secrets that are kept in the briefcase. This could probably 6 | // be kept outside the briefcase, but we use the briefcase as blackboard style runtime state right now. 7 | type SecretsCache interface { 8 | HasCachedSecrets(lifetime util.SecretLifetime) bool 9 | StoreSecrets(lifetime util.SecretLifetime, secrets []SimpleSecret) 10 | GetSecrets(lifetime util.SecretLifetime) []SimpleSecret 11 | } 12 | 13 | func (b *Briefcase) HasCachedSecrets(lifetime util.SecretLifetime) bool { 14 | switch lifetime { 15 | case util.LifetimeToken, util.LifetimeStatic: 16 | return len(b.secretCache[lifetime]) > 0 17 | case util.LifetimeVersion: 18 | b.log.Error().Msgf("secrets with the lifetime of %q are never cached", util.LifetimeVersion) 19 | return false 20 | default: 21 | b.log.Error().Interface("lifetime", lifetime).Msg("internal error: specified lifetime is never cached") 22 | return false 23 | } 24 | } 25 | 26 | func (b *Briefcase) StoreSecrets(lifetime util.SecretLifetime, secrets []SimpleSecret) { 27 | b.secretCache[lifetime] = secrets 28 | } 29 | 30 | func (b *Briefcase) GetSecrets(lifetime util.SecretLifetime) []SimpleSecret { 31 | return b.secretCache[lifetime] 32 | } 33 | -------------------------------------------------------------------------------- /briefcase/ssh.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "path/filepath" 8 | "time" 9 | 10 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 12 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 13 | "golang.org/x/crypto/ssh" 14 | ) 15 | 16 | var neverExpires = time.Unix(0, 0) 17 | 18 | func (b *Briefcase) ShouldRefreshSSHCertificate(sshCertConfig config.SSHCertificateType, expiresBefore time.Time) bool { 19 | entry, ok := b.SSHCertificates[sshCertConfig.OutputPath] 20 | if !ok { 21 | return true 22 | } 23 | 24 | b.log.Debug().Time("expiry", entry.Expiry).Str("outputPath", sshCertConfig.OutputPath).Msg("determined expiry of ssh certificate") 25 | 26 | certExpiresBefore := entry.Expiry.Before(expiresBefore) || entry.Expiry == neverExpires 27 | shouldRefreshBefore := entry.RefreshExpiry != nil && !entry.RefreshExpiry.IsZero() && entry.RefreshExpiry.Before(expiresBefore) 28 | 29 | return certExpiresBefore || shouldRefreshBefore 30 | } 31 | 32 | func createRefreshExpiry(ctx context.Context, forceRefreshTTL time.Duration) *time.Time { 33 | var refreshExpiry *time.Time 34 | // we only add a refresh if a ttl value was set. 35 | if forceRefreshTTL > 0 { 36 | exp := clock.Now(ctx).Add(forceRefreshTTL) 37 | refreshExpiry = &exp 38 | } 39 | return refreshExpiry 40 | } 41 | 42 | // EnrollSSHCertificate adds a managed SSH certificate to briefcase. If forceRefreshTTL is not zero, then it will associate a 43 | // refresh expiry time with the certificate. 44 | func (b *Briefcase) EnrollSSHCertificate(ctx context.Context, sshCertConfig config.SSHCertificateType, forceRefreshTTL time.Duration) error { 45 | if forceRefreshTTL < 0 { 46 | return fmt.Errorf("forceRefreshTTL cannot be negative: %s", forceRefreshTTL) 47 | } 48 | 49 | certificateFilename := filepath.Join(sshCertConfig.OutputPath, util.SSHCertificate) 50 | 51 | log := b.log.With().Str("filename", certificateFilename).Logger() 52 | 53 | log.Debug().Msg("enrolling ssh certificate") 54 | validBefore, err := b.readSSHCertificateValidBefore(certificateFilename) 55 | if err != nil { 56 | log.Debug().Err(err).Msg("failed to read ssh valid before field in certificate") 57 | return err 58 | } 59 | 60 | var validBeforeTime time.Time 61 | 62 | if validBefore == ssh.CertTimeInfinity { 63 | b.log.Warn().Str("sshCertificate", certificateFilename).Msg("ssh certificate never expires") 64 | validBeforeTime = neverExpires 65 | } else { 66 | validBeforeTime = time.Unix(int64(validBefore), 0) 67 | } 68 | 69 | log.Debug().Time("validBefore", validBeforeTime).Msg("ssh certificate validity") 70 | b.SSHCertificates[sshCertConfig.OutputPath] = sshCert{ 71 | Expiry: validBeforeTime, 72 | RefreshExpiry: createRefreshExpiry(ctx, forceRefreshTTL), 73 | Cfg: sshCertConfig, 74 | } 75 | return nil 76 | } 77 | 78 | func (b *Briefcase) readSSHCertificateValidBefore(certificate string) (uint64, error) { 79 | certificateBytes, err := ioutil.ReadFile(certificate) 80 | if err != nil { 81 | return 0, fmt.Errorf("could not read certificate file %q: %w", certificate, err) 82 | } 83 | 84 | // ParseAuthorizedKeys parses a public key from an authorized_keys file used in OpenSSH 85 | pk, _, _, _, err := ssh.ParseAuthorizedKey(certificateBytes) 86 | if err != nil { 87 | return 0, err 88 | } 89 | // pk.Marshal() Marshal returns the serialized key data in SSH wire format, with the name prefix 90 | // ssh.ParsePublicKey is used to unmarshal the returned data 91 | cert, err := ssh.ParsePublicKey(pk.Marshal()) 92 | if err != nil { 93 | return 0, err 94 | } 95 | 96 | sshCert, ok := cert.(*ssh.Certificate) 97 | if !ok { 98 | return 0, fmt.Errorf("could not parse certificate %q", certificate) 99 | } 100 | 101 | return sshCert.ValidBefore, nil 102 | } 103 | -------------------------------------------------------------------------------- /briefcase/ssh_test.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "context" 5 | "crypto/rand" 6 | "crypto/rsa" 7 | "os" 8 | "path" 9 | "testing" 10 | "time" 11 | 12 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 13 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 14 | "github.com/stretchr/testify/assert" 15 | "golang.org/x/crypto/ssh" 16 | testing2 "k8s.io/utils/clock/testing" 17 | ) 18 | 19 | const ( 20 | fakeSSHKeySize = 4096 21 | ) 22 | 23 | var ( 24 | testTime = time.Unix(1443332960, 0) 25 | ) 26 | 27 | func createSSHSignedPublicKey(testTime time.Time, dir string, certTTL time.Duration, t *testing.T) { 28 | assert := assert.New(t) 29 | hostKeys, err := rsa.GenerateKey(rand.Reader, fakeSSHKeySize) 30 | assert.NoError(err) 31 | 32 | clientKeys, err := rsa.GenerateKey(rand.Reader, fakeSSHKeySize) 33 | assert.NoError(err) 34 | clientPKey, err := ssh.NewPublicKey(&clientKeys.PublicKey) 35 | assert.NoError(err) 36 | 37 | signer, err := ssh.NewSignerFromKey(hostKeys) 38 | assert.NoError(err) 39 | 40 | cert := &ssh.Certificate{ 41 | Key: clientPKey, 42 | ValidBefore: uint64(testTime.Add(certTTL).Unix()), 43 | ValidAfter: uint64(testTime.Unix()), 44 | } 45 | assert.NoError(cert.SignCert(rand.Reader, signer)) 46 | 47 | fd, err := os.Create(path.Join(dir, "id_rsa-cert.pub")) 48 | assert.NoError(err) 49 | _, err = fd.Write(ssh.MarshalAuthorizedKey(cert)) 50 | assert.NoError(err) 51 | fd.Close() 52 | } 53 | 54 | func TestSSHCredentialsExpireAndRefreshCheck(t *testing.T) { 55 | assert := assert.New(t) 56 | 57 | ctx := clock.Set(context.Background(), testing2.NewFakeClock(testTime)) 58 | 59 | bc := NewBriefcase(nil) 60 | 61 | tmpDir := t.TempDir() 62 | createSSHSignedPublicKey(testTime, tmpDir, time.Hour, t) 63 | 64 | certConfig := config.SSHCertificateType{ 65 | VaultMount: "ssh", 66 | VaultRole: "user-readonly", 67 | OutputPath: tmpDir, 68 | } 69 | 70 | bc.EnrollSSHCertificate(ctx, certConfig, 0) 71 | 72 | assert.True(bc.ShouldRefreshSSHCertificate(certConfig, clock.Now(ctx).Add(3601*time.Second)), "if cert expires before next check, should return true") 73 | assert.False(bc.ShouldRefreshSSHCertificate(certConfig, clock.Now(ctx).Add(45*60*time.Second)), "if cert expires after next check, should return false") 74 | 75 | // re-enroll, now with a forced refresh time. 76 | bc.EnrollSSHCertificate(ctx, certConfig, 30*time.Minute) 77 | 78 | assert.True(bc.ShouldRefreshSSHCertificate(certConfig, clock.Now(ctx).Add(45*60*time.Second)), "if cert expires after next check, but the tokens refresh ttl expires before next check, should return true") 79 | assert.False(bc.ShouldRefreshSSHCertificate(certConfig, clock.Now(ctx).Add(29*time.Minute)), "if cert expires after next check, and the tokens refresh ttl expires after next check, should return false") 80 | 81 | // create new key with longer expiry 82 | createSSHSignedPublicKey(testTime, tmpDir, 3602*time.Second, t) 83 | bc.EnrollSSHCertificate(ctx, certConfig, 0) 84 | assert.False(bc.ShouldRefreshSSHCertificate(certConfig, clock.Now(ctx).Add(3601*time.Second)), "if cert expires before next check, should return true") 85 | bc.EnrollSSHCertificate(ctx, certConfig, 3600*time.Second) 86 | assert.True(bc.ShouldRefreshSSHCertificate(certConfig, clock.Now(ctx).Add(3601*time.Second)), "if cert expires before next check, should return true") 87 | 88 | // test some edge cases 89 | createSSHSignedPublicKey(testTime, tmpDir, -1*time.Second, t) 90 | bc.EnrollSSHCertificate(ctx, certConfig, 0) 91 | assert.True(bc.ShouldRefreshSSHCertificate(certConfig, clock.Now(ctx)), "if cert already expired, should refresh") 92 | 93 | createSSHSignedPublicKey(testTime, tmpDir, 1*time.Second, t) 94 | assert.Error(bc.EnrollSSHCertificate(ctx, certConfig, -1*time.Second), "negative refresh TTL values should not be allowed") 95 | } 96 | -------------------------------------------------------------------------------- /briefcase/template.go: -------------------------------------------------------------------------------- 1 | package briefcase 2 | 3 | import ( 4 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 5 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 6 | ) 7 | 8 | func (b *Briefcase) ShouldRefreshTemplate(tmpl config.TemplateType) bool { 9 | var exists bool 10 | 11 | if tmpl.Lifetime == util.LifetimeToken { 12 | _, exists = b.TokenScopedTemplates[tmpl.Output] 13 | } else { 14 | _, exists = b.StaticTemplates[tmpl.Output] 15 | } 16 | 17 | return !exists 18 | } 19 | 20 | func (b *Briefcase) EnrollTemplate(tmpl config.TemplateType) { 21 | 22 | b.log.Info().Str("outputFile", tmpl.Output).Interface("lifetime", tmpl.Lifetime).Msg("enrolling template") 23 | 24 | if tmpl.Lifetime == util.LifetimeToken { 25 | b.TokenScopedTemplates[tmpl.Output] = true 26 | } else { 27 | b.StaticTemplates[tmpl.Output] = true 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | var validConfigs = map[string]string{ 11 | "Empty File": ``, 12 | "Only with Version 2": `--- 13 | version: 2`, 14 | } 15 | 16 | var invalidConfigs = map[string]string{ 17 | "Bare word": `kapow`, 18 | "Missing secret path": ` 19 | --- 20 | version: 3 21 | secrets: 22 | - output: path/to/file 23 | key: some-key 24 | `, 25 | "Missing key in secret": ` 26 | --- 27 | version: 3 28 | secrets: 29 | - path: path/to/secret 30 | output: path/to/file 31 | `, 32 | "Missing fields for version scoped secret": `--- 33 | version: 1 34 | secrets: 35 | - key: ex 36 | path: path/to/secret 37 | lifetime: version 38 | `, 39 | "Output specified for version scoped secret": `--- 40 | version: 1 41 | secrets: 42 | - key: ex 43 | output: path/to/file 44 | path: path/to/secret 45 | lifetime: version 46 | fields: 47 | - name: api_key 48 | output: path/to/file 49 | `, 50 | } 51 | 52 | var validSubConfig = map[string]string{ 53 | "sub config1": `--- 54 | version: 3 55 | secrets: 56 | - key: test 57 | path: /secret/test 58 | lifetime: static`, 59 | "sub config2": `--- 60 | version: 3 61 | secrets: 62 | - key: test2 63 | path: /secret/test2 64 | lifetime: static 65 | `, 66 | } 67 | 68 | func TestValidConfigs(t *testing.T) { 69 | for k, v := range validConfigs { 70 | t.Run(k, func(t *testing.T) { 71 | filename := mkConfig(t, t.TempDir(), v) 72 | _, err := ReadConfigFile(filename, "", "", "") 73 | if err != nil { 74 | t.Fatalf("this config must be okay, got error: %v", err) 75 | } 76 | }) 77 | } 78 | } 79 | 80 | func TestInvalidConfigs(t *testing.T) { 81 | for k, v := range invalidConfigs { 82 | t.Run(k, func(t *testing.T) { 83 | filename := mkConfig(t, t.TempDir(), v) 84 | _, err := ReadConfigFile(filename, "", "", "") 85 | if err == nil { 86 | t.Fatal("this config must generate an error") 87 | } 88 | }) 89 | } 90 | } 91 | 92 | func TestConfigDir(t *testing.T) { 93 | 94 | dir := t.TempDir() 95 | for configName, configData := range validSubConfig { 96 | f := mkConfig(t, dir, configData) 97 | t.Logf("created config for: %s : %s", configName, f) 98 | } 99 | emptyConfig := mkConfig(t, dir, "") 100 | config, err := ReadConfigFile(emptyConfig, dir, "", "") 101 | if err != nil { 102 | t.Log("this config must be okay, got error") 103 | t.Fail() 104 | } 105 | 106 | key := SecretType{Key: "test", Path: "/secret/test", Lifetime: "static"} 107 | assert.Contains(t, config.VaultConfig.Secrets, key) 108 | 109 | key2 := SecretType{Key: "test2", Path: "/secret/test2", Lifetime: "static"} 110 | assert.Contains(t, config.VaultConfig.Secrets, key2) 111 | } 112 | 113 | func mkConfig(t *testing.T, directory string, body string) string { 114 | f, err := ioutil.TempFile(directory, "config_test_*.yml") 115 | 116 | if err != nil { 117 | t.Fatalf("could not make temp file: %v", err) 118 | } 119 | 120 | var filename = f.Name() 121 | 122 | if _, err := f.WriteString(body); err != nil { 123 | t.Fatalf("could not write to temp file: %v", err) 124 | } 125 | 126 | if err := f.Close(); err != nil { 127 | t.Fatalf("could not close temp file: %v", err) 128 | } 129 | 130 | return filename 131 | } 132 | -------------------------------------------------------------------------------- /config/templates.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "text/template" 5 | ) 6 | 7 | // v1 of vault-ctrl-tool parsed templates on startup so that typos would be caught during initializing, 8 | // we keep this behaviour. 9 | func (cfg *VaultConfig) ingestTemplates() (map[string]*template.Template, error) { 10 | 11 | templates := make(map[string]*template.Template) 12 | 13 | for _, tpl := range cfg.Templates { 14 | cfg.log.Info().Str("input", tpl.Input).Msg("ingesting template") 15 | t, err := template.ParseFiles(tpl.Input) 16 | if err != nil { 17 | cfg.log.Error().Err(err).Str("input", tpl.Input).Msg("failed to parse template") 18 | return nil, err 19 | } 20 | templates[tpl.Input] = t 21 | } 22 | 23 | return templates, nil 24 | } 25 | -------------------------------------------------------------------------------- /docs/BUILDING.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | At Hootsuite, we have an internal repository that fetches this repository, runs `make build`, then deploys 4 | the generated binary into a number of different forms internally. This document discusses how you can do this too. 5 | 6 | # Snippets from the Hootsuite Build 7 | 8 | Our Makefile assumes this project is checked out into `target/open-source-project`, and therefore has a 9 | target to trigger a build: 10 | 11 | ``` 12 | jenkins-build: 13 | make -C target/open-source-project build 14 | @mkdir -p bin 15 | cp target/open-source-project/bin/vault-ctrl-tool.* bin/ 16 | ``` 17 | 18 | To embed the tool in a Docker container (if you're using it with Kubernetes), you can have a simple 19 | Dockerfile: 20 | 21 | ```dockerfile 22 | FROM ubuntu:latest 23 | ADD bin/vault-ctrl-tool.linux.amd64 /vault-ctrl-tool 24 | CMD [] 25 | ``` 26 | 27 | and then make targets: 28 | 29 | ``` 30 | docker-image: 31 | docker build -t vault-ctrl-tool:latest . 32 | 33 | docker-deploy: 34 | docker tag vault-ctrl-tool:latest docker-registry.hootsuite.com/tools/vault-ctrl-tool:latest 35 | docker tag vault-ctrl-tool:latest docker-registry.hootsuite.com/tools/vault-ctrl-tool:$(BUILD_ID) 36 | docker push docker-registry.hootsuite.com/tools/vault-ctrl-tool:latest 37 | docker push docker-registry.hootsuite.com/tools/vault-ctrl-tool:$(BUILD_ID) 38 | ``` 39 | 40 | Our `Jenkinsfile` works something similar to: 41 | 42 | ```groovy 43 | node('example') { 44 | 45 | stage('Setup') { 46 | checkout scm 47 | } 48 | 49 | stage('Checkout Open Source Project') { 50 | checkout scm: [ 51 | $class : 'GitSCM', 52 | branches : [[name: '*/master']], 53 | doGenerateSubmoduleConfigurations: false, 54 | extensions : [[$class : 'RelativeTargetDirectory', 55 | relativeTargetDir: 'target/open-source-project']], 56 | submoduleCfg : [], 57 | userRemoteConfigs : [[url: 'https://github.com/hootsuite/vault-ctrl-tool.git']] 58 | ] 59 | 60 | } 61 | 62 | stage('Build') { 63 | sh "make jenkins-build" 64 | } 65 | 66 | stage('Deploy') { 67 | sh "make deploy" 68 | } 69 | } 70 | ``` 71 | 72 | Hope this helps! -------------------------------------------------------------------------------- /docs/CONFIGURATION.md: -------------------------------------------------------------------------------- 1 | # Tour Of vault-config.yml 2 | 3 | * [Concepts](#concepts) 4 | * [Vault Token](#vaultToken) 5 | * [Templates](#templates) 6 | * [Secrets](#secrets) 7 | * [SSH Keys](#ssh) 8 | * [AWS](#aws) 9 | 10 | 11 | ### Concepts 12 | 13 | At the top of your configuration file, is a `version`. Upgrading from one configuration version to another may 14 | require some small changes to your configuration file. See the README and CHANGELOG at the root of the project. 15 | 16 | Secrets and templates both have a `lifetime`. Lifetimes are how the tool knows when it needs to rewrite your 17 | secrets or templates. Templates and secrets can use `token` and `static`. There is a third lifetime of 18 | `version` discussed below. Secrets and templates with static lifetimes are never rewritten once they're 19 | initially created. It is assumed the secrets in those files are static, and as a service using this tool, 20 | you would rather have to force restarting things in order to get new secrets (either by restarting your 21 | Kubernetes pod, terminating an EC2 instance in an ASG, or deleting the briefcase file - causing the tool to lose state). 22 | 23 | The lifetime of `token` indicates that the secrets contained in the files become invalid if the Vault token 24 | being used expires. These secrets will be fetched again, and files will be rewritten out of necessity after the tool 25 | re-authenticates. 26 | 27 | The lifetime of `version` is quite special, only valid on secrets stored in a KVv2 backend, and has a limited use case. 28 | Secrets that do not have an `output` may use `version`. When the tool runs, it will always fetch a copy of the secret from 29 | Vault. If the version in Vault is newer than the one in the briefcase, and the new secret is older than 30 seconds, any 30 | fields that specify an `output` will be overwritten. See the [Secrets](#secrets) section below before using this. 31 | 32 | These examples assume you're running with `--input-prefix /etc/vault-config --output-prefix /etc/secrets`. 33 | 34 | ### VaultToken 35 | 36 | ```yaml 37 | # 38 | # Write a copy of the token to /etc/secrets/example/target/vault-token - This can be used by your 39 | # service if you interact with Vault directly and want to take care of keeping your credentials 40 | # refreshed, etc. This is useful if you have some use case not covered by the tool and want 41 | # to interact with Vault directly. Note that vault-ctrl-tool may rewrite this file if it needs 42 | # to relogin to Vault when running in sidecar mode. 43 | vaultToken: 44 | output: example/target/vault-token 45 | mode: 0777 46 | ``` 47 | 48 | ### Templates 49 | 50 | ```yaml 51 | # 52 | # Templates allow you to make your own custom output files using golang templates, secrets will 53 | # be written to the file as specified. Templates should be included with the ConfigMap where 54 | # the vault-config.yml is specified 55 | templates: 56 | - input: example-test.tpl 57 | output: example/target/test 58 | mode: 0777 59 | lifetime: static 60 | # Read template from /etc/vault-config/example-test.tpl and write to /etc/secrets/example/target/test 61 | ``` 62 | 63 | ### Secrets 64 | 65 | Secrets are a core piece of Vault, so there are a few examples here. 66 | 67 | #### Secrets: Original Example 68 | 69 | ```yaml 70 | # 71 | # The /secret/ backend (aka the v1 kv secrets backend). The specified "path" is read from Vault, 72 | # and the keys returned are made available in templates prefixed with the "key" value below. If 73 | # an "output" file is specified the keys and values are written in JSON format to the output file. 74 | # If "path" is relative, it is prefixed with "/secret/application-config/services/". If "use_key_as_prefix" 75 | # is set to true, then the fields written to the JSON file will be prefixed with the key followed by 76 | # an underscore (so "ex_api_key" and "ex_api_secret" in the example below). 77 | # Note that "key" must be unique across all secrets. 78 | secrets: 79 | - key: ex 80 | path: example/keys 81 | output: example/target/example.secrets 82 | use_key_as_prefix: true 83 | mode: 0777 84 | missingOk: true 85 | lifetime: static 86 | fields: 87 | - name: api_key 88 | output: api/key 89 | - name: api_secret 90 | output: api/secret 91 | - name: license 92 | output: license.key 93 | encoding: base64 94 | 95 | # If "example/keys" had a secret of "foo" with the value "bar", then templates could 96 | # reference {{.ex_foo}} to get "bar", and the file /etc/secrets/example/target/example.secrets would 97 | # have the JSON of '{"ex_foo": "bar"}' in it (as "use_key_as_prefix" is set to true). 98 | # If Vault doesn't have an "example/keys", then it will be 99 | # noisily ignored as 'missingOk' is set to true -- the tool cannot tell the difference between a path it 100 | # cannot access and a path with no secrets. Be warned. The contents of the individual fields in the 101 | # secret ("api_key" and "api_secret") will be written verbatim to "/etc/secrets/api/key" and 102 | # "/etc/secrets/api/secret" if the fields present. The field "license" will be written to 103 | # "/etc/secrets/license.key", but the value stored in Vault will be manually base64 decoded. Fields must 104 | # be manually base64 encoded before being written to take advangate of "encoding: base64". 105 | # NOTE: All files share the same file mode. 106 | # NOTE: If you have multiple secrets sharing the same output file, they will use the file mode 107 | # of the first stanza. 108 | ``` 109 | 110 | #### Secrets: Pinned Version 111 | 112 | ```yaml 113 | # A small example using "pinnedVersion". This only works on KVv2 secrets and is very dangerous to use. 114 | # If you forget that pinnedVersion is set and update your secrets, systems using the old version will cease to function 115 | # which can easily lead to outages. Use only sparingly for testing. Do not use this defensively "just in case someone" 116 | # changes the secret. 117 | 118 | secrets: 119 | - path: example/keys 120 | output: example.secrets 121 | mode: 0700 122 | missingOk: false 123 | pinnedVersion: 2 124 | lifetime: static 125 | 126 | ``` 127 | 128 | #### Secrets: "Version" Lifetime 129 | 130 | ```yaml 131 | # Version lifetimes are available for secrets that do not have an "output". They inherently go against the 132 | # existing workflow that one would expect from vault-ctrl-tool. 133 | 134 | secrets: 135 | - key: ex 136 | path: example/keys 137 | mode: 0777 138 | missingOk: false 139 | lifetime: version 140 | touchfile: /etc/third-party/last-refresh 141 | fields: 142 | - name: api_key 143 | output: api/key 144 | - name: api_secret 145 | output: api/secret 146 | 147 | # Each time vault-ctrl-tool runs to synchronize the on-disk output with what is in Vault, it will fetch the above 148 | # secret from Vault. If the version in Vault is newer, and at least 30 seconds old, it will rewrite the output 149 | # of the fields (in this case "api/key" and "api/secret"). After it rewrites the fields, it will "touch" the 150 | # listed "touchfile" (in this case "/etc/third-party/last-refresh"). Services that want to be notified when there 151 | # are changes must watch the "touchfile" which will be touched after all the fields are updated. 152 | ``` 153 | 154 | ### SSH 155 | 156 | ```yaml 157 | # 158 | # The tool will create a public/private keypair and have the public key signed by Vault at the 159 | # mount point specified requesting the role specified. Files written are only readable by the 160 | # owner and will all exist in the "/etc/secrets/ssh-key/" directory under the names "id_rsa", 161 | # "id_rsa.pub" and "id_rsa-cert.pub". Services can use the files directly 162 | # ( ssh -i /etc/secrets/ssh-key/id_rsa ), or do with them as they wish. In sidecar mode, the public 163 | # key will be signed periodically to ensure it doesn't expire, writing it to the same 164 | # location. Services that copy the file elsewhere will need to periodically update their copy. 165 | sshCertificates: 166 | - vaultMountPoint: ssh/keyprovider 167 | vaultRole: jenkins 168 | outputPath: ssh-key 169 | ``` 170 | 171 | ### AWS 172 | 173 | ```yaml 174 | # The tool will fetch AWS credentials from Vault on your behalf (Vault has permission to 175 | # sts:AssumeRole a number of roles). It will manage these credentials and pass them to your 176 | # service. Running in sidecar mode, the credentials will be kept refreshed and will be rewritten 177 | # hourly so they're always fresh. Services that copy the output file somewhere else will need to 178 | # periodically update their copy. Services are instead encouraged to set AWS_CONFIG_FILE and 179 | # AWS_SHARED_CREDENTIALS_FILE to the config and credentials files in outputPath. 180 | aws: 181 | - awsProfile: default 182 | vaultMountPoint: aws 183 | vaultRole: jenkins 184 | awsRegion: us-east-1 185 | outputPath: aws 186 | mode: 0777 187 | - awsProfile: special 188 | vaultMountPoint: aws 189 | vaultRole: special-role 190 | awsRegion: us-east-1 191 | outputPath: aws 192 | mode: 0777 193 | # The above will output a "/etc/secrets/aws/config" and "/etc/secrets/aws/credentials" with 194 | # two AWS profiles ("default", and "special") which can be specified with AWS_PROFILE. 195 | ``` 196 | -------------------------------------------------------------------------------- /docs/EC2.md: -------------------------------------------------------------------------------- 1 | # Using EC2 2 | 3 | Vault Control Tool can run both on startup to authenticate an EC2 instance to Vault, as well as periodically 4 | in a cronjob to keep various secrets fresh for your system. 5 | 6 | Vault Control Tool supports both authentication types present in the aws auth method - iam and ec2. 7 | 8 | ## IAM 9 | 10 | ### Setup 11 | 12 | Assuming you have already bound an IAM role to your EC2 instance, you'll need to create a role in Vault that associates 13 | this IAM role with specific policies. 14 | 15 | ```bash 16 | vault write auth/aws/role/example auth_type=iam bound_iam_principal_arn=example-iam-role-arn policies=example max_ttl=500h 17 | ``` 18 | 19 | Vault Control Tool requires the `--iam-auth-role` flag to be set to this role name in order to authenticate to Vault using it. 20 | 21 | ### On Startup 22 | 23 | Assuming you have a `vault-config.yml` in `/etc/vault-ctrl-tool`, initialization is as easy as: 24 | 25 | ```bash 26 | export VAULT_ADDR=https://vault.service.consul:8200/ 27 | vault-ctrl-tool --iam-auth-role=example --init --input-prefix=/etc/vault-ctrl-tool --output-prefix=/etc/vault-ctrl-tool 28 | ``` 29 | 30 | ### Sidecar 31 | 32 | You can either launch Vault Control Tool as a process, or call it from cron. The later is the recommended 33 | mechanism: 34 | 35 | ```bash 36 | export VAULT_ADDR=https://vault.service.consul:8200/ 37 | vault-ctrl-tool --sidecar --one-shot --input-prefix=/etc/vault-ctrl-tool --output-prefix=/etc/vault-ctrl-tool 38 | ``` 39 | 40 | ## EC2 41 | 42 | ### Setup 43 | 44 | Your AMI creation process will ultimately generate an AMI id for your disk image. This AMI needs to be 45 | registered with Vault and given specific policies. 46 | 47 | ```bash 48 | vault write auth/aws-ec2/ami-6a616d6573 bound_ami_id=ami-6a616d6573 policies=service-policy bound_subnet_id=subnet-.... 49 | ``` 50 | 51 | This creates an AWS role called `ami-6a616d6573` associated with the same AMI. 52 | 53 | Vault Control Tool will automatically use the current AMI as the role name when authenticating without any 54 | other configuration needed. 55 | 56 | ### Testing 57 | 58 | The first time an AMI authenticates a nonce is established. If you are wanting to test Vault Control Tool 59 | that is using a different mechanism already, you will need to collect its nonce to do testing (or reset 60 | the nonce for the instance in the whitelist). 61 | 62 | ### On Startup 63 | 64 | Assuming you have a `vault-config.yml` in `/etc/vault-ctrl-tool`, initialization is as easy as: 65 | 66 | ```bash 67 | export VAULT_ADDR=https://vault.service.consul:8200/ 68 | vault-ctrl-tool --ec2-auth --init --input-prefix=/etc/vault-ctrl-tool --output-prefix=/etc/vault-ctrl-tool 69 | ``` 70 | 71 | ### Sidecar 72 | 73 | You can either launch Vault Control Tool as a process, or call it from cron. The later is the recommended 74 | mechanism: 75 | 76 | ```bash 77 | export VAULT_ADDR=https://vault.service.consul:8200/ 78 | vault-ctrl-tool --sidecar --one-shot --input-prefix=/etc/vault-ctrl-tool --output-prefix=/etc/vault-ctrl-tool 79 | ``` 80 | -------------------------------------------------------------------------------- /docs/KUBERNETES.md: -------------------------------------------------------------------------------- 1 | # Using Kubernetes 2 | 3 | Vault Control Tool can easily be run as both an init container to populate various files, and a sidecar to 4 | keep vault tokens and short-lived credentials fresh while your service is running. You will need a Service Account, 5 | some mounts for the configuration and leases, configuration for the init and sidecar containers and then the actual 6 | vault configuration. 7 | 8 | ## Failure Handling 9 | 10 | By default, Vault tokens live for an absolute maximum of 31 days (this is tuned in each authentication backend), they 11 | cannot be renewed beyond this and are expired by the server. If this is your setup, Vault Control Tool will 12 | re-authenticate using the ServiceAccount (obtaining a new token), and re-fetch any AWS credentials, vault token files, 13 | SSH certificates, and secrets/templates with a "token" lifetime. 14 | 15 | This can also happen sooner if Vault Control Tool is unable to successfully contact the Vault server and renew the 16 | token before its TTL passes. 17 | 18 | If a token expires, credentials a service is using will cease to work. Because of the nature of Kubernetes pods, services 19 | can either manually catch this and reread configuration files, or can exit when this happens, allowing Kubernetes to 20 | restart their container and read in the new credentials. 21 | 22 | If you are using secrets or templates with "token" lifetimes, it is recommended to tune your Kubernetes authentication 23 | backend to never expire tokens, but instead issue them with a "period" that must be refreshed by the sidecar. 24 | 25 | ## Service Account 26 | 27 | Pods authenticate themselves to Vault by using their ServiceAccount Token. So you'll need a ServiceAccount mounted 28 | in your pod: 29 | 30 | ```yaml 31 | apiVersion: v1 32 | kind: ServiceAccount 33 | metadata: 34 | name: my-example 35 | namespace: default 36 | # If you have specific authentication required to access your Docker registry, you'll need to 37 | # include the imagePullSecrets too. 38 | imagePullSecrets: 39 | - name: docker-registry-auth 40 | --- 41 | # Allow my-service to use the TokenReview API 42 | apiVersion: rbac.authorization.k8s.io/v1beta1 43 | kind: ClusterRoleBinding 44 | metadata: 45 | name: my-example-tokenreview-binding 46 | namespace: default 47 | roleRef: 48 | apiGroup: rbac.authorization.k8s.io 49 | kind: ClusterRole 50 | name: system:auth-delegator 51 | subjects: 52 | - kind: ServiceAccount 53 | name: my-example 54 | namespace: default 55 | ``` 56 | 57 | ## Mounts 58 | 59 | This tool needs to read its configuration from one mount, and requires a place to writeout its lease information. The 60 | configuration is only needed for the init container. 61 | 62 | ```yaml 63 | ... 64 | spec: 65 | volumes: 66 | # This volume is shared between the vault-ctrl-tool and any container needing secrets 67 | - name: vault-secrets-volume 68 | emptyDir: {} 69 | # This volume is exclusive to the vault-ctrl-tool init and sidecar containers. 70 | - name: vault-leases-volume 71 | emptyDir: {} 72 | # This volume is exclusive to the vault-ctrl-tool init container. 73 | - name: vault-config-volume 74 | configMap: 75 | name: my-example-vault-configmap 76 | serviceAccount: my-example 77 | ``` 78 | 79 | ## Init Container 80 | ```yaml 81 | initContainers: 82 | - name: vault-init 83 | image: docker-registry.hootsuite.com/tools/vault-ctrl-tool:latest 84 | resources: 85 | limits: 86 | cpu: 1 87 | memory: 200Mi 88 | requests: 89 | cpu: 0.1 90 | memory: 64Mi 91 | env: 92 | # You can set VAULT_ADDR in different ways, or possibly have it setup as an external service. -- YMMV 93 | - name: VAULT_ADDR 94 | valueFrom: 95 | configMapKeyRef: 96 | name: vault 97 | key: vault_address 98 | optional: false 99 | # This is the path inside Vault where authentication requests will be sent. 100 | - name: K8S_LOGIN_PATH 101 | valueFrom: 102 | configMapKeyRef: 103 | name: vault 104 | key: vault_k8s_path 105 | optional: false 106 | command: 107 | - "/vault-ctrl-tool" 108 | - "--init" 109 | - "--k8s-auth-role" 110 | - "my-example" 111 | - "--input-prefix" 112 | - "/etc/vault-config" 113 | - "--output-prefix" 114 | - "/etc/secrets" 115 | volumeMounts: 116 | - name: vault-secrets-volume 117 | mountPath: "/etc/secrets" 118 | - name: vault-config-volume 119 | mountPath: "/etc/vault-config" 120 | - name: vault-leases-volume 121 | mountPath: "/tmp/vault-leases" 122 | 123 | ``` 124 | 125 | ## Sidecar Container 126 | 127 | ```yaml 128 | - name: vault-sidecar 129 | image: docker-registry.hootsuite.com/tools/vault-ctrl-tool:latest 130 | resources: 131 | limits: 132 | cpu: 1 133 | memory: 200Mi 134 | requests: 135 | cpu: 0.1 136 | memory: 64Mi 137 | env: 138 | # You can set VAULT_ADDR in different ways, or possibly have it setup as an external service. -- YMMV 139 | - name: VAULT_ADDR 140 | valueFrom: 141 | configMapKeyRef: 142 | name: vault 143 | key: vault_address 144 | optional: false 145 | command: 146 | - "/vault-ctrl-tool" 147 | - "--sidecar" 148 | - "--input-prefix" 149 | - "/etc/vault-config" 150 | - "--output-prefix" 151 | - "/etc/secrets" 152 | volumeMounts: 153 | - name: vault-secrets-volume 154 | mountPath: "/etc/secrets" 155 | - name: vault-config-volume 156 | mountPath: "/etc/vault-config" 157 | - name: vault-leases-volume 158 | mountPath: "/tmp/vault-leases" 159 | ``` 160 | -------------------------------------------------------------------------------- /docs/examples/absolute-ping.yml: -------------------------------------------------------------------------------- 1 | version: 3 2 | # This will insert an example secret into an existing template file. Note that the "path" of the 3 | # secret starts with "/" so it's looked up at its absolute path. 4 | 5 | # Useful if you already have a configuration file that just needs some secrets populated into it. 6 | 7 | secrets: 8 | - key: ping 9 | path: /secret/ping 10 | missingOk: false 11 | lifetime: static 12 | templates: 13 | - input: template-ping/example.tpl 14 | output: template-ping/absolute-example.txt 15 | mode: 0666 16 | lifetime: static 17 | 18 | 19 | # SETUP 20 | # 1. Create a secret: vault write secret/ping ping=pong 21 | # 2. Ensure VAULT_TOKEN and VAULT_ADDR are set. 22 | 23 | # RUNNING 24 | # vault-ctrl-tool --init --output-prefix=/tmp/v-c-t --config=template-ping.yml 25 | 26 | # OUTPUT 27 | # cat /tmp/v-c-t/template-ping/absolute-example.txt 28 | -------------------------------------------------------------------------------- /docs/examples/cleanup.txt: -------------------------------------------------------------------------------- 1 | After each example, there will be a file called "/tmp/vault-leases/vault-ctrl-tool.leases" created which tracked the 2 | work the tool did. 3 | 4 | You can delete the work the tool did by running: vault-ctrl-tool --cleanup 5 | 6 | -------------------------------------------------------------------------------- /docs/examples/force-refreshing-credentials.md: -------------------------------------------------------------------------------- 1 | Aside from allow credentials to be refreshed just prior to their expirty (i.e. within the renew-interval), you can configure a 2 | force refresh TTL which will make vault-ctrl-tool attempt to renew dynamically generated credentials (AWS STS) prior to their actual expiry. 3 | This is to allow sensitive workloads to have a buffer in case of Vault issues. For example a STS token generated with a TTL of 1hr (+ using a 10m renew interval) 4 | will be renewed at approx 50 minutes (as it would expire before the next interval). This means that if begins to Vault experiences issues at 49 minutes, there is only a 5 | ten minute window between that and when systems using vault-ctrl-tool may experience outages due to expired credentials. 6 | 7 | By generating credentials with a longer ttl and forcing them to be renewed prior to their expiry time, you can ensure that systems using vault-ctrl-tool have some buffer while issues are addressed. 8 | 9 | For example, by setting a force refresh time of 1hr, and an STS TTL of 3 hr you can ensure in the event of a Vault outage that systems can still run up to two hours - giving you time to deal issues 10 | without any disruption of service 11 | 12 | ```bash 13 | vault-ctrl-tool --sidecar --output-prefix=/tmp/v-c-t --force-refresh-ttl=60m --renew-interval=10m --sts-ttl=3h 14 | ``` -------------------------------------------------------------------------------- /docs/examples/refreshing-leases.txt: -------------------------------------------------------------------------------- 1 | To keep any leases fresh, you can run: 2 | 3 | vault-ctrl-tool --sidecar --output-prefix=/tmp/v-c-t --renew-interval=1s --config=simple.yml 4 | 5 | This will refresh once a second. The default is 9 minutes which is likely sufficient for you. 6 | 7 | -------------------------------------------------------------------------------- /docs/examples/relative-ping.yml: -------------------------------------------------------------------------------- 1 | version: 3 2 | # This will insert an example secret into an existing template file. 3 | # Note the path is relative, so v-c-t prefixes it automatically with /secret/application-config/services/secret/ 4 | # This can be overridden with --secret-prefix. 5 | 6 | # Useful if you already have a configuration file that just needs some secrets populated into it. 7 | 8 | secrets: 9 | - key: ping 10 | path: ping 11 | missingOk: false 12 | lifetime: static 13 | templates: 14 | - input: template-ping/relative-example.tpl 15 | output: template-ping/relative-example.txt 16 | mode: 0666 17 | lifetime: static 18 | 19 | 20 | # SETUP 21 | # 1. Create a secret: vault write secret/application-config/services/ping hello=world 22 | # 2. Ensure VAULT_TOKEN and VAULT_ADDR are set. 23 | 24 | # RUNNING 25 | # vault-ctrl-tool --init --output-prefix=/tmp/v-c-t --config=relative-ping.yml 26 | 27 | # OUTPUT 28 | # cat /tmp/v-c-t/template-ping/relative-example.txt -------------------------------------------------------------------------------- /docs/examples/simple.yml: -------------------------------------------------------------------------------- 1 | # This will write your vault token to /tmp/v-c-t/simple/token.txt - useful if you have systems that already know how 2 | # to access Vault if only they had a token. 3 | 4 | vaultToken: 5 | output: simple/token.txt 6 | mode: 0666 7 | 8 | # SETUP 9 | # 1. Ensure VAULT_TOKEN and VAULT_ADDR are set correctly. 10 | 11 | # RUNNING 12 | # vault-ctrl-tool --init --output-prefix=/tmp/v-c-t --config=simple.yml 13 | 14 | # OUTPUT 15 | # cat /tmp/v-c-t/simple/token.txt 16 | 17 | -------------------------------------------------------------------------------- /docs/examples/template-ping/example.tpl: -------------------------------------------------------------------------------- 1 | You say ping, then I say {{.ping_ping}}. 2 | -------------------------------------------------------------------------------- /docs/examples/template-ping/relative-example.tpl: -------------------------------------------------------------------------------- 1 | Hello, {{.ping_hello}}! 2 | -------------------------------------------------------------------------------- /e2e/e2e_fixtures.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/golang/mock/gomock" 7 | "github.com/hashicorp/vault/api" 8 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 9 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 10 | mtrics "github.com/hootsuite/vault-ctrl-tool/v2/metrics" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/syncer" 12 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 13 | mock_vaultclient "github.com/hootsuite/vault-ctrl-tool/v2/vaultclient/mocks" 14 | "github.com/rs/zerolog" 15 | zlog "github.com/rs/zerolog/log" 16 | "os" 17 | "path" 18 | "testing" 19 | ) 20 | 21 | func Secret(secretJSON string) *api.Secret { 22 | var secret api.Secret 23 | if err := json.Unmarshal([]byte(secretJSON), &secret); err != nil { 24 | panic(fmt.Sprintf("could not parse JSON. JSON=%q Error=%v", secretJSON, err)) 25 | } 26 | return &secret 27 | } 28 | 29 | // vaultTokenJSON has a valid token that is not going to expire any time soon. It includes an accessor, id, and username 30 | // that must not change. 31 | // language=JSON 32 | const vaultTokenJSON = `{ 33 | "request_id": "7dbcff81-3182-c523-8c50-3be49a578d25", 34 | "lease_id": "", 35 | "lease_duration": 0, 36 | "renewable": false, 37 | "data": { 38 | "accessor": "unit-test-accessor", 39 | "creation_time": 1604433628, 40 | "creation_ttl": 32400, 41 | "display_name": "unit-test-token", 42 | "entity_id": "07ef9c85-9f14-1272-eb56-92b7bfd21500", 43 | "expire_time": "2040-11-03T21:00:28.797810827-08:00", 44 | "explicit_max_ttl": 0, 45 | "id": "unit-test-token", 46 | "issue_time": "2020-11-03T12:00:28.797823501-08:00", 47 | "meta": { 48 | "username": "unit.tests" 49 | }, 50 | "num_uses": 0, 51 | "orphan": true, 52 | "path": "auth/fake", 53 | "policies": [ 54 | "default" 55 | ], 56 | "renewable": true, 57 | "ttl": 32387, 58 | "type": "service" 59 | }, 60 | "warnings": null 61 | }` 62 | 63 | // exampleSecretJSON is a non-destroyed secret with two bits of data, and a version of 3. 64 | // language=JSON 65 | const exampleSecretJSON = `{ 66 | "request_id": "8c472fc1-f389-d0c8-fec0-83d9a9930a40", 67 | "lease_id": "", 68 | "lease_duration": 0, 69 | "renewable": false, 70 | "data": { 71 | "data": { 72 | "bar": "bbbb", 73 | "foo": "aaaa" 74 | }, 75 | "metadata": { 76 | "created_time": "2019-10-02T22:42:10.724886003Z", 77 | "deletion_time": "", 78 | "destroyed": false, 79 | "version": 3 80 | } 81 | }, 82 | "warnings": null 83 | }` 84 | 85 | // exampleSecretFreshV4JSON is just like exampleSecretV4JSON, except the created_time is a specific 86 | // value to test against. 87 | // language=JSON 88 | const exampleSecretFreshV4JSON = `{ 89 | "request_id": "8c472fc1-f389-d0c8-fec0-83d9a9930a40", 90 | "lease_id": "", 91 | "lease_duration": 0, 92 | "renewable": false, 93 | "data": { 94 | "data": { 95 | "bar": "bbbb2", 96 | "foo": "aaaa2" 97 | }, 98 | "metadata": { 99 | "created_time": "2019-10-02T22:52:10.724886003Z", 100 | "deletion_time": "", 101 | "destroyed": false, 102 | "version": 4 103 | } 104 | }, 105 | "warnings": null 106 | }` 107 | 108 | // exampleSecretV4JSON is just like exampleSecretJSON except the version has been incremented 109 | // and the values of the secrets are different. The created_time is 31s later than exampleSecretJSON. 110 | // language=JSON 111 | const exampleSecretV4JSON = `{ 112 | "request_id": "8c472fc1-f389-d0c8-fec0-83d9a9930a40", 113 | "lease_id": "", 114 | "lease_duration": 0, 115 | "renewable": false, 116 | "data": { 117 | "data": { 118 | "bar": "bbbb2", 119 | "foo": "aaaa2" 120 | }, 121 | "metadata": { 122 | "created_time": "2019-10-02T22:42:41.724886003Z", 123 | "deletion_time": "", 124 | "destroyed": false, 125 | "version": 4 126 | } 127 | }, 128 | "warnings": null 129 | }` 130 | 131 | // exampleBase64SecretJSON returns a secret with a field "foo64" whose value was written into 132 | // vaule already base64 encoded. 133 | const exampleBase64SecretJSON = `{ 134 | "request_id": "8c472fc1-f389-d0c8-fec0-83d9a9930a40", 135 | "lease_id": "", 136 | "lease_duration": 0, 137 | "renewable": false, 138 | "data": { 139 | "data": { 140 | "foo64": "SGVsbG8gSG9vdHN1aXRl" 141 | }, 142 | "metadata": { 143 | "created_time": "2019-10-02T22:42:41.724886003Z", 144 | "deletion_time": "", 145 | "destroyed": false, 146 | "version": 4 147 | } 148 | }, 149 | "warnings": null 150 | }` 151 | 152 | type SyncFixture struct { 153 | log zerolog.Logger 154 | workDir string 155 | ctrl *gomock.Controller 156 | vaultClient *mock_vaultclient.MockVaultClient 157 | cfg *config.ControlToolConfig 158 | metrics *mtrics.Metrics 159 | cliFlags *util.CliFlags 160 | bcase *briefcase.Briefcase 161 | syncer *syncer.Syncer 162 | } 163 | 164 | func setupSync(t *testing.T, configBody string, cliArgs []string) *SyncFixture { 165 | return setupSyncWithDir(t, configBody, cliArgs, t.TempDir()) 166 | } 167 | 168 | // setupSyncWithDir lets you share the briefcase from one run with another run, 169 | // good for doing an --init run, then a --sidecar --one-shot run. 170 | func setupSyncWithDir(t *testing.T, configBody string, cliArgs []string, workDir string) *SyncFixture { 171 | 172 | ctrl := gomock.NewController(t) 173 | 174 | log := zlog.Output(zerolog.ConsoleWriter{Out: os.Stdout}).Level(zerolog.DebugLevel) 175 | 176 | cfg, err := config.ReadConfig(log, []byte(configBody), workDir, workDir) 177 | 178 | if err != nil { 179 | t.Fatal(err) 180 | } 181 | 182 | vaultClient := mock_vaultclient.NewMockVaultClient(ctrl) 183 | vaultClient.EXPECT().Address().Return("unit-tests").AnyTimes() 184 | 185 | metrics := mtrics.NewMetrics() 186 | 187 | var bcase *briefcase.Briefcase 188 | 189 | bcase, err = briefcase.LoadBriefcase(path.Join(workDir, "briefcase"), metrics) 190 | if err != nil { 191 | bcase = briefcase.NewBriefcase(metrics) 192 | } 193 | 194 | var args []string 195 | 196 | args = append(args, cliArgs...) 197 | args = append(args, "--output-prefix", workDir, 198 | "--input-prefix", workDir, 199 | "--leases-file", path.Join(workDir, "briefcase")) 200 | 201 | cliFlags, err := util.ProcessFlags(args) 202 | 203 | if err != nil { 204 | t.Fatal(err) 205 | } 206 | 207 | s := syncer.NewSyncer(log, cfg, vaultClient, bcase, metrics) 208 | 209 | return &SyncFixture{ 210 | log: log, 211 | workDir: workDir, 212 | ctrl: ctrl, 213 | vaultClient: vaultClient, 214 | cfg: cfg, 215 | metrics: metrics, 216 | cliFlags: cliFlags, 217 | bcase: bcase, 218 | syncer: s, 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /e2e/e2e_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "testing" 10 | "time" 11 | 12 | "github.com/golang/mock/gomock" 13 | "github.com/hashicorp/vault/api" 14 | mtrics "github.com/hootsuite/vault-ctrl-tool/v2/metrics" 15 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 16 | "github.com/stretchr/testify/assert" 17 | testing2 "k8s.io/utils/clock/testing" 18 | ) 19 | 20 | // TestSyncWithPinnedVersion ensures that when requesting a specific version of a secret in a config file cascades 21 | // that request to Vault. 22 | func TestSyncWithPinnedVersion(t *testing.T) { 23 | 24 | fixture := setupSync(t, ` 25 | --- 26 | version: 3 27 | secrets: 28 | - key: example 29 | path: path/in/vault 30 | missingOk: false 31 | mode: 0700 32 | pinnedVersion: 3 33 | output: example-output 34 | lifetime: static 35 | `, []string{ 36 | "--init", 37 | "--vault-token", "unit-test-token"}) 38 | 39 | vaultToken := Secret(vaultTokenJSON) 40 | fixture.vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).Return(vaultToken, nil).AnyTimes() 41 | fixture.vaultClient.EXPECT().ServiceSecretPrefix(gomock.Any()).Return("/prefix/") 42 | fixture.vaultClient.EXPECT().SetToken(gomock.Any()).AnyTimes() 43 | 44 | fixture.vaultClient.EXPECT().ReadWithData(gomock.Any(), gomock.Any()).DoAndReturn( 45 | func(path string, data map[string][]string) (*api.Secret, error) { 46 | 47 | // Expect a request for the absolute secret path of version 3. 48 | assert.Equal(t, "/prefix/path/in/vault", path) 49 | assert.Len(t, data, 1) 50 | assert.Equal(t, []string{"3"}, data["version"]) 51 | response := Secret(exampleSecretJSON) 52 | return response, nil 53 | }).Times(1) 54 | 55 | fakeClock := testing2.NewFakeClock(time.Now()) 56 | ctx := clock.Set(context.Background(), fakeClock) 57 | 58 | vtoken, err := fixture.syncer.GetVaultToken(ctx, *fixture.cliFlags) 59 | assert.NoError(t, err) 60 | err = fixture.syncer.PerformSync(ctx, vtoken, fakeClock.Now().AddDate(1, 0, 0), *fixture.cliFlags) 61 | assert.NoError(t, err) 62 | assert.FileExists(t, path.Join(fixture.workDir, "example-output")) 63 | assert.Equal(t, 1, fixture.metrics.Counter(mtrics.SecretUpdates)) 64 | assert.Equal(t, 0, fixture.metrics.Counter(mtrics.VaultTokenWritten)) 65 | } 66 | 67 | // TestSyncVersionScope - when a KVv2 secret gets a new version and it is at least 30 seconds old, the 68 | // field associated with the secret must be updated. 69 | func TestSyncVersionScope(t *testing.T) { 70 | 71 | const configBody = `--- 72 | version: 3 73 | secrets: 74 | - key: example 75 | path: path/in/vault 76 | missingOk: false 77 | mode: 0700 78 | lifetime: version 79 | fields: 80 | - name: foo 81 | output: foo 82 | ` 83 | 84 | sharedDir := t.TempDir() 85 | 86 | fixture1 := setupSyncWithDir(t, configBody, []string{"--init", 87 | "--vault-token", "unit-test-token"}, sharedDir) 88 | 89 | vaultToken := Secret(vaultTokenJSON) 90 | fixture1.vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).Return(vaultToken, nil).AnyTimes() 91 | fixture1.vaultClient.EXPECT().ServiceSecretPrefix(gomock.Any()).Return("/prefix/") 92 | fixture1.vaultClient.EXPECT().SetToken(gomock.Any()).AnyTimes() 93 | 94 | fixture1.vaultClient.EXPECT().Read(gomock.Any()).DoAndReturn( 95 | func(path string) (*api.Secret, error) { 96 | assert.Equal(t, "/prefix/path/in/vault", path) 97 | response := Secret(exampleSecretJSON) 98 | return response, nil 99 | }).Times(1) 100 | 101 | fakeClock := testing2.NewFakeClock(time.Now()) 102 | ctx := clock.Set(context.Background(), fakeClock) 103 | 104 | vtoken, err := fixture1.syncer.GetVaultToken(ctx, *fixture1.cliFlags) 105 | assert.NoError(t, err) 106 | err = fixture1.syncer.PerformSync(ctx, vtoken, fakeClock.Now().AddDate(1, 0, 0), *fixture1.cliFlags) 107 | 108 | assert.NoError(t, err) 109 | assert.FileExists(t, path.Join(fixture1.workDir, "foo")) 110 | 111 | foobytes, _ := ioutil.ReadFile(path.Join(fixture1.workDir, "foo")) 112 | assert.Equal(t, "aaaa", string(foobytes)) 113 | 114 | assert.Equal(t, 1, fixture1.metrics.Counter(mtrics.SecretUpdates)) 115 | assert.Equal(t, 0, fixture1.metrics.Counter(mtrics.VaultTokenWritten)) 116 | 117 | // Now, do this again, except with a new version of the secret 118 | 119 | fixture2 := setupSyncWithDir(t, configBody, []string{"--sidecar", "--one-shot", "--vault-token", "unit-test-token"}, sharedDir) 120 | 121 | fixture2.vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).Return(vaultToken, nil).AnyTimes() 122 | fixture2.vaultClient.EXPECT().ServiceSecretPrefix(gomock.Any()).Return("/prefix/") 123 | fixture2.vaultClient.EXPECT().SetToken(gomock.Any()).AnyTimes() 124 | 125 | fixture2.vaultClient.EXPECT().Read(gomock.Any()).DoAndReturn( 126 | func(path string) (*api.Secret, error) { 127 | assert.Equal(t, "/prefix/path/in/vault", path) 128 | // return "v4" of the secret 129 | response := Secret(exampleSecretV4JSON) 130 | return response, nil 131 | }).Times(1) 132 | 133 | vtoken, err = fixture2.syncer.GetVaultToken(ctx, *fixture2.cliFlags) 134 | assert.NoError(t, err) 135 | err = fixture2.syncer.PerformSync(ctx, vtoken, fakeClock.Now().AddDate(1, 0, 0), *fixture1.cliFlags) 136 | 137 | assert.NoError(t, err) 138 | assert.FileExists(t, path.Join(fixture2.workDir, "foo")) 139 | 140 | foobytes, _ = ioutil.ReadFile(path.Join(fixture2.workDir, "foo")) 141 | 142 | // Since the secret is quite old, expect the field to be updated. 143 | assert.Equal(t, "aaaa2", string(foobytes)) 144 | assert.Equal(t, 1, fixture1.metrics.Counter(mtrics.SecretUpdates)) 145 | assert.Equal(t, 0, fixture1.metrics.Counter(mtrics.VaultTokenWritten)) 146 | 147 | } 148 | 149 | // TestSyncVersionScope - when a KVv2 secret gets a new version and it is not 30 seconds old, nothing 150 | // should be updated. 151 | func TestSyncVersionScopeWithFreshSecret(t *testing.T) { 152 | 153 | const configBody = `--- 154 | version: 3 155 | secrets: 156 | - key: example 157 | path: path/in/vault 158 | missingOk: false 159 | mode: 0700 160 | lifetime: version 161 | touchfile: test-touchfile 162 | fields: 163 | - name: foo 164 | output: foo 165 | ` 166 | 167 | // Step 1: There is nothing in the briefcase, so the field will be written. 168 | sharedDir := t.TempDir() 169 | 170 | fixture1 := setupSyncWithDir(t, configBody, []string{"--init", 171 | "--vault-token", "unit-test-token"}, sharedDir) 172 | 173 | vaultToken := Secret(vaultTokenJSON) 174 | fixture1.vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).Return(vaultToken, nil).AnyTimes() 175 | fixture1.vaultClient.EXPECT().ServiceSecretPrefix(gomock.Any()).Return("/prefix/") 176 | fixture1.vaultClient.EXPECT().SetToken(gomock.Any()).AnyTimes() 177 | 178 | fixture1.vaultClient.EXPECT().Read(gomock.Any()).DoAndReturn( 179 | func(path string) (*api.Secret, error) { 180 | assert.Equal(t, "/prefix/path/in/vault", path) 181 | response := Secret(exampleSecretJSON) 182 | return response, nil 183 | }).Times(1) 184 | 185 | // This is 10 seconds after the time in exampleSecretFreshV4JSON 186 | fakeClock := testing2.NewFakeClock(time.Date(2019, 10, 2, 22, 52, 20, 0, time.UTC)) 187 | 188 | ctx := clock.Set(context.Background(), fakeClock) 189 | vtoken, err := fixture1.syncer.GetVaultToken(ctx, *fixture1.cliFlags) 190 | assert.NoError(t, err) 191 | err = fixture1.syncer.PerformSync(ctx, vtoken, fakeClock.Now().AddDate(1, 0, 0), *fixture1.cliFlags) 192 | 193 | assert.NoError(t, err) 194 | assert.FileExists(t, path.Join(fixture1.workDir, "foo")) 195 | 196 | foobytes, _ := ioutil.ReadFile(path.Join(fixture1.workDir, "foo")) 197 | assert.Equal(t, "aaaa", string(foobytes)) 198 | 199 | assert.Equal(t, 1, fixture1.metrics.Counter(mtrics.SecretUpdates)) 200 | assert.Equal(t, 0, fixture1.metrics.Counter(mtrics.VaultTokenWritten)) 201 | 202 | // Expect the "touchfile" to exist since the fields were written. 203 | assert.FileExists(t, path.Join(fixture1.workDir, "test-touchfile")) 204 | assert.NoError(t, os.Remove(path.Join(fixture1.workDir, "test-touchfile"))) 205 | 206 | // Now, do this again, except with a new version of the secret in Vault 207 | 208 | fixture2 := setupSyncWithDir(t, configBody, []string{"--sidecar", "--one-shot", "--vault-token", "unit-test-token"}, sharedDir) 209 | 210 | fixture2.vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).Return(vaultToken, nil).AnyTimes() 211 | fixture2.vaultClient.EXPECT().ServiceSecretPrefix(gomock.Any()).Return("/prefix/") 212 | fixture2.vaultClient.EXPECT().SetToken(gomock.Any()).AnyTimes() 213 | 214 | fixture2.vaultClient.EXPECT().Read(gomock.Any()).DoAndReturn( 215 | func(path string) (*api.Secret, error) { 216 | assert.Equal(t, "/prefix/path/in/vault", path) 217 | // return "v4" of the secret, but with a created_timestamp that isn't old enough. 218 | response := Secret(exampleSecretFreshV4JSON) 219 | return response, nil 220 | }).Times(1) 221 | 222 | vtoken, err = fixture2.syncer.GetVaultToken(ctx, *fixture2.cliFlags) 223 | assert.NoError(t, err) 224 | err = fixture2.syncer.PerformSync(ctx, vtoken, fakeClock.Now().AddDate(1, 0, 0), *fixture1.cliFlags) 225 | 226 | assert.NoError(t, err) 227 | 228 | // Expect the touchfile to _not_ exist, since the fields were not updated. 229 | assert.NoFileExists(t, path.Join(fixture2.workDir, "test-touchfile")) 230 | 231 | assert.FileExists(t, path.Join(fixture2.workDir, "foo")) 232 | 233 | foobytes, _ = ioutil.ReadFile(path.Join(fixture2.workDir, "foo")) 234 | assert.Equal(t, "aaaa", string(foobytes)) 235 | 236 | assert.Equal(t, 0, fixture2.metrics.Counter(mtrics.SecretUpdates)) 237 | assert.Equal(t, 0, fixture2.metrics.Counter(mtrics.VaultTokenWritten)) 238 | } 239 | 240 | // TestSyncWithEmptyConfig ensures that when a configuration file is empty, the service still runs, but doesn't 241 | // actually do anything. 242 | func TestSyncWithEmptyConfig(t *testing.T) { 243 | 244 | fixture := setupSync(t, ` 245 | --- 246 | version: 3 247 | `, []string{"--vault-token", "unit-test-token", 248 | "--init"}) 249 | 250 | fixture.vaultClient.EXPECT().Address().Return("unit-tests").AnyTimes() 251 | 252 | var secret api.Secret 253 | if err := json.Unmarshal([]byte(vaultTokenJSON), &secret); err != nil { 254 | t.Fatal(err) 255 | } 256 | 257 | fixture.vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).Return(&secret, nil).AnyTimes() 258 | fixture.vaultClient.EXPECT().SetToken(gomock.Any()).AnyTimes() 259 | 260 | fakeClock := testing2.NewFakeClock(time.Now()) 261 | ctx := clock.Set(context.Background(), fakeClock) 262 | vtoken, err := fixture.syncer.GetVaultToken(ctx, *fixture.cliFlags) 263 | assert.NoError(t, err) 264 | err = fixture.syncer.PerformSync(ctx, vtoken, fakeClock.Now().AddDate(1, 0, 0), *fixture.cliFlags) 265 | 266 | assert.NoError(t, err) 267 | assert.Equal(t, 1, fixture.metrics.Counter(mtrics.BriefcaseReset)) 268 | assert.Equal(t, 0, fixture.metrics.Counter(mtrics.VaultTokenWritten)) 269 | assert.Equal(t, 0, fixture.metrics.Counter(mtrics.VaultTokenRefreshed)) 270 | assert.Equal(t, 0, fixture.metrics.Counter(mtrics.SecretUpdates)) 271 | } 272 | 273 | // TestBase64Field 274 | func TestBase64Field(t *testing.T) { 275 | 276 | fixture := setupSync(t, ` 277 | --- 278 | version: 3 279 | secrets: 280 | - key: example 281 | path: path/in/vault 282 | missingOk: false 283 | mode: 0700 284 | lifetime: static 285 | fields: 286 | - name: foo64 287 | output: foo-output.txt 288 | encoding: base64 289 | `, []string{"--vault-token", "unit-test-token", 290 | "--init"}) 291 | 292 | fixture.vaultClient.EXPECT().Address().Return("unit-tests").AnyTimes() 293 | 294 | var secret api.Secret 295 | if err := json.Unmarshal([]byte(vaultTokenJSON), &secret); err != nil { 296 | t.Fatal(err) 297 | } 298 | 299 | fixture.vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).Return(&secret, nil).AnyTimes() 300 | fixture.vaultClient.EXPECT().ServiceSecretPrefix(gomock.Any()).Return("/prefix/") 301 | fixture.vaultClient.EXPECT().SetToken(gomock.Any()).AnyTimes() 302 | 303 | fixture.vaultClient.EXPECT().Read(gomock.Any()).DoAndReturn( 304 | func(path string) (*api.Secret, error) { 305 | assert.Equal(t, "/prefix/path/in/vault", path) 306 | response := Secret(exampleBase64SecretJSON) 307 | return response, nil 308 | }).Times(1) 309 | 310 | fakeClock := testing2.NewFakeClock(time.Now()) 311 | ctx := clock.Set(context.Background(), fakeClock) 312 | vtoken, err := fixture.syncer.GetVaultToken(ctx, *fixture.cliFlags) 313 | assert.NoError(t, err) 314 | err = fixture.syncer.PerformSync(ctx, vtoken, fakeClock.Now().AddDate(1, 0, 0), *fixture.cliFlags) 315 | 316 | assert.NoError(t, err) 317 | 318 | outputFile := path.Join(fixture.workDir, "foo-output.txt") 319 | assert.FileExists(t, outputFile) 320 | 321 | foo64Bytes, err := ioutil.ReadFile(outputFile) 322 | assert.Equal(t, "Hello Hootsuite", string(foo64Bytes)) 323 | 324 | assert.Equal(t, 1, fixture.metrics.Counter(mtrics.BriefcaseReset)) 325 | assert.Equal(t, 1, fixture.metrics.Counter(mtrics.SecretUpdates)) 326 | assert.Equal(t, 0, fixture.metrics.Counter(mtrics.VaultTokenWritten)) 327 | assert.Equal(t, 0, fixture.metrics.Counter(mtrics.VaultTokenRefreshed)) 328 | } 329 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hootsuite/vault-ctrl-tool/v2 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.42.44 7 | github.com/golang/mock v1.5.0 8 | github.com/hashicorp/vault/api v1.3.1 9 | github.com/prometheus/client_golang v1.12.1 10 | github.com/rs/zerolog v1.26.1 11 | github.com/stretchr/testify v1.7.0 12 | golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 13 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 14 | gopkg.in/yaml.v2 v2.4.0 15 | k8s.io/api v0.23.3 16 | k8s.io/apimachinery v0.23.3 17 | k8s.io/client-go v0.23.3 18 | k8s.io/utils v0.0.0-20220127004650-9b3446523e65 19 | ) 20 | 21 | require ( 22 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect 23 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 24 | github.com/armon/go-metrics v0.3.10 // indirect 25 | github.com/armon/go-radix v1.0.0 // indirect 26 | github.com/beorn7/perks v1.0.1 // indirect 27 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 28 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 29 | github.com/davecgh/go-spew v1.1.1 // indirect 30 | github.com/fatih/color v1.13.0 // indirect 31 | github.com/go-logr/logr v1.2.2 // indirect 32 | github.com/gogo/protobuf v1.3.2 // indirect 33 | github.com/golang/protobuf v1.5.2 // indirect 34 | github.com/golang/snappy v0.0.4 // indirect 35 | github.com/google/go-cmp v0.5.7 // indirect 36 | github.com/google/gofuzz v1.2.0 // indirect 37 | github.com/googleapis/gnostic v0.5.5 // indirect 38 | github.com/hashicorp/errwrap v1.1.0 // indirect 39 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 40 | github.com/hashicorp/go-hclog v1.1.0 // indirect 41 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 42 | github.com/hashicorp/go-multierror v1.1.1 // indirect 43 | github.com/hashicorp/go-plugin v1.4.3 // indirect 44 | github.com/hashicorp/go-retryablehttp v0.7.0 // indirect 45 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 46 | github.com/hashicorp/go-secure-stdlib/mlock v0.1.2 // indirect 47 | github.com/hashicorp/go-secure-stdlib/parseutil v0.1.2 // indirect 48 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 49 | github.com/hashicorp/go-sockaddr v1.0.2 // indirect 50 | github.com/hashicorp/go-uuid v1.0.2 // indirect 51 | github.com/hashicorp/go-version v1.4.0 // indirect 52 | github.com/hashicorp/golang-lru v0.5.4 // indirect 53 | github.com/hashicorp/hcl v1.0.0 // indirect 54 | github.com/hashicorp/vault/sdk v0.3.0 // indirect 55 | github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect 56 | github.com/jmespath/go-jmespath v0.4.0 // indirect 57 | github.com/json-iterator/go v1.1.12 // indirect 58 | github.com/mattn/go-colorable v0.1.12 // indirect 59 | github.com/mattn/go-isatty v0.0.14 // indirect 60 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 61 | github.com/mitchellh/copystructure v1.2.0 // indirect 62 | github.com/mitchellh/go-homedir v1.1.0 // indirect 63 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 64 | github.com/mitchellh/mapstructure v1.4.3 // indirect 65 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 66 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 67 | github.com/modern-go/reflect2 v1.0.2 // indirect 68 | github.com/oklog/run v1.1.0 // indirect 69 | github.com/pierrec/lz4 v2.6.1+incompatible // indirect 70 | github.com/pmezard/go-difflib v1.0.0 // indirect 71 | github.com/prometheus/client_model v0.2.0 // indirect 72 | github.com/prometheus/common v0.32.1 // indirect 73 | github.com/prometheus/procfs v0.7.3 // indirect 74 | github.com/ryanuber/go-glob v1.0.0 // indirect 75 | go.uber.org/atomic v1.9.0 // indirect 76 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect 77 | golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect 78 | golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect 79 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect 80 | golang.org/x/text v0.3.7 // indirect 81 | golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect 82 | google.golang.org/appengine v1.6.7 // indirect 83 | google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 // indirect 84 | google.golang.org/grpc v1.44.0 // indirect 85 | google.golang.org/protobuf v1.27.1 // indirect 86 | gopkg.in/inf.v0 v0.9.1 // indirect 87 | gopkg.in/square/go-jose.v2 v2.6.0 // indirect 88 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 89 | k8s.io/klog/v2 v2.40.1 // indirect 90 | k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf // indirect 91 | sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect 92 | sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect 93 | sigs.k8s.io/yaml v1.3.0 // indirect 94 | ) 95 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 9 | "github.com/rs/zerolog" 10 | "github.com/rs/zerolog/log" 11 | "golang.org/x/crypto/ssh/terminal" 12 | ) 13 | 14 | var ( 15 | buildVersion string 16 | commitVersion string 17 | ) 18 | 19 | func setupLogging(debug bool) { 20 | log.Logger = log.With().Caller().Logger() 21 | 22 | if debug { 23 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 24 | log.Debug().Msg("debug logging enabled") 25 | } else { 26 | zerolog.SetGlobalLevel(zerolog.InfoLevel) 27 | } 28 | 29 | if terminal.IsTerminal(int(os.Stdout.Fd())) { 30 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout}) 31 | } 32 | } 33 | 34 | func main() { 35 | flags, err := util.ProcessFlags(os.Args[1:]) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | setupLogging(flags.DebugLogLevel) 41 | 42 | log.Debug().Interface("flags", flags).Msg("cli flags") 43 | 44 | switch flags.RunMode() { 45 | case util.ModeShowVersion: 46 | fmt.Printf("Version: %s\n", buildVersion) 47 | fmt.Printf("Commit: %s\n", commitVersion) 48 | case util.ModeInit: 49 | if err := PerformInit(context.Background(), *flags); err != nil { 50 | panic(err) 51 | } 52 | case util.ModeSidecar: 53 | if err := PerformSidecar(context.Background(), *flags); err != nil { 54 | panic(err) 55 | } 56 | case util.ModeOneShotSidecar: 57 | if err := PerformOneShotSidecar(context.Background(), *flags); err != nil { 58 | panic(err) 59 | } 60 | case util.ModeCleanup: 61 | if err := PerformCleanup(*flags); err != nil { 62 | fmt.Printf("Cleanup failed: %s\n", err) 63 | } 64 | case util.ModeUnknown: 65 | panic("unknown run mode") 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | stdlog "log" 6 | "net/http" 7 | "os" 8 | "sync" 9 | 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | type MetricName string 16 | 17 | const BriefcaseReset MetricName = "BriefcaseReset" 18 | const VaultTokenWritten MetricName = "VaultTokenWritten" 19 | const VaultTokenRefreshed MetricName = "VaultTokenRefreshed" 20 | const SecretUpdates MetricName = "SecretUpdates" 21 | 22 | type Metrics struct { 23 | mutex sync.RWMutex 24 | counters map[MetricName]int 25 | 26 | SidecarSyncErrors prometheus.Counter 27 | SidecarVaultTokenErrors prometheus.Counter 28 | SidecarSecretErrors prometheus.Counter 29 | } 30 | 31 | func metricName(name string) string { 32 | return fmt.Sprintf("vault_ctrl_tool_%s", name) 33 | } 34 | 35 | var ( 36 | // SidecarSyncErrors is incremented each time any part of the sidecar sync loop fails. 37 | SidecarSyncErrors = prometheus.NewCounter(prometheus.CounterOpts{ 38 | Name: metricName("sidecar_sync_errors"), 39 | Help: "errors while any stage of sidecar sync mode", 40 | }) 41 | // SidecarVaultTokenErrors is incremented each time the sidecar sync loop either fails to 42 | // find a vault token, or there is an error validating it. 43 | SidecarVaultTokenErrors = prometheus.NewCounter(prometheus.CounterOpts{ 44 | Name: metricName("sidecar_vault_token_errors"), 45 | Help: "errors while fetching and validating vault token stage of sidecar sync", 46 | }) 47 | // SidecarSecretErrors is incremented each time the sidecar sync loop fails to sync any of 48 | // its secrets (i.e. secret, aws, ssh, etc...). 49 | SidecarSecretErrors = prometheus.NewCounter(prometheus.CounterOpts{ 50 | Name: metricName("sidecar_secret_errors"), 51 | Help: "errors while renewing secrets", 52 | }) 53 | ) 54 | 55 | func init() { 56 | prometheus.MustRegister( 57 | SidecarSecretErrors, 58 | SidecarVaultTokenErrors, 59 | SidecarSyncErrors, 60 | ) 61 | } 62 | 63 | // NewMetrics constructs a new metrics object. 64 | func NewMetrics() *Metrics { 65 | mtrcs := &Metrics{ 66 | counters: make(map[MetricName]int), 67 | SidecarSyncErrors: SidecarSyncErrors, 68 | SidecarVaultTokenErrors: SidecarVaultTokenErrors, 69 | SidecarSecretErrors: SidecarSecretErrors, 70 | } 71 | 72 | return mtrcs 73 | } 74 | 75 | func (m *Metrics) Increment(name MetricName) { 76 | if m == nil { 77 | return 78 | } 79 | m.mutex.Lock() 80 | defer m.mutex.Unlock() 81 | m.counters[name]++ 82 | } 83 | 84 | func (m *Metrics) Decrement(name MetricName) { 85 | if m == nil { 86 | return 87 | } 88 | m.mutex.Lock() 89 | defer m.mutex.Unlock() 90 | m.counters[name]-- 91 | } 92 | 93 | func (m *Metrics) Counter(name MetricName) int { 94 | if m == nil { 95 | return 0 96 | } 97 | 98 | m.mutex.RLock() 99 | defer m.mutex.RUnlock() 100 | val := m.counters[name] 101 | return val 102 | } 103 | 104 | func (m *Metrics) IncrementBy(name MetricName, val int) { 105 | if m == nil { 106 | return 107 | } 108 | m.mutex.Lock() 109 | defer m.mutex.Unlock() 110 | m.counters[name] += val 111 | } 112 | 113 | // MetricsHandler instruments a prometheus metrics handler on "/metrics" and begins 114 | // listening on the specified address. 115 | func MetricsHandler(addr string, term chan os.Signal) { 116 | log.Info().Str("addr", addr).Msg("starting metrics server") 117 | 118 | go func() { 119 | mux := http.NewServeMux() 120 | mux.Handle("/metrics", promhttp.Handler()) 121 | 122 | srv := &http.Server{ 123 | Handler: mux, 124 | Addr: addr, 125 | ErrorLog: stdlog.Default(), 126 | } 127 | 128 | defer srv.Close() 129 | if err := srv.ListenAndServe(); err != nil { 130 | log.Error().Err(err).Msg("failed to start metrics server, shutting down") 131 | term <- os.Interrupt 132 | } 133 | }() 134 | } 135 | -------------------------------------------------------------------------------- /perform.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | "time" 10 | 11 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 12 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 13 | "github.com/hootsuite/vault-ctrl-tool/v2/metrics" 14 | "github.com/hootsuite/vault-ctrl-tool/v2/syncer" 15 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 16 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 17 | "github.com/hootsuite/vault-ctrl-tool/v2/vaultclient" 18 | zlog "github.com/rs/zerolog/log" 19 | ) 20 | 21 | const ShutdownFileCheckFrequency = 18 * time.Second 22 | 23 | func PerformOneShotSidecar(ctx context.Context, flags util.CliFlags) error { 24 | 25 | mtrics := metrics.NewMetrics() 26 | lockHandle, err := util.LockFile(flags.BriefcaseFilename + ".lck") 27 | if err != nil { 28 | zlog.Error().Err(err).Msg("could not create exclusive flock") 29 | return err 30 | } 31 | defer lockHandle.Unlock(false) 32 | 33 | zlog.Debug().Str("briefcase", flags.BriefcaseFilename).Str("buildVersion", buildVersion).Msg("starting oneshot") 34 | bc, err := briefcase.LoadBriefcase(flags.BriefcaseFilename, mtrics) 35 | if err != nil { 36 | zlog.Warn().Str("briefcase", flags.BriefcaseFilename).Err(err).Msg("could not load briefcase - starting an empty one") 37 | bc = briefcase.NewBriefcase(mtrics) 38 | } 39 | 40 | sync, err := syncer.SetupSyncer(flags, bc, mtrics) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | vaultToken, err := sync.GetVaultToken(ctx, flags) 46 | if err != nil { 47 | return fmt.Errorf("failed to get vault token: %w", err) 48 | } 49 | return sync.PerformSync(ctx, vaultToken, clock.Now(ctx).Add(flags.RenewInterval*2), flags) 50 | } 51 | 52 | func PerformInit(ctx context.Context, flags util.CliFlags) error { 53 | 54 | zlog.Info().Str("buildVersion", buildVersion).Msg("starting") 55 | mtrics := metrics.NewMetrics() 56 | 57 | lockHandle, err := util.LockFile(flags.BriefcaseFilename + ".lck") 58 | if err != nil { 59 | zlog.Error().Err(err).Msg("could not create exclusive flock") 60 | return err 61 | } 62 | defer lockHandle.Unlock(false) 63 | 64 | if stat, err := os.Stat(flags.BriefcaseFilename); err == nil && stat != nil { 65 | zlog.Warn().Str("filename", flags.BriefcaseFilename).Msg("running in init mode, but briefcase file already exists") 66 | if flags.AuthMechanism() == util.KubernetesAuth { 67 | zlog.Warn().Msg("running in kuberenetes - performing oneshot sidecar instead of init") 68 | _ = lockHandle.Unlock(true) 69 | return PerformOneShotSidecar(ctx, flags) 70 | } 71 | } 72 | 73 | sync, err := syncer.SetupSyncer(flags, briefcase.NewBriefcase(mtrics), mtrics) 74 | 75 | if err != nil { 76 | return fmt.Errorf("failed to setup syncer: %w", err) 77 | } 78 | 79 | vaultToken, err := sync.GetVaultToken(ctx, flags) 80 | if err != nil { 81 | return fmt.Errorf("failed to get vault token: %w", err) 82 | } 83 | return sync.PerformSync(ctx, vaultToken, clock.Now(ctx).Add(24*time.Hour), flags) 84 | } 85 | 86 | func sidecarSync(ctx context.Context, mtrcs *metrics.Metrics, flags util.CliFlags) error { 87 | lockHandle, err := util.LockFile(flags.BriefcaseFilename + ".lck") 88 | if err != nil { 89 | return fmt.Errorf("could not create exclusive flock: %w", err) 90 | } 91 | defer lockHandle.Unlock(true) 92 | 93 | sync, err := makeSyncer(flags, mtrcs) 94 | 95 | if err != nil { 96 | return fmt.Errorf("could not create syncer: %w", err) 97 | } 98 | 99 | vaultToken, err := sync.GetVaultToken(ctx, flags) 100 | if err != nil { 101 | return fmt.Errorf("could not get valid token: %w", err) 102 | } 103 | if err := sync.PerformSync(ctx, vaultToken, clock.Now(ctx).Add(flags.RenewInterval*2), flags); err != nil { 104 | return fmt.Errorf("could not peform sync: %w", err) 105 | } 106 | 107 | return nil 108 | } 109 | 110 | // PerformSidecar runs vault-ctrl-tool in sidecar mode. Each renew interval, it will retrieve a Vault 111 | // token and check if it is valid. However in the case where it cannot validate the validity of the 112 | // token (such in the case of a network issue with the Vault API), it will continue with checking if 113 | // dynamic secrets require renewal. 114 | // Failure to renew credentials will cause the sidecar to terminate. 115 | func PerformSidecar(ctx context.Context, flags util.CliFlags) error { 116 | 117 | c := make(chan os.Signal, 1) 118 | signal.Notify(c, os.Interrupt) 119 | signal.Notify(c, syscall.SIGTERM) 120 | 121 | mtrcs := metrics.NewMetrics() 122 | // if metrics server stops running then it will initiate shutdown. 123 | metrics.MetricsHandler(fmt.Sprintf(":%d", flags.PrometheusPort), c) 124 | 125 | go func() { 126 | zlog.Info().Str("renewInterval", flags.RenewInterval.String()).Str("buildVersion", buildVersion).Msg("starting") 127 | 128 | if err := sidecarSync(ctx, mtrcs, flags); err != nil { 129 | if flags.TerminateOnSyncFailure { 130 | zlog.Error().Err(err).Msg("failed initial sidecar sync, terminating") 131 | c <- os.Interrupt 132 | } else { 133 | zlog.Error().Err(err).Msg("failed initial sidecar sync") 134 | mtrcs.SidecarSyncErrors.Inc() 135 | } 136 | } 137 | renewTicker := time.NewTicker(flags.RenewInterval) 138 | defer renewTicker.Stop() 139 | 140 | jobCompletionTicker := time.NewTicker(ShutdownFileCheckFrequency) 141 | defer jobCompletionTicker.Stop() 142 | 143 | for { 144 | select { 145 | case <-renewTicker.C: 146 | zlog.Info().Msg("heartbeat") 147 | if err := sidecarSync(ctx, mtrcs, flags); err != nil { 148 | mtrcs.SidecarSyncErrors.Inc() 149 | if flags.TerminateOnSyncFailure { 150 | zlog.Error().Err(err).Msg("failed sidecar sync, terminating") 151 | c <- os.Interrupt 152 | } else { 153 | zlog.Error().Err(err).Msg("failed sidecar sync") 154 | mtrcs.SidecarSyncErrors.Inc() 155 | } 156 | } 157 | case <-jobCompletionTicker.C: 158 | if flags.ShutdownTriggerFile != "" { 159 | zlog.Debug().Str("triggerFile", flags.ShutdownTriggerFile).Msg("performing completion check against file") 160 | if _, err := os.Stat(flags.ShutdownTriggerFile); err == nil { 161 | zlog.Info().Str("triggerFile", flags.ShutdownTriggerFile).Msg("trigger file present; exiting") 162 | c <- os.Interrupt 163 | } 164 | } 165 | } 166 | } 167 | }() 168 | 169 | <-c 170 | zlog.Info().Msg("shutting down") 171 | return nil 172 | } 173 | 174 | func PerformCleanup(flags util.CliFlags) error { 175 | 176 | log := zlog.With().Str("configFile", flags.ConfigFile).Str("briefcase", flags.BriefcaseFilename).Logger() 177 | 178 | log.Info().Msg("performing cleanup") 179 | 180 | bc, err := briefcase.LoadBriefcase(flags.BriefcaseFilename, nil) 181 | if err != nil { 182 | log.Warn().Err(err).Msg("could not open briefcase") 183 | } else { 184 | 185 | if flags.RevokeOnCleanup && bc.AuthTokenLease.Token != "" { 186 | vaultClient, err := vaultclient.NewVaultClient(flags.ServiceSecretPrefix, flags.VaultClientTimeout, flags.VaultClientRetries) 187 | if err != nil { 188 | log.Error().Err(err).Msg("could not create new vault client to revoke token") 189 | } else { 190 | vaultClient.SetToken(bc.AuthTokenLease.Token) 191 | if err := vaultClient.Delegate().Auth().Token().RevokeSelf("ignored"); err != nil { 192 | log.Warn().Err(err).Msg("unable to revoke vault token") 193 | } 194 | } 195 | } 196 | 197 | if err := os.Remove(flags.BriefcaseFilename); err != nil { 198 | log.Warn().Err(err).Msg("could not remove briefcase") 199 | } 200 | } 201 | 202 | cfg, err := config.ReadConfigFile(flags.ConfigFile, flags.ConfigDir, flags.InputPrefix, flags.OutputPrefix) 203 | if err != nil { 204 | log.Warn().Msg("could not read config file - unsure what to cleanup") 205 | return fmt.Errorf("could not read config file %q: %w", flags.ConfigFile, err) 206 | } 207 | 208 | cfg.VaultConfig.Cleanup() 209 | 210 | log.Info().Msg("cleanup finished") 211 | 212 | return nil 213 | } 214 | 215 | func makeSyncer(flags util.CliFlags, mtrcs *metrics.Metrics) (*syncer.Syncer, error) { 216 | bc, err := briefcase.LoadBriefcase(flags.BriefcaseFilename, mtrcs) 217 | if err != nil { 218 | zlog.Warn().Str("briefcase", flags.BriefcaseFilename).Err(err).Msg("could not load briefcase - starting an empty one") 219 | bc = briefcase.NewBriefcase(mtrcs) 220 | } 221 | 222 | sync, err := syncer.SetupSyncer(flags, bc, mtrcs) 223 | if err != nil { 224 | return nil, err 225 | } 226 | 227 | return sync, nil 228 | } 229 | -------------------------------------------------------------------------------- /secrets/json_secrets.go: -------------------------------------------------------------------------------- 1 | package secrets 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 10 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 12 | "github.com/rs/zerolog" 13 | zlog "github.com/rs/zerolog/log" 14 | ) 15 | 16 | func WriteComposite(composite config.CompositeSecretFile, cache briefcase.SecretsCache) error { 17 | log := zlog.With().Str("filename", composite.Filename).Logger() 18 | 19 | log.Debug().Interface("compositeCfg", composite).Msg("writing composite secrets file") 20 | 21 | util.MustMkdirAllForFile(composite.Filename) 22 | 23 | file, err := os.OpenFile(composite.Filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, composite.Mode) 24 | 25 | if err != nil { 26 | return fmt.Errorf("couldn't open file %q: %w", composite.Filename, err) 27 | } 28 | 29 | defer file.Close() 30 | 31 | var kvSecrets []briefcase.SimpleSecret 32 | 33 | // make a copy 34 | kvSecrets = append(kvSecrets, cache.GetSecrets(util.LifetimeStatic)...) 35 | kvSecrets = append(kvSecrets, cache.GetSecrets(util.LifetimeVersion)...) 36 | 37 | if composite.Lifetime == util.LifetimeToken { 38 | kvSecrets = append(kvSecrets, cache.GetSecrets(util.LifetimeToken)...) 39 | } 40 | 41 | data, err := collectSecrets(log, composite, kvSecrets) 42 | 43 | if err != nil { 44 | return fmt.Errorf("could not output secrets file: %w", err) 45 | } 46 | 47 | if len(data) > 0 { 48 | err = json.NewEncoder(file).Encode(&data) 49 | 50 | if err != nil { 51 | return fmt.Errorf("failed to save secrets into %q: %w", composite.Filename, err) 52 | } 53 | } 54 | 55 | return nil 56 | } 57 | 58 | func WriteSecretFields(secret config.SecretType, kvSecrets []briefcase.SimpleSecret) (int, error) { 59 | mode, err := util.StringToFileMode(secret.Mode) 60 | count := 0 61 | 62 | if err != nil { 63 | return count, fmt.Errorf("could not parse file mode %q for key %q: %w", 64 | secret.Mode, secret.Key, err) 65 | } 66 | 67 | // output all the field files 68 | for _, field := range secret.Fields { 69 | if field.Output != "" { 70 | if err := writeField(secret, kvSecrets, field, *mode); err != nil { 71 | return count, err 72 | } 73 | count++ 74 | } 75 | } 76 | return count, nil 77 | } 78 | 79 | func writeField(secret config.SecretType, kvSecrets []briefcase.SimpleSecret, field config.SecretFieldType, mode os.FileMode) error { 80 | value := findSimpleSecretValue(kvSecrets, secret.Key, field.Name) 81 | 82 | if value == nil { 83 | if secret.IsMissingOk { 84 | zlog.Warn().Str("field", field.Name).Str("key", secret.Key).Str("output", field.Output).Msg("no secret found with key and missingOk=true, so no output will be written") 85 | } else { 86 | return fmt.Errorf("field %q not found in secret with key %q", field.Name, secret.Key) 87 | } 88 | 89 | } else { 90 | util.MustMkdirAllForFile(field.Output) 91 | 92 | file, err := os.OpenFile(field.Output, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) 93 | if err != nil { 94 | return fmt.Errorf("couldn't open file %q: %w", field.Output, err) 95 | } 96 | 97 | defer file.Close() 98 | 99 | zlog.Info().Str("field", field.Name).Str("key", secret.Key).Str("output", field.Output).Str("encoding", field.Encoding).Msg("writing field to file") 100 | 101 | switch field.Encoding { 102 | case util.EncodingBase64: 103 | decoded, err := base64.StdEncoding.DecodeString(fmt.Sprint(value)) 104 | if err != nil { 105 | return fmt.Errorf("failed to base64 decode field %q for secret %q: %w", field.Name, secret.Key, err) 106 | } 107 | _, err = fmt.Fprint(file, string(decoded)) 108 | default: 109 | _, err = fmt.Fprint(file, value) 110 | } 111 | if err != nil { 112 | return fmt.Errorf("failed writing secret to file %q: %w", field.Output, err) 113 | } 114 | } 115 | 116 | return nil 117 | } 118 | 119 | func findSimpleSecretValue(secrets []briefcase.SimpleSecret, key, field string) interface{} { 120 | for _, s := range secrets { 121 | if s.Key == key && s.Field == field { 122 | return s.Value 123 | } 124 | } 125 | return nil 126 | } 127 | 128 | func collectSecrets(log zerolog.Logger, composite config.CompositeSecretFile, kvSecrets []briefcase.SimpleSecret) (map[string]interface{}, error) { 129 | 130 | data := make(map[string]interface{}) 131 | 132 | log.Info().Msg("collecting composite secrets") 133 | 134 | for _, secret := range composite.Secrets { 135 | if secret.UseKeyAsPrefix { 136 | for _, s := range kvSecrets { 137 | if s.Key == secret.Key { 138 | key := secret.Key + "_" + s.Field 139 | if _, dupe := data[key]; dupe { 140 | log.Error().Str("field", s.Field).Str("prefix", secret.Key).Msg("the secret with this prefix causes there to be a duplicate entry") 141 | return nil, fmt.Errorf("the secret field %q with prefix %q causes there to be a duplicate", 142 | s.Field, secret.Key) 143 | } 144 | zlog.Debug().Str("key", key).Msg("collecting key") 145 | data[key] = s.Value 146 | } 147 | } 148 | } else { 149 | for _, s := range kvSecrets { 150 | if s.Key == secret.Key { 151 | if _, dupe := data[s.Field]; dupe { 152 | log.Error().Str("field", s.Field).Msg("this field causes there to be a duplicate entry") 153 | return nil, fmt.Errorf("the secret field %q causes there to be a duplicate", s.Field) 154 | } 155 | data[s.Field] = s.Value 156 | zlog.Debug().Str("field", s.Field).Msg("collecting field") 157 | } 158 | } 159 | } 160 | } 161 | return data, nil 162 | } 163 | -------------------------------------------------------------------------------- /secrets/sts.go: -------------------------------------------------------------------------------- 1 | package secrets 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 10 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/vaultclient" 12 | "github.com/rs/zerolog/log" 13 | ) 14 | 15 | func WriteAWSSTSCreds(creds *vaultclient.AWSSTSCredential, awsConfig config.AWSType) error { 16 | 17 | mode, err := util.StringToFileMode(awsConfig.Mode) 18 | if err != nil { 19 | return fmt.Errorf("could not parse %q as a file mode: %w", mode, err) 20 | } 21 | 22 | wipCfgFilename := filepath.Join(awsConfig.OutputPath, "config.wip") 23 | wipCredsFilename := filepath.Join(awsConfig.OutputPath, "credentials.wip") 24 | 25 | util.MustMkdirAllForFile(wipCfgFilename) 26 | 27 | if err := writeWIPFiles(wipCfgFilename, wipCredsFilename, creds, awsConfig.Profile, awsConfig.Region, *mode); err != nil { 28 | _ = os.Remove(wipCredsFilename) 29 | _ = os.Remove(wipCredsFilename) 30 | return err 31 | } 32 | 33 | cfgFilename := strings.TrimSuffix(wipCfgFilename, ".wip") 34 | credsFilename := strings.TrimSuffix(wipCredsFilename, ".wip") 35 | 36 | log.Debug().Strs("filenames", []string{wipCfgFilename, cfgFilename, wipCredsFilename, credsFilename}).Msg("atomically renaming .wip files") 37 | err = os.Rename(wipCfgFilename, cfgFilename) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | err = os.Rename(wipCredsFilename, credsFilename) 43 | if err != nil { 44 | return err 45 | } 46 | return nil 47 | } 48 | 49 | func writeWIPFiles(configFilename, credentialsFilename string, 50 | creds *vaultclient.AWSSTSCredential, 51 | awsProfile, awsRegion string, 52 | mode os.FileMode) error { 53 | 54 | log.Debug().Str("awsConfig", configFilename).Str("awsCredentials", credentialsFilename).Msg("writing AWS files") 55 | 56 | util.MustMkdirAllForFile(configFilename) 57 | 58 | configFile, err := os.OpenFile(configFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, mode) 59 | 60 | if err != nil { 61 | return fmt.Errorf("could not create aws config file at %q: %w", configFilename, err) 62 | } 63 | defer configFile.Close() 64 | 65 | credsFile, err := os.OpenFile(credentialsFilename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, mode) 66 | 67 | if err != nil { 68 | return fmt.Errorf("could not create aws credentials file at %q: %w", credentialsFilename, err) 69 | } 70 | defer credsFile.Close() 71 | 72 | header := strings.TrimSpace(awsProfile) 73 | 74 | _, err = fmt.Fprintf(credsFile, `[%s] 75 | aws_access_key_id=%s 76 | aws_secret_access_key=%s 77 | aws_session_token=%s 78 | 79 | `, 80 | header, creds.AccessKey, creds.SecretKey, creds.SessionToken) 81 | if err != nil { 82 | return fmt.Errorf("could not write contents to %q: %w", credentialsFilename, err) 83 | } 84 | 85 | _, err = fmt.Fprintf(configFile, "[%s]\nregion=%s\n\n", header, awsRegion) 86 | if err != nil { 87 | return fmt.Errorf("could not write contents to %q: %w", configFilename, err) 88 | } 89 | 90 | return nil 91 | } 92 | -------------------------------------------------------------------------------- /secrets/template.go: -------------------------------------------------------------------------------- 1 | package secrets 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "text/template" 7 | 8 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 9 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 10 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 11 | zlog "github.com/rs/zerolog/log" 12 | ) 13 | 14 | func WriteTemplate(tpl config.TemplateType, templates map[string]*template.Template, cache briefcase.SecretsCache) error { 15 | 16 | log := zlog.With().Str("output", tpl.Output).Logger() 17 | 18 | tplVars := make(map[string]interface{}) 19 | 20 | for _, s := range cache.GetSecrets(util.LifetimeStatic) { 21 | tplVars[s.Key+"_"+s.Field] = s.Value 22 | } 23 | 24 | if tpl.Lifetime == util.LifetimeToken { 25 | for _, s := range cache.GetSecrets(util.LifetimeToken) { 26 | key := s.Key + "_" + s.Field 27 | if _, dupe := tplVars[key]; dupe { 28 | log.Warn().Str("key", key).Msg("overwriting static secret key with a value from a token-scoped secret") 29 | } 30 | tplVars[key] = s.Value 31 | } 32 | } 33 | 34 | if len(tplVars) == 0 { 35 | log.Warn().Msg("no template variables found. this can be because your secrets are missing and missingOk=true, or if lifetimes of your secrets and template aren't right") 36 | } 37 | 38 | mode, err := util.StringToFileMode(tpl.Mode) 39 | if err != nil { 40 | return fmt.Errorf("could not parse file mode %q for template %q: %w", tpl.Mode, tpl.Input, err) 41 | } 42 | 43 | log.Info().Str("input", tpl.Input).Msg("resolving template") 44 | 45 | util.MustMkdirAllForFile(tpl.Output) 46 | 47 | file, err := os.OpenFile(tpl.Output, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, *mode) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | if err := templates[tpl.Input].Option("missingkey=error").Execute(file, tplVars); err != nil { 53 | return fmt.Errorf("failed to write template %q: %w", tpl.Output, err) 54 | } 55 | 56 | _ = file.Close() 57 | 58 | log.Debug().Msg("done executing template") 59 | 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /secrets/token.go: -------------------------------------------------------------------------------- 1 | package secrets 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 8 | "github.com/hootsuite/vault-ctrl-tool/v2/metrics" 9 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 10 | zlog "github.com/rs/zerolog/log" 11 | ) 12 | 13 | func WriteVaultToken(m *metrics.Metrics, tokenCfg config.VaultTokenType, vaultToken string) error { 14 | 15 | if tokenCfg.Output == "" { 16 | zlog.Warn().Interface("tokenCfg", tokenCfg).Msg("no output file specified to write vault token") 17 | return nil 18 | } 19 | 20 | zlog.Info().Str("outputFile", tokenCfg.Output).Msg("writing Vault token to file") 21 | 22 | mode, err := util.StringToFileMode(tokenCfg.Mode) 23 | 24 | if err != nil { 25 | return fmt.Errorf("could not parse file mode %q for %q: %w", tokenCfg.Mode, tokenCfg.Output, err) 26 | } 27 | 28 | util.MustMkdirAllForFile(tokenCfg.Output) 29 | 30 | file, err := os.OpenFile(tokenCfg.Output, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, *mode) 31 | if err != nil { 32 | return fmt.Errorf("failed to create Vault token file %q: %w", tokenCfg.Output, err) 33 | } 34 | 35 | defer file.Close() 36 | 37 | _, err = fmt.Fprintf(file, "%s\n", vaultToken) 38 | 39 | if err != nil { 40 | return fmt.Errorf("failed to create Vault token file %q: %w", tokenCfg.Output, err) 41 | } 42 | 43 | m.Increment(metrics.VaultTokenWritten) 44 | return nil 45 | } 46 | -------------------------------------------------------------------------------- /syncer/compare.go: -------------------------------------------------------------------------------- 1 | package syncer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 9 | "github.com/hootsuite/vault-ctrl-tool/v2/secrets" 10 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/util/clock" 12 | ) 13 | 14 | func (s *Syncer) compareSecrets(ctx context.Context, updates *int) error { 15 | for _, secret := range s.config.VaultConfig.Secrets { 16 | log := s.log.With().Interface("secretCfg", secret).Logger() 17 | log.Debug().Msg("checking secret") 18 | 19 | switch secret.Lifetime { 20 | // Secrets with "version" lifetime are automatically updated when the secret is updated in Vault. This is 21 | // different than Token / Static lifetimes, so the code is a bit messier. At some point there could 22 | // be a desire for version scoped templates/composites/etc/etc at which point it becomes worthwhile 23 | // to rearrange this code. 24 | case util.LifetimeVersion: 25 | 26 | simpleSecrets, err := s.readSecret(secret) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | if len(simpleSecrets) > 0 { 32 | ss := simpleSecrets[0] 33 | if ss.Version == nil { 34 | return fmt.Errorf("no version number associated with secret %q and lifetime is %q", 35 | secret.Key, util.LifetimeVersion) 36 | } 37 | 38 | briefcaseVersion := s.briefcase.VersionScopedSecrets[secret.Path] 39 | 40 | log.Debug().Int64("secretVersion", *ss.Version). 41 | Int64("briefcaseSecretVersion", briefcaseVersion). 42 | Time("secretTimestamp", *ss.CreatedTime). 43 | Time("now", clock.Now(ctx)). 44 | Msg("comparing briefcase version of secret to current version") 45 | 46 | if briefcaseVersion == 0 || 47 | (briefcaseVersion < *ss.Version && 48 | ss.CreatedTime.Add(30*time.Second).Before(clock.Now(ctx))) { 49 | 50 | count, err := secrets.WriteSecretFields(secret, simpleSecrets) 51 | if err != nil { 52 | return fmt.Errorf("could not write secret %q: %w", secret.Path, err) 53 | } 54 | *updates += count 55 | 56 | if count > 0 { 57 | if err := util.TouchFile(secret.TouchFile); err != nil { 58 | log.Warn().Str("touchfile", secret.TouchFile).Err(err).Msg("failed to 'touch' touchfile.") 59 | } 60 | } 61 | s.briefcase.VersionScopedSecrets[secret.Path] = *ss.Version 62 | } else { 63 | log.Debug().Msg("not updating secret") 64 | } 65 | } else { 66 | log.Warn().Msg("no fields returned for secret") 67 | } 68 | case util.LifetimeToken, util.LifetimeStatic: 69 | if s.briefcase.ShouldRefreshSecret(secret) { 70 | log.Debug().Msg("refreshing secret") 71 | 72 | if secret.Lifetime == util.LifetimeToken { 73 | if err := s.cacheSecrets(util.LifetimeToken); err != nil { 74 | return err 75 | } 76 | } 77 | 78 | if err := s.cacheSecrets(util.LifetimeStatic); err != nil { 79 | return err 80 | } 81 | 82 | var kvSecrets []briefcase.SimpleSecret 83 | 84 | // make a copy 85 | kvSecrets = append(kvSecrets, s.briefcase.GetSecrets(util.LifetimeStatic)...) 86 | kvSecrets = append(kvSecrets, s.briefcase.GetSecrets(util.LifetimeVersion)...) 87 | 88 | if secret.Lifetime == util.LifetimeToken { 89 | kvSecrets = append(kvSecrets, s.briefcase.GetSecrets(util.LifetimeToken)...) 90 | } 91 | 92 | count, err := secrets.WriteSecretFields(secret, kvSecrets) 93 | if err != nil { 94 | log.Error().Err(err).Msg("failed to write secret") 95 | return err 96 | } 97 | *updates += count 98 | s.briefcase.EnrollSecret(secret) 99 | } 100 | default: 101 | log.Error().Str("lifetime", string(secret.Lifetime)).Msg("internal error: missing code to sync secrets with lifetime") 102 | } 103 | } 104 | return nil 105 | } 106 | 107 | func (s *Syncer) compareTemplates(updates *int) error { 108 | for _, tmpl := range s.config.VaultConfig.Templates { 109 | log := s.log.With().Interface("tmplCfg", tmpl).Logger() 110 | log.Debug().Msg("checking template") 111 | if s.briefcase.ShouldRefreshTemplate(tmpl) { 112 | if updates != nil { 113 | *updates++ 114 | } 115 | log.Debug().Msg("refreshing template") 116 | 117 | if tmpl.Lifetime == util.LifetimeToken { 118 | if err := s.cacheSecrets(util.LifetimeToken); err != nil { 119 | return err 120 | } 121 | } 122 | 123 | if err := s.cacheSecrets(util.LifetimeStatic); err != nil { 124 | return err 125 | } 126 | 127 | if err := secrets.WriteTemplate(tmpl, s.config.Templates, s.briefcase); err != nil { 128 | log.Error().Err(err).Msg("failed to write template") 129 | return err 130 | } 131 | log.Debug().Msg("enrolling template") 132 | s.briefcase.EnrollTemplate(tmpl) 133 | } 134 | } 135 | return nil 136 | } 137 | 138 | func (s *Syncer) compareSSHCertificates(ctx context.Context, updates *int, nextSync time.Time, forceRefreshTTL time.Duration) error { 139 | for _, ssh := range s.config.VaultConfig.SSHCertificates { 140 | log := s.log.With().Interface("sshCfg", ssh).Logger() 141 | log.Debug().Msg("checking SSH certificate") 142 | 143 | if s.briefcase.ShouldRefreshSSHCertificate(ssh, nextSync) { 144 | if updates != nil { 145 | *updates++ 146 | } 147 | log.Debug().Msg("refreshing ssh certificate") 148 | 149 | if err := s.vaultClient.CreateSSHCertificate(ssh); err != nil { 150 | log.Error().Err(err).Msg("failed to fetch SSH certificate credentials") 151 | return err 152 | } 153 | 154 | if err := s.briefcase.EnrollSSHCertificate(ctx, ssh, forceRefreshTTL); err != nil { 155 | log.Error().Err(err).Msg("failed to enroll SSH certificate in briefcase") 156 | return err 157 | } 158 | } 159 | } 160 | return nil 161 | } 162 | 163 | func (s *Syncer) compareAWS(ctx context.Context, updates *int, nextSync time.Time, stsTTL, forceRefreshTTL time.Duration) error { 164 | for _, aws := range s.config.VaultConfig.AWS { 165 | log := s.log.With().Interface("awsCfg", aws).Logger() 166 | log.Debug().Msg("checking AWS STS credential") 167 | 168 | if s.briefcase.AWSCredentialShouldRefreshBefore(aws, nextSync) || s.briefcase.AWSCredentialExpiresBefore(aws, nextSync) { 169 | if updates != nil { 170 | *updates++ 171 | } 172 | 173 | log.Debug(). 174 | Bool("forcedRefreshBeforeNextHearbeat", s.briefcase.AWSCredentialShouldRefreshBefore(aws, nextSync)). 175 | Bool("credentialExpiresBeforeNextHeartbeat", s.briefcase.AWSCredentialExpiresBefore(aws, nextSync)). 176 | Msg("refreshing AWS STS credential") 177 | 178 | creds, secret, err := s.vaultClient.FetchAWSSTSCredential(aws, stsTTL) 179 | 180 | if err != nil { 181 | log.Error().Err(err).Msg("failed to fetch AWS STS credentials") 182 | return err 183 | } 184 | 185 | if err := secrets.WriteAWSSTSCreds(creds, aws); err != nil { 186 | log.Error().Err(err).Msg("failed to write file with AWS STS credentials") 187 | return err 188 | } 189 | 190 | s.briefcase.EnrollAWSCredential(ctx, secret.Secret, aws, forceRefreshTTL) 191 | } 192 | } 193 | return nil 194 | } 195 | -------------------------------------------------------------------------------- /syncer/sync.go: -------------------------------------------------------------------------------- 1 | package syncer 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "path/filepath" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/hashicorp/vault/api" 14 | "github.com/hootsuite/vault-ctrl-tool/v2/metrics" 15 | 16 | "github.com/hootsuite/vault-ctrl-tool/v2/vaulttoken" 17 | 18 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 19 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 20 | "github.com/hootsuite/vault-ctrl-tool/v2/secrets" 21 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 22 | "github.com/hootsuite/vault-ctrl-tool/v2/vaultclient" 23 | "github.com/rs/zerolog" 24 | "github.com/rs/zerolog/log" 25 | ) 26 | 27 | // Syncer performs Vault secrets synchronizations. 28 | type Syncer struct { 29 | log zerolog.Logger 30 | config *config.ControlToolConfig 31 | vaultClient vaultclient.VaultClient 32 | briefcase *briefcase.Briefcase 33 | metrics *metrics.Metrics 34 | } 35 | 36 | func NewSyncer(log zerolog.Logger, cfg *config.ControlToolConfig, vaultClient vaultclient.VaultClient, briefcase *briefcase.Briefcase, metrics *metrics.Metrics) *Syncer { 37 | return &Syncer{ 38 | log: log, 39 | config: cfg, 40 | vaultClient: vaultClient, 41 | briefcase: briefcase, 42 | metrics: metrics, 43 | } 44 | } 45 | 46 | func SetupSyncer(flags util.CliFlags, bc *briefcase.Briefcase, m *metrics.Metrics) (*Syncer, error) { 47 | log, cfg, vaultClient, err := configureSyncerDependencies(flags) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | syncer := NewSyncer(log, cfg, vaultClient, bc, m) 53 | 54 | return syncer, nil 55 | } 56 | 57 | func configureSyncerDependencies(flags util.CliFlags) (zerolog.Logger, *config.ControlToolConfig, vaultclient.VaultClient, error) { 58 | 59 | log := log.With().Str("cfg", flags.ConfigFile).Logger() 60 | 61 | cfg, err := config.ReadConfigFile(flags.ConfigFile, flags.ConfigDir, flags.InputPrefix, flags.OutputPrefix) 62 | if err != nil { 63 | return log, nil, nil, err 64 | } 65 | 66 | vaultClient, err := vaultclient.NewVaultClient(flags.ServiceSecretPrefix, flags.VaultClientTimeout, flags.VaultClientRetries) 67 | if err != nil { 68 | log.Error().Err(err).Msg("could not create vault client") 69 | return log, nil, nil, err 70 | } 71 | 72 | return log, cfg, vaultClient, nil 73 | } 74 | 75 | // PerformSync does primary VCT syncing logic by obtaining a Vault token and checking it's validity. 76 | // If a token is found, but cannot be validated this will return a wrapped ErrorCouldNotValidateToken error. 77 | func (s *Syncer) GetVaultToken(ctx context.Context, flags util.CliFlags) (vaulttoken.VaultToken, error) { 78 | vaultToken := s.obtainVaultToken(flags) 79 | 80 | if err := s.checkVaultToken(vaultToken, flags); err != nil { 81 | s.metrics.SidecarVaultTokenErrors.Inc() 82 | return nil, fmt.Errorf("failed to check token: %w", err) 83 | } 84 | 85 | return vaultToken, nil 86 | } 87 | 88 | // PerformSync does primary VCT syncing logic by obtaining a refreshing dynamic credentials. 89 | func (s *Syncer) PerformSync(ctx context.Context, vaultToken vaulttoken.VaultToken, nextSync time.Time, flags util.CliFlags) error { 90 | s.vaultClient.SetToken(vaultToken.TokenID()) 91 | 92 | // First we compare the vault token we're using with the one in the briefcase. If it's different, then 93 | // we reset the briefcase to start over. We do this here to ease the briefcase compare below. We also 94 | // write it to a file if configured at this point 95 | if s.briefcase.AuthTokenLease.Token != vaultToken.TokenID() { 96 | s.log.Debug().Msg("briefcase token differs from current token, resetting briefcase") 97 | s.briefcase = s.briefcase.ResetBriefcase() 98 | if s.config.VaultConfig.VaultToken.Output != "" { 99 | if err := secrets.WriteVaultToken(s.metrics, s.config.VaultConfig.VaultToken, vaultToken.TokenID()); err != nil { 100 | return fmt.Errorf("could not write vault token: %w", err) 101 | } 102 | } 103 | if err := s.briefcase.EnrollVaultToken(ctx, vaultToken.Wrapped()); err != nil { 104 | return fmt.Errorf("could not enroll vault token into briefcase: %w", err) 105 | } 106 | } 107 | 108 | if s.briefcase.ShouldRefreshVaultToken(ctx) { 109 | s.log.Debug().Msg("refreshing vault token against server") 110 | secret, err := s.vaultClient.RefreshVaultToken() 111 | if err != nil { 112 | s.metrics.SidecarSyncErrors.Inc() 113 | return fmt.Errorf("could not refresh vault token: %w", err) 114 | } 115 | s.metrics.Increment(metrics.VaultTokenRefreshed) 116 | 117 | if err := s.briefcase.EnrollVaultToken(ctx, util.NewWrappedToken(secret, s.briefcase.AuthTokenLease.Renewable)); err != nil { 118 | s.metrics.SidecarSyncErrors.Inc() 119 | return fmt.Errorf("could not enroll refreshed vault token into briefcase: %w", err) 120 | } 121 | } 122 | 123 | err := s.compareConfigToBriefcase(ctx, nextSync, flags.STSTTL, flags.ForceRefreshTTL) 124 | if err != nil { 125 | s.metrics.SidecarSyncErrors.Inc() 126 | return fmt.Errorf("could not compare config against briefcase: %w", err) 127 | } 128 | 129 | err = s.briefcase.SaveAs(flags.BriefcaseFilename) 130 | if err != nil { 131 | return fmt.Errorf("could not save briefcase as '%s': %w", flags.BriefcaseFilename, err) 132 | } 133 | return nil 134 | } 135 | 136 | // compareConfigToBriefcase does what it says on the tin. Given the list of secrets expected to exist (listed in the config), 137 | // compare that to the secrets that are being tracked in the briefcase. If they need to be refreshed, then refresh them 138 | // and update the briefcase. 139 | func (s *Syncer) compareConfigToBriefcase(ctx context.Context, nextSync time.Time, stsTTL, forceRefreshTTL time.Duration) error { 140 | updates := 0 141 | 142 | if err := s.compareAWS(ctx, &updates, nextSync, stsTTL, forceRefreshTTL); err != nil { 143 | return err 144 | } 145 | 146 | if err := s.compareSSHCertificates(ctx, &updates, nextSync, forceRefreshTTL); err != nil { 147 | return err 148 | } 149 | 150 | if err := s.compareTemplates(&updates); err != nil { 151 | return err 152 | } 153 | 154 | if err := s.compareSecrets(ctx, &updates); err != nil { 155 | return err 156 | } 157 | 158 | for _, composite := range s.config.Composites { 159 | log := s.log.With().Interface("compositeFilename", composite.Filename).Logger() 160 | log.Debug().Msg("checking composite secret") 161 | if s.briefcase.ShouldRefreshComposite(*composite) { 162 | updates++ 163 | log.Debug().Msg("refreshing composite") 164 | if composite.Lifetime == util.LifetimeToken { 165 | if err := s.cacheSecrets(util.LifetimeToken); err != nil { 166 | return err 167 | } 168 | } 169 | if err := s.cacheSecrets(util.LifetimeStatic); err != nil { 170 | return err 171 | } 172 | 173 | if err := secrets.WriteComposite(*composite, s.briefcase); err != nil { 174 | log.Error().Err(err).Msg("failed to write composite json secret") 175 | return err 176 | } 177 | log.Debug().Msg("enrolling composite secret") 178 | s.briefcase.EnrollComposite(*composite) 179 | } 180 | } 181 | 182 | s.metrics.IncrementBy(metrics.SecretUpdates, updates) 183 | s.log.Info().Int("updates", updates).Msg("done comparing configuration against briefcase") 184 | return nil 185 | } 186 | 187 | // obtainVaultToken works in conjunction with a "VaultToken" object. This object uses the briefcase, CLI flags, 188 | // and environment variables to try to find a workable vault token. This function will build an "authenticator" 189 | // whose job it is to authenticate against Vault using whatever material is specified and come up with a new 190 | // vault token if needed. 191 | func (s *Syncer) obtainVaultToken(flags util.CliFlags) vaulttoken.VaultToken { 192 | 193 | log := s.log.With().Str("vaultAddr", s.vaultClient.Address()).Logger() 194 | 195 | log.Info().Msg("obtaining vault token") 196 | 197 | token := vaulttoken.NewVaultToken(s.briefcase, s.vaultClient, flags.VaultTokenArg, flags.CliVaultTokenRenewable) 198 | return token 199 | } 200 | 201 | func (s *Syncer) checkVaultToken(token vaulttoken.VaultToken, flags util.CliFlags) error { 202 | if err := token.CheckAndRefresh(); err != nil { 203 | if errors.Is(err, vaulttoken.ErrNoValidVaultTokenAvailable) { 204 | log.Debug().Err(err).Msg("no vault token already available, performing authentication") 205 | 206 | authenticator, err := vaultclient.NewAuthenticator(s.vaultClient, flags) 207 | if err != nil { 208 | log.Error().Err(err).Msg("unable to create authenticator") 209 | return err 210 | } 211 | log.Debug().Str("authenticator", fmt.Sprintf("%+v", authenticator)).Msg("authenticator created") 212 | secret, err := authenticator.Authenticate() 213 | if err != nil { 214 | log.Error().Err(err).Msg("authentication failed") 215 | return err 216 | } 217 | 218 | accessor, err := secret.TokenAccessor() 219 | if err != nil { 220 | log.Error().Err(err).Msg("could not get accessor of new vault token") 221 | return err 222 | } 223 | 224 | log.Info().Str("accessor", accessor).Msg("authentication successful") 225 | 226 | err = token.Set(secret) 227 | if err != nil { 228 | log.Error().Err(err).Msg("could not store vault token") 229 | return err 230 | } 231 | } else { 232 | log.Error().Err(err).Msg("could not establish vault token") 233 | return err 234 | } 235 | } 236 | 237 | log.Info().Str("accessor", token.Accessor()).Msg("using valid token") 238 | 239 | return nil 240 | } 241 | 242 | // cacheSecrets has the job of fetching secrets from Vault, if they're needed. The need is based on a few things, but 243 | // mostly on the "lifetime" of the secret. Static secrets are only fetched once, token-lifetime are refetched if the 244 | // token being used changes. 245 | func (s *Syncer) cacheSecrets(lifetime util.SecretLifetime) error { 246 | if s.briefcase.HasCachedSecrets(lifetime) { 247 | return nil 248 | } 249 | 250 | var simpleSecrets []briefcase.SimpleSecret 251 | 252 | for _, secret := range s.config.VaultConfig.Secrets { 253 | if secret.Lifetime == lifetime { 254 | 255 | // The same key could be in different paths, but we don't allow this because it's confusing. 256 | for _, s := range simpleSecrets { 257 | if s.Key == secret.Key { 258 | return fmt.Errorf("duplicate secret key %q", secret.Key) 259 | } 260 | } 261 | 262 | if secretData, err := s.readSecret(secret); err != nil { 263 | return err 264 | } else { 265 | simpleSecrets = append(simpleSecrets, secretData...) 266 | } 267 | } 268 | } 269 | 270 | s.briefcase.StoreSecrets(lifetime, simpleSecrets) 271 | 272 | return nil 273 | } 274 | 275 | // readSecret ingests the specified secret with whatever parameters it has. It returns an array of "simplesecret" which is really 276 | // an array of key=value for each field in the secret. Errors will occur if the specified secret is required to be in KVv2 277 | // (for metadata) but it's not. 278 | func (s *Syncer) readSecret(secret config.SecretType) ([]briefcase.SimpleSecret, error) { 279 | var simpleSecrets []briefcase.SimpleSecret 280 | 281 | key := secret.Key 282 | 283 | log := s.log.With().Str("path", secret.Path).Str("vaultAddr", s.vaultClient.Address()).Logger() 284 | 285 | // Some secrets require metadata to be processed correctly based on their configuration. 286 | if s.config.VaultConfig.ConfigVersion < 2 && secret.NeedsMetadata() { 287 | log.Error().Msg("In order to process this secret, metadata is needed, but metadata is only available for config files version 2 and above.") 288 | return nil, fmt.Errorf("secret %q requires metadata, but version of the config file version is %d and metadata is not available until 2 or later", 289 | secret.Key, s.config.VaultConfig.ConfigVersion) 290 | } 291 | 292 | log.Info().Msg("fetching secret") 293 | 294 | var path string 295 | 296 | if !strings.HasPrefix(secret.Path, "/") { 297 | path = filepath.Join(s.vaultClient.ServiceSecretPrefix(s.config.VaultConfig.ConfigVersion), secret.Path) 298 | } else { 299 | path = secret.Path 300 | } 301 | 302 | log.Debug().Msg("reading secret from Vault") 303 | 304 | var response *api.Secret 305 | var err error 306 | 307 | if secret.PinnedVersion != nil { 308 | log.Debug().Int("pinnedVersion", *secret.PinnedVersion).Msg("fetching specific version") 309 | response, err = s.vaultClient.ReadWithData(path, map[string][]string{ 310 | "version": {strconv.Itoa(*secret.PinnedVersion)}, 311 | }) 312 | } else { 313 | response, err = s.vaultClient.Read(path) 314 | } 315 | 316 | if err != nil { 317 | return nil, fmt.Errorf("error fetching secret %q from %q: %w", path, s.vaultClient.Address(), err) 318 | } 319 | 320 | if response == nil { 321 | // For migration purposes, we allow some secrets to not exist. 322 | if secret.IsMissingOk { 323 | log.Info().Msg("no response reading secrets from path (either access is denied or there are no secrets). Ignoring since missingOk is set in the config") 324 | } else { 325 | return nil, fmt.Errorf("no response returned fetching secrets") 326 | } 327 | } else { 328 | 329 | // If this is a KVv1 secret, then the fields are returned directly in the "data" stanza of the response. 330 | // If this is a KVv2 secret, then the "data" stanza of the response has two sub-sections: "data" and 331 | // "metadata". This code breaks if a KVv1 secret has a field called "data". 332 | 333 | var secretData map[string]interface{} 334 | var secretMetadata map[string]interface{} 335 | var secretVersion *int64 336 | var secretCreated *time.Time 337 | 338 | if s.config.VaultConfig.ConfigVersion < 2 { 339 | secretData = response.Data 340 | } else { 341 | var hasMetadata, hasData bool 342 | // We guess we're in KVv2 if there's both a "data" and "metadata" in the response "data" stanza. 343 | secretData, hasData = response.Data["data"].(map[string]interface{}) 344 | secretMetadata, hasMetadata = response.Data["metadata"].(map[string]interface{}) 345 | 346 | // It's a failure if we need metadata to process this secret, and we're not a KVv2 secret. 347 | if secret.NeedsMetadata() && (!hasData || !hasMetadata) { 348 | return nil, fmt.Errorf("error getting KVv2 secret %q from %q: probably not in a KVv2 path", path, s.vaultClient.Address()) 349 | } 350 | 351 | if !(hasData && hasMetadata) { 352 | secretData = response.Data 353 | secretMetadata = nil 354 | } 355 | } 356 | 357 | if secretMetadata != nil { 358 | log.Debug().Str("metadata", fmt.Sprintf("%+v", secretMetadata)).Msg("retrieved metadata") 359 | 360 | // I've had this value come back as both json.Number, and a float.. *shrug* 361 | if v, ok := secretMetadata["version"]; ok { 362 | var vers int64 363 | var err error 364 | switch val := v.(type) { 365 | case json.Number: 366 | vers, err = val.Int64() 367 | case float64: 368 | vers = int64(val) 369 | default: 370 | log.Warn().Str("type", fmt.Sprintf("%T", v)).Msg("unknown type for version metadata") 371 | vers, err = strconv.ParseInt(fmt.Sprintf("%v", v), 10, 64) 372 | } 373 | if err != nil { 374 | log.Error().Err(err).Interface("version", v).Msg("could not convert to integer") 375 | return nil, fmt.Errorf("could not convert %q to integer: %w", v, err) 376 | } 377 | secretVersion = &vers 378 | } else { 379 | return nil, fmt.Errorf("no version metadata field for secret %q from %q", path, s.vaultClient.Address()) 380 | } 381 | 382 | if ts, ok := secretMetadata["created_time"]; ok { 383 | parsedTime, err := time.Parse(time.RFC3339Nano, ts.(string)) 384 | if err != nil { 385 | return nil, fmt.Errorf("unable to parse created_time timestamp %q for secret %q from %q", 386 | ts, path, s.vaultClient.Address()) 387 | } 388 | secretCreated = &parsedTime 389 | } else { 390 | return nil, fmt.Errorf("no created_time field for secret %q from %q", path, s.vaultClient.Address()) 391 | } 392 | 393 | } else { 394 | log.Debug().Msg("no metadata retrieved") 395 | } 396 | 397 | for f, v := range secretData { 398 | simpleSecrets = append(simpleSecrets, briefcase.SimpleSecret{ 399 | Key: key, 400 | Field: f, 401 | Value: v, 402 | Version: secretVersion, 403 | CreatedTime: secretCreated, 404 | }) 405 | } 406 | } 407 | 408 | return simpleSecrets, nil 409 | } 410 | -------------------------------------------------------------------------------- /util/clock/clock.go: -------------------------------------------------------------------------------- 1 | package clock 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "k8s.io/utils/clock" 8 | ) 9 | 10 | // using these utility methods, a FakeClock (NewFakeClock) can be "injected" via a context.Context. Code that needs 11 | // the current time can call "clock.Now(ctx)". Note that I'm using the Kubernetes "utils/clock", which also has 12 | // support for a bunch of other methods that I haven't proxied here. 13 | 14 | var contextKey = "vctClock" 15 | 16 | type Factory func(ctx context.Context) clock.Clock 17 | 18 | func Now(ctx context.Context) time.Time { 19 | return Get(ctx).Now() 20 | } 21 | 22 | func SetFactory(ctx context.Context, f Factory) context.Context { 23 | return context.WithValue(ctx, &contextKey, f) 24 | } 25 | 26 | // Set creates a new Context using the supplied Clock. 27 | func Set(ctx context.Context, c clock.Clock) context.Context { 28 | return SetFactory(ctx, func(context.Context) clock.Clock { return c }) 29 | } 30 | 31 | func Get(ctx context.Context) clock.Clock { 32 | if v := ctx.Value(&contextKey); v != nil { 33 | if f, ok := v.(Factory); ok { 34 | return f(ctx) 35 | } 36 | } 37 | return clock.RealClock{} 38 | } 39 | -------------------------------------------------------------------------------- /util/constants.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | const VaultEC2AuthPath = "/v1/auth/aws-ec2/login" 4 | 5 | // EnableKubernetesVaultTokenAuthentication (see references for description) 6 | // Disable this at compile time if you don't use this feature. 7 | const EnableKubernetesVaultTokenAuthentication = true 8 | 9 | // SSHCertificate is public key, signed by Vault. 10 | const SSHCertificate = "id_rsa-cert.pub" 11 | 12 | // SecretLifetime is used to describe secrets lifetime description. 13 | type SecretLifetime string 14 | 15 | // Secrets and templates can have a lifetime associated with them, those without an explicit lifetime 16 | // have a "static" lifetime for backwards expectations. 17 | const LifetimeStatic SecretLifetime = "static" 18 | const LifetimeToken SecretLifetime = "token" 19 | 20 | // LifetimeVersion is a hack. It will refresh fields of secrets when the version of the secret increases. It 21 | // does not support composite secrets, or anything else. If this winds up being valuable, the interactions 22 | // between briefcase<->config will need to be rewritten since both other lifetimes operate with the exact 23 | // opposite philosophy. 24 | const LifetimeVersion SecretLifetime = "version" 25 | 26 | // fields can be encoded - those base64 encoded are decoded before being written to output files. They're not 27 | // decoded if they're part of a template / etc / etc. 28 | const EncodingBase64 = "base64" 29 | const EncodingNone = "none" 30 | -------------------------------------------------------------------------------- /util/flags.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "gopkg.in/alecthomas/kingpin.v2" 10 | ) 11 | 12 | // CliFlags contains all flags for the vault-ctrl-tool application. 13 | // v1 of vault-ctrl-tool had some bad ideas about parsing command line arguments. This is kept for compatibility. 14 | type CliFlags struct { 15 | ShowVersion bool // Display version and exit 16 | PerformInit bool // run in "init" mode 17 | PerformSidecar bool // run in "sidecar" mode 18 | PerformOneShot bool // even though running in sidecar mode, only run things once and then exit. 19 | PerformCleanup bool // cleanup everything in the leases file 20 | RevokeOnCleanup bool // also revoke everything when cleaning up 21 | RenewInterval time.Duration // when in sidecar mode, this is the expected period between checks 22 | BriefcaseFilename string // absolute location of briefcase 23 | ShutdownTriggerFile string // if this file exists, the sidecar will shutdown 24 | VaultTokenArg string // v-c-t will accept a vault token as a command line arg 25 | EC2AuthEnabled bool // use "registered AMI" to authenticate an EC2 instance 26 | EC2Nonce string // Nonce used for re-authenticating EC2 instances 27 | IAMAuthRole string // Role to use when performing IAM authentication of EC2 instances 28 | IAMVaultAuthBackend string // Override IAM auth path in Vault 29 | ConfigFile string // location of vault-config, either relative to input prefix, or absolute 30 | ConfigDir string // location of vault-config directory, either relative to input prefix, or absolute 31 | OutputPrefix string // prefix to use when writing output files 32 | InputPrefix string // prefix to use when looking for input files 33 | ServiceSecretPrefix string // override prefix for relative KV secrets 34 | KubernetesLoginPath string // path to use in Vault for Kubernetes authentication 35 | ServiceAccountToken string // path to the ServiceAccount token file for Kubernetes authentication 36 | KubernetesAuthRole string // enables Kubernetes auth, and sets role to use with Kubernetes authentication 37 | DebugLogLevel bool // enable debug logging 38 | CliVaultTokenRenewable bool // is the vault token supplied on the command line renewable? 39 | ForceRefreshTTL time.Duration // secrets will be refreshed after this duration, regardless of their expiry. 40 | STSTTL time.Duration // configures what TTL to use for AWS STS tokens. 41 | EnablePrometheusMetrics bool // configures whether to enable prometheus metrics server for sidecar mode. 42 | PrometheusPort int // configures port on which to serve prometheus metrics endpoint 43 | VaultClientTimeout time.Duration // configures HTTP timeouts for Vault client connections. 44 | VaultClientRetries int // configures HTTP retries for Vault client connections. 45 | TerminateOnSyncFailure bool // If enabled in sidecar mode, will cause tool to terminate if there is a failure to perform sync. 46 | } 47 | 48 | type RunMode int 49 | 50 | const ( 51 | ModeShowVersion RunMode = iota 52 | ModeInit 53 | ModeSidecar 54 | ModeOneShotSidecar 55 | ModeCleanup 56 | ModeUnknown 57 | ) 58 | 59 | type AuthMechanismType int 60 | 61 | const ( 62 | EC2AMIAuth AuthMechanismType = iota 63 | EC2IAMAuth 64 | KubernetesAuth 65 | UnknownAuth 66 | ) 67 | 68 | func (f *CliFlags) AuthMechanism() AuthMechanismType { 69 | if f.KubernetesAuthRole != "" { 70 | return KubernetesAuth 71 | } 72 | 73 | if f.EC2AuthEnabled { 74 | return EC2AMIAuth 75 | } 76 | 77 | if f.IAMAuthRole != "" { 78 | return EC2IAMAuth 79 | } 80 | 81 | return UnknownAuth 82 | } 83 | 84 | func (f *CliFlags) RunMode() RunMode { 85 | if f.ShowVersion { 86 | return ModeShowVersion 87 | } 88 | 89 | if f.PerformInit { 90 | return ModeInit 91 | } 92 | 93 | if f.PerformSidecar { 94 | if f.PerformOneShot { 95 | return ModeOneShotSidecar 96 | } 97 | return ModeSidecar 98 | } 99 | 100 | if f.PerformCleanup { 101 | return ModeCleanup 102 | } 103 | return ModeUnknown 104 | } 105 | 106 | func ProcessFlags(args []string) (*CliFlags, error) { 107 | var flags CliFlags 108 | 109 | app := kingpin.New("vault-ctrl-tool", "A handy tool for interacting with HashiCorp Vault\n\n"+ 110 | "Boolean flags can be disabled through using the complement flag by prefixing it with 'no-' (for example: '--no-token-renewable).") 111 | 112 | app.Flag("init", "Run in init mode, process templates and exit.").Default("false").BoolVar(&flags.PerformInit) 113 | app.Flag("config", "Full path of the config file to read.").Default("vault-config.yml").StringVar(&flags.ConfigFile) 114 | app.Flag("config-dir", "Full path of the config directory to read config files. Be aware that the config version is read only once so all sub config should have the same version.").Default("").StringVar(&flags.ConfigDir) 115 | app.Flag("output-prefix", "Path to prefix to all output files (such as /etc/secrets)").StringVar(&flags.OutputPrefix) 116 | app.Flag("input-prefix", "Path to prefix on all files being read; including the config file. Only the main config file (--config-file) will have his root values read. Those in the config directory will be ignored.").StringVar(&flags.InputPrefix) 117 | app.Flag("secret-prefix", "Vault path to prepend to secrets with relative paths").StringVar(&flags.ServiceSecretPrefix) 118 | 119 | app.Flag("renew-interval", "Interval to renew credentials").Default("9m").DurationVar(&flags.RenewInterval) 120 | app.Flag("leases-file", "Full path to briefcase file.").Default("/tmp/vault-leases/vault-ctrl-tool.leases").StringVar(&flags.BriefcaseFilename) 121 | app.Flag("shutdown-trigger-file", "When running as a daemon, the presence of this file will cause the daemon to stop").StringVar(&flags.ShutdownTriggerFile) 122 | app.Flag("one-shot", "Combined with --sidecar, will perform one iteration of work and exit. For crontabs, etc.").Default("false").BoolVar(&flags.PerformOneShot) 123 | 124 | app.Flag("cleanup", "Using the leases file, erase any created output files.").Default("false").BoolVar(&flags.PerformCleanup) 125 | app.Flag("revoke", "During --cleanup, revoke the Vault authentication token.").Default("false").BoolVar(&flags.RevokeOnCleanup) 126 | 127 | // Sidecar options 128 | app.Flag("sidecar", "Run in side-car mode, refreshing leases as needed.").Default("false").BoolVar(&flags.PerformSidecar) 129 | app.Flag("renew-lease-duration", "unused, kept for compatibility").Default("1h").Duration() 130 | app.Flag("vault-token", "Vault token to use during initialization; overrides VAULT_TOKEN environment variable").StringVar(&flags.VaultTokenArg) 131 | app.Flag("token-renewable", "Is the token supplied on the command line renewable?").Default("true").BoolVar(&flags.CliVaultTokenRenewable) 132 | app.Flag("force-refresh-ttl", "If set, secrets will be refreshed after this period regardless of whether they are set to expire (just uses tokenn TTL if zero)").Default("0s").DurationVar(&flags.ForceRefreshTTL) 133 | 134 | // Kubernetes Authentication 135 | app.Flag("k8s-token-file", "Service account token path").Default("/var/run/secrets/kubernetes.io/serviceaccount/token").StringVar(&flags.ServiceAccountToken) 136 | app.Flag("k8s-login-path", "Vault path to authenticate against").Default(os.Getenv("K8S_LOGIN_PATH")).StringVar(&flags.KubernetesLoginPath) 137 | app.Flag("k8s-auth-role", "Kubernetes authentication role").StringVar(&flags.KubernetesAuthRole) 138 | 139 | // EC2 Authentication 140 | app.Flag("ec2-auth", "Use EC2 metadata to authenticate to Vault").Default("false").BoolVar(&flags.EC2AuthEnabled) 141 | app.Flag("ec2-vault-nonce", "Nonce to use if re-authenticating.").Default("").StringVar(&flags.EC2Nonce) 142 | 143 | // IAM Authentication 144 | app.Flag("iam-auth-role", "The role used to perform iam authentication").Default("").StringVar(&flags.IAMAuthRole) 145 | app.Flag("iam-vault-auth-backend", "The name of the auth backend in Vault to perform iam authentication against. Defaults to `aws`.").Default("aws").StringVar(&flags.IAMVaultAuthBackend) 146 | 147 | // STS Authentication 148 | app.Flag("sts-ttl", "The TTL to use for generating AWS STS tokens, if set to zero then will not override TTL. Defaults to 0").Default("0s").DurationVar(&flags.STSTTL) 149 | 150 | // Show version 151 | app.Flag("version", "Display build version").Default("false").BoolVar(&flags.ShowVersion) 152 | 153 | // Shared options 154 | app.Flag("debug", "Log at debug level").Default("false").BoolVar(&flags.DebugLogLevel) 155 | 156 | // Flags for smoothing out edge cases. 157 | app.Flag("ignore-non-renewable-auth", "ignored; kept for compatibility").Default("false").Bool() 158 | app.Flag("never-scrub", "ignored; kept for compatibility").Default("false").Bool() 159 | 160 | // Metrics options 161 | app.Flag("enable-prometheus-metrics", "enables prometheus metrics to be served on prometheus-metrics port").Default("false").BoolVar(&flags.EnablePrometheusMetrics) 162 | app.Flag("prometheus-port", "specifies prometheus metrics port").Default("9191").IntVar(&flags.PrometheusPort) 163 | 164 | // Vault client options 165 | app.Flag("vault-client-timeout", "timeout duration for vault client HTTP timeouts").Default("30s").DurationVar(&flags.VaultClientTimeout) 166 | app.Flag("vault-client-retries", "number of retries to be performed for vault client operations").Default("2").IntVar(&flags.VaultClientRetries) 167 | 168 | // Sidecar mode options 169 | app.Flag("terminate-on-sync-failure", "if enabled in sidecar mode, will cause tool to terminate if there is a failure to perform sync").Default("true").BoolVar(&flags.TerminateOnSyncFailure) 170 | 171 | _, err := app.Parse(args) 172 | if err != nil { 173 | return nil, fmt.Errorf("could not parse arguments: %w", err) 174 | } 175 | 176 | if flags.EC2AuthEnabled && flags.IAMAuthRole != "" { 177 | return nil, errors.New("specify exactly one of --ec2-auth or --iam-auth-role") 178 | } 179 | 180 | actions := 0 181 | if flags.PerformInit { 182 | actions++ 183 | 184 | if flags.PerformOneShot { 185 | return nil, errors.New("the --one-shot flag can only be used in --sidecar mode") 186 | } 187 | } 188 | 189 | if flags.PerformSidecar { 190 | actions++ 191 | } 192 | 193 | if flags.ShowVersion { 194 | actions++ 195 | } 196 | 197 | if flags.PerformCleanup { 198 | actions++ 199 | if flags.PerformOneShot { 200 | return nil, errors.New("the --one-shot flag can only be used in --sidecar mode") 201 | } 202 | } 203 | 204 | if actions != 1 { 205 | return nil, errors.New("specify exactly one of --init, --sidecar, --version or --cleanup flags") 206 | } 207 | 208 | return &flags, nil 209 | } 210 | -------------------------------------------------------------------------------- /util/locking.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "syscall" 9 | 10 | zlog "github.com/rs/zerolog/log" 11 | ) 12 | 13 | type LockHandle struct { 14 | filename string 15 | osFile *os.File 16 | locked bool 17 | } 18 | 19 | // LockFile sets an exclusive provisional file lock on a file (creating it if needed). It's basically a wrapper 20 | // around flock(, LOCK_EX), but hides the file descriptor from the caller since file descriptors aren't very Go-like. 21 | // Returns a non-nil lock handle which can be passed to lh.Unlock(). Note that "Unlock" will attempt to delete the file. 22 | func LockFile(filename string) (*LockHandle, error) { 23 | 24 | absFilename, err := filepath.Abs(filename) 25 | if err != nil { 26 | return nil, fmt.Errorf("could not determine absolute path of %q: %w", filename, err) 27 | } 28 | 29 | MustMkdirAllForFile(filename) 30 | 31 | fp, err := os.Create(absFilename) 32 | if err != nil { 33 | return nil, fmt.Errorf("could not open file %q to lock: %w", absFilename, err) 34 | } 35 | 36 | zlog.Debug().Str("lockfile", absFilename).Msg("attempting exclusive lock") 37 | if err := syscall.Flock(int(fp.Fd()), syscall.LOCK_EX); err != nil { 38 | return nil, fmt.Errorf("could not flock file %q: %w", absFilename, err) 39 | } 40 | zlog.Debug().Str("lockfile", absFilename).Msg("acquired exclusive lock") 41 | 42 | return &LockHandle{ 43 | filename: absFilename, 44 | osFile: fp, 45 | locked: true, 46 | }, nil 47 | } 48 | 49 | // Unlock calls flock(, LOCK_UN) on the file being used for locking. If panicOnUnlockFailure is true, and the 50 | // syscall to unlock it fails, it will panic (vs just return an error). The panic is only for the flock syscall, 51 | // other errors (already unlocked / bad args / couldn't delete file, etc) will always be returned as an error. 52 | func (lh *LockHandle) Unlock(panicOnUnlockFailure bool) error { 53 | if lh == nil { 54 | return errors.New("cannot unlock nil LockHandle") 55 | } 56 | 57 | if !lh.locked { 58 | return fmt.Errorf("multiple calls to unlock file %q", lh.filename) 59 | } 60 | 61 | zlog.Debug().Str("lockfile", lh.filename).Msg("going to unlock lockfile") 62 | if err := syscall.Flock(int(lh.osFile.Fd()), syscall.LOCK_UN); err != nil { 63 | wrapped := fmt.Errorf("could not release exclusive lock on %q: %w", lh.filename, err) 64 | if panicOnUnlockFailure { 65 | panic(wrapped) 66 | } else { 67 | return wrapped 68 | } 69 | } 70 | lh.locked = false 71 | zlog.Debug().Str("lockfile", lh.filename).Msg("lockfile unlocked") 72 | 73 | if err := lh.osFile.Close(); err != nil { 74 | return fmt.Errorf("failed to close file %q after unlocking successfully: %w", lh.filename, err) 75 | } 76 | 77 | // I don't care if we can't delete the lock file. 78 | _ = os.Remove(lh.filename) 79 | 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /util/modes.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | ) 7 | 8 | func StringToFileMode(fileMode string) (*os.FileMode, error) { 9 | var mode os.FileMode 10 | 11 | if fileMode == "" { 12 | mode = os.FileMode(0400) 13 | } else { 14 | i, err := strconv.ParseInt(fileMode, 8, 32) 15 | 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | mode = os.FileMode(int32(i)) 21 | } 22 | 23 | return &mode, nil 24 | } 25 | -------------------------------------------------------------------------------- /util/modes_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestStringToFileMode(t *testing.T) { 9 | if mode, e := StringToFileMode(""); mode == nil || *mode != os.FileMode(0400) || e != nil { 10 | t.Errorf("Empty file mode must default to 0400, not %v.", *mode) 11 | } 12 | 13 | if mode, e := StringToFileMode("777"); mode == nil || *mode != os.FileMode(0777) || e != nil { 14 | t.Errorf("Mode 777 should yield a filemode of 0777, not %v.", *mode) 15 | } 16 | 17 | if mode, e := StringToFileMode("a=rwx"); mode != nil || e == nil { 18 | t.Error("Symbolic mode is not supported.") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /util/paths.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "path/filepath" 7 | 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func AbsolutePath(prefix string, filename string) string { 12 | var calcPath string 13 | 14 | if path.IsAbs(filename) { 15 | // .Abs calls Clean, so even though this is absolute, we still run it through .Abs to 16 | // remove multiple slashes, ..'s, etc. 17 | calcPath = filename 18 | } else { 19 | if prefix != "" { 20 | calcPath = path.Join(prefix, filename) 21 | } else { 22 | calcPath = filename 23 | } 24 | } 25 | 26 | abs, err := filepath.Abs(calcPath) 27 | if err != nil { 28 | log.Fatal().Err(err).Str("prefix", prefix).Str("calculatedPath", calcPath).Msg("could not determine absolute path") 29 | } 30 | 31 | return abs 32 | } 33 | 34 | func MustMkdirAllForFile(filename string) { 35 | err := os.MkdirAll(filepath.Dir(filename), os.ModePerm) 36 | if err != nil { 37 | log.Fatal().Str("filename", filename).Err(err).Msg("failed to create all needed directories") 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /util/touch.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | "time" 6 | ) 7 | 8 | func TouchFile(absFilename string) error { 9 | _, err := os.Stat(absFilename) 10 | if os.IsNotExist(err) { 11 | file, err := os.Create(absFilename) 12 | 13 | if err != nil { 14 | return err 15 | } 16 | defer file.Close() 17 | } else { 18 | currentTime := time.Now().Local() 19 | err = os.Chtimes(absFilename, currentTime, currentTime) 20 | if err != nil { 21 | return err 22 | } 23 | } 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /util/wrapped_token.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "github.com/hashicorp/vault/api" 4 | 5 | type WrappedToken struct { 6 | *api.Secret 7 | Renewable bool 8 | } 9 | 10 | func NewWrappedToken(secret *api.Secret, renewable bool) *WrappedToken { 11 | return &WrappedToken{ 12 | Secret: secret, 13 | Renewable: renewable, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /vaultclient/auth.go: -------------------------------------------------------------------------------- 1 | package vaultclient 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 8 | "github.com/rs/zerolog" 9 | zlog "github.com/rs/zerolog/log" 10 | ) 11 | 12 | type authenticator struct { 13 | log zerolog.Logger 14 | vaultClient VaultClient 15 | } 16 | 17 | type ec2amiAuthenticator struct { 18 | authenticator 19 | // ec2ami 20 | ec2Nonce string 21 | } 22 | 23 | type ec2iamAuthenticator struct { 24 | authenticator 25 | // ec2iam 26 | awsRegion string 27 | iamAuthRole string 28 | iamVaultAuthBackend string 29 | } 30 | 31 | type kubernetesAuthenticator struct { 32 | authenticator 33 | // kubernetes 34 | serviceAccountToken string 35 | k8sLoginPath string 36 | k8sAuthRole string 37 | } 38 | type Authenticator interface { 39 | Authenticate() (*util.WrappedToken, error) 40 | } 41 | 42 | func NewAuthenticator(client VaultClient, cliFlags util.CliFlags) (Authenticator, error) { 43 | log := zlog.With().Str("vaultAddr", client.Address()).Logger() 44 | 45 | shared := authenticator{ 46 | log: log, 47 | vaultClient: client, 48 | } 49 | 50 | mechanism := cliFlags.AuthMechanism() 51 | switch mechanism { 52 | case util.EC2IAMAuth: 53 | region := os.Getenv("AWS_DEFAULT_REGION") 54 | if region == "" { 55 | log.Debug().Msg("using hardcoded us-east-1 region") 56 | region = "us-east-1" 57 | } 58 | authn := &ec2iamAuthenticator{ 59 | authenticator: shared, 60 | awsRegion: region, 61 | iamAuthRole: cliFlags.IAMAuthRole, 62 | iamVaultAuthBackend: cliFlags.IAMVaultAuthBackend, 63 | } 64 | return authn, nil 65 | case util.EC2AMIAuth: 66 | authn := &ec2amiAuthenticator{ 67 | authenticator: shared, 68 | ec2Nonce: cliFlags.EC2Nonce, 69 | } 70 | return authn, nil 71 | case util.KubernetesAuth: 72 | authn := &kubernetesAuthenticator{ 73 | authenticator: shared, 74 | serviceAccountToken: cliFlags.ServiceAccountToken, 75 | k8sLoginPath: cliFlags.KubernetesLoginPath, 76 | k8sAuthRole: cliFlags.KubernetesAuthRole, 77 | } 78 | return authn, nil 79 | case util.UnknownAuth: 80 | return nil, fmt.Errorf("no authentication mechanism specified") 81 | default: 82 | log.Error().Interface("mechanism", mechanism).Msg("internal error: un-coded authentication mechanism") 83 | return nil, fmt.Errorf("internal error: un-coded authentication mechanism: %v", mechanism) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /vaultclient/auth_aws_ami.go: -------------------------------------------------------------------------------- 1 | package vaultclient 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "strings" 9 | 10 | "github.com/hashicorp/vault/api" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 12 | ) 13 | 14 | func (auth *ec2amiAuthenticator) Authenticate() (*util.WrappedToken, error) { 15 | 16 | secret, err := auth.performEC2AMIAuth() 17 | if err != nil { 18 | auth.log.Error().Err(err).Msg("ec2 ami authentication failed") 19 | return nil, err 20 | } 21 | return secret, nil 22 | } 23 | 24 | func (auth *ec2amiAuthenticator) performEC2AMIAuth() (*util.WrappedToken, error) { 25 | 26 | type login struct { 27 | Role string `json:"role"` 28 | Pkcs7 string `json:"pkcs7"` 29 | Nonce string `json:"nonce,omitempty"` 30 | } 31 | 32 | pkcs7, err := auth.fetchPKCS7() 33 | 34 | if err != nil { 35 | return nil, err 36 | } 37 | auth.log.Debug().Int("len(pkcs7)", len(pkcs7)).Msg("fetched PKCS7 payload") 38 | 39 | ami, err := auth.fetchAMI() 40 | if err != nil { 41 | return nil, err 42 | } 43 | auth.log.Debug().Str("ami", ami).Msg("found current AMI") 44 | 45 | req := auth.vaultClient.Delegate().NewRequest(http.MethodPost, util.VaultEC2AuthPath) 46 | 47 | authValues := login{Role: ami, Pkcs7: pkcs7, Nonce: auth.ec2Nonce} 48 | err = req.SetJSONBody(authValues) 49 | if err != nil { 50 | auth.log.Error().Err(err).Str("ami", ami).Msg("failed to create authentication request") 51 | return nil, fmt.Errorf("failed to create authentication request: %w", err) 52 | } 53 | 54 | auth.log.Info().Str("url", req.URL.String()).Str("ami", ami).Msg("sending EC2 AMI request") 55 | 56 | response, err := auth.vaultClient.Delegate().RawRequest(req) 57 | if err != nil { 58 | auth.log.Error().Err(err).Msg("failed to process authentication request") 59 | return nil, err 60 | } 61 | 62 | if response.Error() != nil { 63 | auth.log.Error().Err(response.Error()).Msg("authentication request failed") 64 | return nil, fmt.Errorf("authentication request failed: %w", response.Error()) 65 | } 66 | 67 | var secret api.Secret 68 | 69 | err = json.NewDecoder(response.Body).Decode(&secret) 70 | if err != nil { 71 | return nil, fmt.Errorf("could not parse response: %w", err) 72 | } 73 | 74 | return util.NewWrappedToken(&secret, true), nil 75 | } 76 | 77 | func (auth *ec2amiAuthenticator) fetchAMI() (string, error) { 78 | resp, err := http.Get("http://169.254.169.254/latest/meta-data/ami-id") 79 | 80 | if err != nil { 81 | return "", err 82 | } 83 | 84 | defer resp.Body.Close() 85 | 86 | body, err := ioutil.ReadAll(resp.Body) 87 | 88 | if err != nil { 89 | return "", err 90 | } 91 | 92 | return string(body), nil 93 | } 94 | 95 | func (auth *ec2amiAuthenticator) fetchPKCS7() (string, error) { 96 | resp, err := http.Get("http://169.254.169.254/latest/dynamic/instance-identity/pkcs7") 97 | 98 | if err != nil { 99 | return "", err 100 | } 101 | 102 | defer resp.Body.Close() 103 | 104 | body, err := ioutil.ReadAll(resp.Body) 105 | 106 | if err != nil { 107 | return "", err 108 | } 109 | 110 | pkcs7 := strings.Replace(string(body), "\n", "", -1) 111 | 112 | return pkcs7, nil 113 | } 114 | -------------------------------------------------------------------------------- /vaultclient/auth_aws_iam.go: -------------------------------------------------------------------------------- 1 | package vaultclient 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 8 | "io/ioutil" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/credentials" 12 | "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" 13 | "github.com/aws/aws-sdk-go/aws/ec2metadata" 14 | "github.com/aws/aws-sdk-go/aws/endpoints" 15 | "github.com/aws/aws-sdk-go/aws/session" 16 | "github.com/aws/aws-sdk-go/service/sts" 17 | "github.com/hashicorp/vault/api" 18 | ) 19 | 20 | func (auth *ec2iamAuthenticator) Authenticate() (*util.WrappedToken, error) { 21 | secret, err := auth.performEC2IAMAuth() 22 | if err != nil { 23 | auth.log.Error().Err(err).Msg("ec2 iam authentication failed") 24 | return nil, err 25 | } 26 | return secret, nil 27 | } 28 | 29 | //stsSigningResolver is borrowed from https://github.com/hashicorp/vault/blob/master/builtin/credential/aws/cli.go 30 | func (auth *ec2iamAuthenticator) stsSigningResolver(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) { 31 | defaultEndpoint, err := endpoints.DefaultResolver().EndpointFor(service, region, optFns...) 32 | if err != nil { 33 | return defaultEndpoint, err 34 | } 35 | 36 | defaultEndpoint.SigningRegion = region 37 | return defaultEndpoint, nil 38 | } 39 | 40 | //generateLoginData is borrowed from https://github.com/hashicorp/vault/blob/master/builtin/credential/aws/cli.go 41 | func (auth *ec2iamAuthenticator) generateLoginData(creds *credentials.Credentials, configuredRegion string) (map[string]interface{}, error) { 42 | loginData := make(map[string]interface{}) 43 | 44 | stsSession, err := session.NewSessionWithOptions(session.Options{ 45 | Config: aws.Config{ 46 | Credentials: creds, 47 | Region: &configuredRegion, 48 | EndpointResolver: endpoints.ResolverFunc(auth.stsSigningResolver), 49 | }, 50 | }) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | var params *sts.GetCallerIdentityInput 56 | svc := sts.New(stsSession) 57 | stsRequest, _ := svc.GetCallerIdentityRequest(params) 58 | 59 | if err := stsRequest.Sign(); err != nil { 60 | return nil, err 61 | } 62 | 63 | // Now extract out the relevant parts of the request 64 | headersJSON, err := json.Marshal(stsRequest.HTTPRequest.Header) 65 | if err != nil { 66 | return nil, err 67 | } 68 | requestBody, err := ioutil.ReadAll(stsRequest.HTTPRequest.Body) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | loginData["iam_http_request_method"] = stsRequest.HTTPRequest.Method 74 | loginData["iam_request_url"] = base64.StdEncoding.EncodeToString([]byte(stsRequest.HTTPRequest.URL.String())) 75 | loginData["iam_request_headers"] = base64.StdEncoding.EncodeToString(headersJSON) 76 | loginData["iam_request_body"] = base64.StdEncoding.EncodeToString(requestBody) 77 | 78 | return loginData, nil 79 | } 80 | 81 | func (auth *ec2iamAuthenticator) getSecret(creds *credentials.Credentials) (*api.Secret, error) { 82 | 83 | loginData, err := auth.generateLoginData(creds, auth.awsRegion) 84 | if err != nil { 85 | return nil, err 86 | } 87 | if loginData == nil { 88 | return nil, fmt.Errorf("got nil response from generateLoginData") 89 | } 90 | 91 | loginData["role"] = auth.iamAuthRole 92 | 93 | secret, err := auth.vaultClient.Delegate().Logical().Write(fmt.Sprintf("auth/%s/login", auth.iamVaultAuthBackend), loginData) 94 | if err != nil { 95 | return nil, err 96 | } 97 | if secret == nil { 98 | return nil, fmt.Errorf("empty response from credential provider") 99 | } 100 | 101 | return secret, nil 102 | } 103 | 104 | func (auth *authenticator) getCredentialsFromRole() (*credentials.Credentials, error) { 105 | awsSession, err := session.NewSession() 106 | if err != nil { 107 | return nil, fmt.Errorf("could not create a new session to use with the AWS SDK: %w", err) 108 | } 109 | 110 | roleProvider := &ec2rolecreds.EC2RoleProvider{ 111 | Client: ec2metadata.New(awsSession), 112 | } 113 | creds := credentials.NewCredentials(roleProvider) 114 | 115 | _, err = creds.Get() 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | return creds, nil 121 | } 122 | 123 | func (auth *ec2iamAuthenticator) performEC2IAMAuth() (*util.WrappedToken, error) { 124 | 125 | auth.log.Info().Msg("starting authenticating with IAM role") 126 | 127 | creds, err := auth.getCredentialsFromRole() 128 | if err != nil { 129 | return nil, fmt.Errorf("could not get IAM Role credentials: %w", err) 130 | } 131 | 132 | auth.log.Info().Str("role", auth.iamAuthRole).Str("vault_auth_path", auth.iamVaultAuthBackend).Msg("performing authentication") 133 | 134 | secret, err := auth.getSecret(creds) 135 | 136 | if err != nil { 137 | return nil, fmt.Errorf("could not authenticate to vault using IAM role authentication: %w", err) 138 | } 139 | 140 | return util.NewWrappedToken(secret, true), nil 141 | } 142 | -------------------------------------------------------------------------------- /vaultclient/auth_kubernetes.go: -------------------------------------------------------------------------------- 1 | package vaultclient 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | v12 "k8s.io/api/core/v1" 10 | "net/http" 11 | "strconv" 12 | 13 | "github.com/hashicorp/vault/api" 14 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 15 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/client-go/kubernetes" 17 | "k8s.io/client-go/rest" 18 | ) 19 | 20 | func (auth *kubernetesAuthenticator) Authenticate() (*util.WrappedToken, error) { 21 | secret, err := auth.performKubernetesAuth() 22 | if err != nil { 23 | auth.log.Error().Err(err).Msg("kubernetes authentication failed") 24 | return nil, err 25 | } 26 | 27 | return secret, nil 28 | } 29 | 30 | func (auth *kubernetesAuthenticator) performKubernetesAuth() (*util.WrappedToken, error) { 31 | type login struct { 32 | JWT string `json:"jwt"` 33 | Role string `json:"role"` 34 | } 35 | 36 | secret, err := auth.tryHardCodedToken() 37 | 38 | if err == nil { 39 | return secret, nil 40 | } 41 | 42 | auth.log.Debug().Err(err).Msg("could not authenticate using hard-coded ConfigMap vault-token - ignoring") 43 | 44 | auth.log.Info().Str("serviceAccountToken", auth.serviceAccountToken).Msg("reading service account token") 45 | 46 | tokenBytes, err := ioutil.ReadFile(auth.serviceAccountToken) 47 | if err != nil { 48 | return nil, fmt.Errorf("could not read service account token file %q: %w", auth.serviceAccountToken, err) 49 | } 50 | 51 | auth.log.Info().Str("authPath", auth.k8sLoginPath).Str("k8sRole", auth.k8sAuthRole).Msg("authenticating") 52 | 53 | req := auth.vaultClient.Delegate().NewRequest(http.MethodPost, fmt.Sprintf("/v1/auth/%s/login", auth.k8sLoginPath)) 54 | err = req.SetJSONBody(&login{JWT: string(tokenBytes), Role: auth.k8sAuthRole}) 55 | if err != nil { 56 | return nil, fmt.Errorf("failed to parse JSON body: %w", err) 57 | } 58 | 59 | resp, err := auth.vaultClient.Delegate().RawRequest(req) 60 | if err != nil { 61 | return nil, fmt.Errorf("failed to perform Kubernetes auth request: %w", err) 62 | } 63 | 64 | if resp.Error() != nil { 65 | return nil, resp.Error() 66 | } 67 | 68 | var body api.Secret 69 | 70 | err = json.NewDecoder(resp.Body).Decode(&body) 71 | if err != nil { 72 | return nil, fmt.Errorf("error parsing response: %w", err) 73 | } 74 | 75 | return util.NewWrappedToken(&body, true), nil 76 | } 77 | 78 | // If there is a ConfigMap named vault-token in the default namespace, use the token it stores 79 | // Developers running Kubernetes clusters locally do not have the ability to have their services authenticate to Vault. 80 | // To work around this, the bootstrapping shell scripts for dev clusters create a configmap called "vault-token" 81 | // with their Vault token in it. This stanza checks for that special configmap and uses it. 82 | func (auth *kubernetesAuthenticator) tryHardCodedToken() (*util.WrappedToken, error) { 83 | if util.EnableKubernetesVaultTokenAuthentication { 84 | config, err := rest.InClusterConfig() 85 | // If we cannot create the in cluster config, that means we are not running inside of Kubernetes 86 | if err != nil { 87 | return nil, fmt.Errorf("could not create cluster config - this will fail if this is running outside of Kubernetes: %w", err) 88 | } 89 | 90 | clientset, err := kubernetes.NewForConfig(config) 91 | if err != nil { 92 | return nil, fmt.Errorf("could not create ClientSet to call Kubernetes API: %w", err) 93 | } 94 | 95 | configMaps, err := clientset.CoreV1().ConfigMaps("default").List(context.Background(), v1.ListOptions{FieldSelector: "metadata.name=vault-token"}) 96 | if err != nil { 97 | return nil, fmt.Errorf("failed to get ConfigMaps filtered on the metadata.name=vault-token") 98 | } else if len(configMaps.Items) == 1 { 99 | return auth.ProcessConfigMap(configMaps.Items[0]) 100 | } else { 101 | return nil, errors.New("multiple ConfigMaps were returned when filtering ConfigMaps with metadata.name=vault-token; please remove all but one") 102 | } 103 | } 104 | 105 | return nil, errors.New("hard coded vault-token ConfigMap disabled") 106 | } 107 | 108 | func (auth *kubernetesAuthenticator) ProcessConfigMap(item v12.ConfigMap) (*util.WrappedToken, error) { 109 | if token, exists := item.Data["token"]; exists { 110 | auth.log.Info().Msg("logging into Vault server %q with token from vault-token ConfigMap") 111 | 112 | secret, err := auth.vaultClient.VerifyVaultToken(token) 113 | if err != nil { 114 | return nil, fmt.Errorf("failed to authenticate to Vault server %q using token from vault-token ConfigMap: %w", auth.vaultClient.Address(), err) 115 | } 116 | if secret == nil { 117 | return nil, fmt.Errorf("got nil secret authenticating to Vault Server %q using token from vault-token ConfigMap", auth.vaultClient.Address()) 118 | } 119 | 120 | // For backwards compatibility, tokens are expected to be renewable. This can be overridden if "renewable: false" is set in the configmap. 121 | renewable := true 122 | 123 | if renewableOverride, exists := item.Data["renewable"]; exists { 124 | if renewable, err = strconv.ParseBool(renewableOverride); err != nil { 125 | return nil, fmt.Errorf("ConfigMap vault-token key \"renewable\" has value %q which cannot be parsed as boolean :%w", 126 | renewableOverride, err) 127 | } 128 | } 129 | 130 | if !renewable { 131 | auth.log.Info().Msg("using non-renewable Vault token from Kubernetes ConfigMap") 132 | } 133 | return util.NewWrappedToken(secret, renewable), nil 134 | } else { 135 | return nil, errors.New("missing token field in vault-token ConfigMap") 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /vaultclient/creds_aws_sts.go: -------------------------------------------------------------------------------- 1 | package vaultclient 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "time" 7 | 8 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 9 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 10 | ) 11 | 12 | type AWSSTSCredential struct { 13 | AccessKey string 14 | SecretKey string 15 | SessionToken string 16 | } 17 | 18 | func (vc *wrappedVaultClient) FetchAWSSTSCredential(awsConfig config.AWSType, stsTTL time.Duration) (*AWSSTSCredential, *util.WrappedToken, error) { 19 | 20 | path := filepath.Join(awsConfig.VaultMountPoint, "creds", awsConfig.VaultRole) 21 | 22 | log := vc.log.With().Str("path", path). 23 | Str("outputPath", awsConfig.OutputPath).Logger() 24 | 25 | log.Info().Msg("fetching AWS STS credentials") 26 | 27 | var data map[string]interface{} 28 | if stsTTL != 0 { // use default if ttl is 0. 29 | data = map[string]interface{}{ 30 | "ttl": stsTTL.String(), 31 | } 32 | } 33 | 34 | result, err := vc.Delegate().Logical().Write(path, data) 35 | if err != nil { 36 | log.Error().Err(err).Msg("failed to fetch AWS credentials") 37 | return nil, nil, fmt.Errorf("could not fetch AWS credentials from %q: %w", path, err) 38 | } 39 | 40 | accessKey := result.Data["access_key"] 41 | secretKey := result.Data["secret_key"] 42 | // aka sessionToken 43 | securityToken := result.Data["security_token"] 44 | 45 | log.Debug().Interface("accessKey", accessKey).Msg("received AWS access key") 46 | 47 | return &AWSSTSCredential{ 48 | AccessKey: accessKey.(string), 49 | SecretKey: secretKey.(string), 50 | SessionToken: securityToken.(string), 51 | }, util.NewWrappedToken(result, true), nil 52 | } 53 | -------------------------------------------------------------------------------- /vaultclient/creds_sshcert.go: -------------------------------------------------------------------------------- 1 | package vaultclient 2 | 3 | import ( 4 | "crypto/rand" 5 | "crypto/rsa" 6 | "crypto/x509" 7 | "encoding/pem" 8 | "fmt" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "syscall" 13 | 14 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 15 | 16 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 17 | "github.com/rs/zerolog" 18 | "golang.org/x/crypto/ssh" 19 | ) 20 | 21 | // SSHPrivateKey is the name of the output file with the the SSH private key (think: ssh -i id_rsa ....). 22 | const SSHPrivateKey = "id_rsa" 23 | 24 | // SSHPublicKey is the corresponding public key, used for signing. 25 | const SSHPublicKey = "id_rsa.pub" 26 | 27 | func (vc *wrappedVaultClient) CreateSSHCertificate(ssh config.SSHCertificateType) error { 28 | 29 | log := vc.log.With().Str("vaultRole", ssh.VaultRole).Logger() 30 | 31 | privateKeyFilename := filepath.Join(ssh.OutputPath, SSHPrivateKey) 32 | publicKeyFilename := filepath.Join(ssh.OutputPath, SSHPublicKey) 33 | 34 | // I'd use util.MustMakeDirAllForFile, but I want to set the directory permission 35 | if err := os.MkdirAll(ssh.OutputPath, 0700); err != nil { 36 | return fmt.Errorf("could not make directory path %q: %w", ssh.OutputPath, err) 37 | } 38 | 39 | log.Info().Str("privateKey", privateKeyFilename).Str("publicKey", publicKeyFilename).Msg("generating SSH keypair") 40 | 41 | if err := vc.generateKeyPair(privateKeyFilename, publicKeyFilename); err != nil { 42 | return fmt.Errorf("failed to generate SSH keys: %w", err) 43 | } 44 | if err := vc.signKey(log, ssh.OutputPath, ssh.VaultMount, ssh.VaultRole); err != nil { 45 | return fmt.Errorf("failed to sign SSH key: %w", err) 46 | } 47 | 48 | return nil 49 | } 50 | 51 | func (vc *wrappedVaultClient) generateKeyPair(privateKeyFilename, publicKeyFilename string) error { 52 | 53 | privateKey, err := rsa.GenerateKey(rand.Reader, 4096) 54 | if err != nil { 55 | return fmt.Errorf("could not generate RSA key: %w", err) 56 | } 57 | 58 | // Write a SSH private key.. 59 | privateKeyFile, err := os.OpenFile(privateKeyFilename, syscall.O_RDWR|syscall.O_CREAT|syscall.O_TRUNC, 0600) 60 | if err != nil { 61 | return fmt.Errorf("could not create private key file %q: %w", privateKeyFilename, err) 62 | } 63 | defer privateKeyFile.Close() 64 | 65 | privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} 66 | if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { 67 | return fmt.Errorf("could not PEM encode private key %q: %w", privateKeyFilename, err) 68 | } 69 | 70 | // Write SSH public key.. 71 | pub, err := ssh.NewPublicKey(&privateKey.PublicKey) 72 | if err != nil { 73 | return fmt.Errorf("could not create public SSH key %q: %w", publicKeyFilename, err) 74 | } 75 | 76 | err = ioutil.WriteFile(publicKeyFilename, ssh.MarshalAuthorizedKey(pub), 0600) 77 | if err != nil { 78 | return fmt.Errorf("could not write public SSH key %q: %w", publicKeyFilename, err) 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func (vc *wrappedVaultClient) signKey(log zerolog.Logger, outputPath string, vaultMount string, vaultRole string) error { 85 | log.Debug().Str("outputPath", outputPath).Str("vaultMount", vaultMount).Msg("signing SSH keys") 86 | 87 | vaultSSH := vc.Delegate().SSHWithMountPoint(vaultMount) 88 | 89 | publicKeyFilename := filepath.Join(outputPath, SSHPublicKey) 90 | certificateFilename := filepath.Join(outputPath, util.SSHCertificate) 91 | 92 | publicKeyBytes, err := ioutil.ReadFile(publicKeyFilename) 93 | if err != nil { 94 | return fmt.Errorf("could not read SSH public key %q: %w", publicKeyFilename, err) 95 | } 96 | 97 | resp, err := vaultSSH.SignKey(vaultRole, map[string]interface{}{ 98 | "public_key": string(publicKeyBytes), 99 | }) 100 | if err != nil { 101 | return fmt.Errorf("failed to sign SSH key: %w", err) 102 | } 103 | 104 | signedKey := resp.Data["signed_key"] 105 | if signedKey == nil { 106 | return fmt.Errorf("did not receive a signed_key from Vault at %q when signing key at %q with \"%s/sign/%s\"", 107 | vc.Address(), outputPath, vaultMount, vaultRole) 108 | } 109 | signedKeyString, ok := resp.Data["signed_key"].(string) 110 | if !ok { 111 | return fmt.Errorf("could not convert signed_key to string") 112 | } 113 | 114 | log.Info().Str("certificateFile", certificateFilename).Msg("writing SSH certificate") 115 | 116 | if err := ioutil.WriteFile(certificateFilename, []byte(signedKeyString), 0600); err != nil { 117 | return fmt.Errorf("could not write certificate file: %w", err) 118 | } 119 | 120 | return nil 121 | } 122 | -------------------------------------------------------------------------------- /vaultclient/mocks/vaultclient.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: vaultclient/vaultclient.go 3 | 4 | // Package mock_vaultclient is a generated GoMock package. 5 | package mock_vaultclient 6 | 7 | import ( 8 | reflect "reflect" 9 | time "time" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | api "github.com/hashicorp/vault/api" 13 | config "github.com/hootsuite/vault-ctrl-tool/v2/config" 14 | util "github.com/hootsuite/vault-ctrl-tool/v2/util" 15 | vaultclient "github.com/hootsuite/vault-ctrl-tool/v2/vaultclient" 16 | ) 17 | 18 | // MockVaultClient is a mock of VaultClient interface. 19 | type MockVaultClient struct { 20 | ctrl *gomock.Controller 21 | recorder *MockVaultClientMockRecorder 22 | } 23 | 24 | // MockVaultClientMockRecorder is the mock recorder for MockVaultClient. 25 | type MockVaultClientMockRecorder struct { 26 | mock *MockVaultClient 27 | } 28 | 29 | // NewMockVaultClient creates a new mock instance. 30 | func NewMockVaultClient(ctrl *gomock.Controller) *MockVaultClient { 31 | mock := &MockVaultClient{ctrl: ctrl} 32 | mock.recorder = &MockVaultClientMockRecorder{mock} 33 | return mock 34 | } 35 | 36 | // EXPECT returns an object that allows the caller to indicate expected use. 37 | func (m *MockVaultClient) EXPECT() *MockVaultClientMockRecorder { 38 | return m.recorder 39 | } 40 | 41 | // Address mocks base method. 42 | func (m *MockVaultClient) Address() string { 43 | m.ctrl.T.Helper() 44 | ret := m.ctrl.Call(m, "Address") 45 | ret0, _ := ret[0].(string) 46 | return ret0 47 | } 48 | 49 | // Address indicates an expected call of Address. 50 | func (mr *MockVaultClientMockRecorder) Address() *gomock.Call { 51 | mr.mock.ctrl.T.Helper() 52 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Address", reflect.TypeOf((*MockVaultClient)(nil).Address)) 53 | } 54 | 55 | // CreateSSHCertificate mocks base method. 56 | func (m *MockVaultClient) CreateSSHCertificate(sshConfig config.SSHCertificateType) error { 57 | m.ctrl.T.Helper() 58 | ret := m.ctrl.Call(m, "CreateSSHCertificate", sshConfig) 59 | ret0, _ := ret[0].(error) 60 | return ret0 61 | } 62 | 63 | // CreateSSHCertificate indicates an expected call of CreateSSHCertificate. 64 | func (mr *MockVaultClientMockRecorder) CreateSSHCertificate(sshConfig interface{}) *gomock.Call { 65 | mr.mock.ctrl.T.Helper() 66 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateSSHCertificate", reflect.TypeOf((*MockVaultClient)(nil).CreateSSHCertificate), sshConfig) 67 | } 68 | 69 | // Delegate mocks base method. 70 | func (m *MockVaultClient) Delegate() *api.Client { 71 | m.ctrl.T.Helper() 72 | ret := m.ctrl.Call(m, "Delegate") 73 | ret0, _ := ret[0].(*api.Client) 74 | return ret0 75 | } 76 | 77 | // Delegate indicates an expected call of Delegate. 78 | func (mr *MockVaultClientMockRecorder) Delegate() *gomock.Call { 79 | mr.mock.ctrl.T.Helper() 80 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delegate", reflect.TypeOf((*MockVaultClient)(nil).Delegate)) 81 | } 82 | 83 | // FetchAWSSTSCredential mocks base method. 84 | func (m *MockVaultClient) FetchAWSSTSCredential(awsConfig config.AWSType, stsTTL time.Duration) (*vaultclient.AWSSTSCredential, *util.WrappedToken, error) { 85 | m.ctrl.T.Helper() 86 | ret := m.ctrl.Call(m, "FetchAWSSTSCredential", awsConfig, stsTTL) 87 | ret0, _ := ret[0].(*vaultclient.AWSSTSCredential) 88 | ret1, _ := ret[1].(*util.WrappedToken) 89 | ret2, _ := ret[2].(error) 90 | return ret0, ret1, ret2 91 | } 92 | 93 | // FetchAWSSTSCredential indicates an expected call of FetchAWSSTSCredential. 94 | func (mr *MockVaultClientMockRecorder) FetchAWSSTSCredential(awsConfig, stsTTL interface{}) *gomock.Call { 95 | mr.mock.ctrl.T.Helper() 96 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchAWSSTSCredential", reflect.TypeOf((*MockVaultClient)(nil).FetchAWSSTSCredential), awsConfig, stsTTL) 97 | } 98 | 99 | // Read mocks base method. 100 | func (m *MockVaultClient) Read(arg0 string) (*api.Secret, error) { 101 | m.ctrl.T.Helper() 102 | ret := m.ctrl.Call(m, "Read", arg0) 103 | ret0, _ := ret[0].(*api.Secret) 104 | ret1, _ := ret[1].(error) 105 | return ret0, ret1 106 | } 107 | 108 | // Read indicates an expected call of Read. 109 | func (mr *MockVaultClientMockRecorder) Read(arg0 interface{}) *gomock.Call { 110 | mr.mock.ctrl.T.Helper() 111 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockVaultClient)(nil).Read), arg0) 112 | } 113 | 114 | // ReadWithData mocks base method. 115 | func (m *MockVaultClient) ReadWithData(arg0 string, arg1 map[string][]string) (*api.Secret, error) { 116 | m.ctrl.T.Helper() 117 | ret := m.ctrl.Call(m, "ReadWithData", arg0, arg1) 118 | ret0, _ := ret[0].(*api.Secret) 119 | ret1, _ := ret[1].(error) 120 | return ret0, ret1 121 | } 122 | 123 | // ReadWithData indicates an expected call of ReadWithData. 124 | func (mr *MockVaultClientMockRecorder) ReadWithData(arg0, arg1 interface{}) *gomock.Call { 125 | mr.mock.ctrl.T.Helper() 126 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadWithData", reflect.TypeOf((*MockVaultClient)(nil).ReadWithData), arg0, arg1) 127 | } 128 | 129 | // RefreshVaultToken mocks base method. 130 | func (m *MockVaultClient) RefreshVaultToken() (*api.Secret, error) { 131 | m.ctrl.T.Helper() 132 | ret := m.ctrl.Call(m, "RefreshVaultToken") 133 | ret0, _ := ret[0].(*api.Secret) 134 | ret1, _ := ret[1].(error) 135 | return ret0, ret1 136 | } 137 | 138 | // RefreshVaultToken indicates an expected call of RefreshVaultToken. 139 | func (mr *MockVaultClientMockRecorder) RefreshVaultToken() *gomock.Call { 140 | mr.mock.ctrl.T.Helper() 141 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshVaultToken", reflect.TypeOf((*MockVaultClient)(nil).RefreshVaultToken)) 142 | } 143 | 144 | // ServiceSecretPrefix mocks base method. 145 | func (m *MockVaultClient) ServiceSecretPrefix(configVersion int) string { 146 | m.ctrl.T.Helper() 147 | ret := m.ctrl.Call(m, "ServiceSecretPrefix", configVersion) 148 | ret0, _ := ret[0].(string) 149 | return ret0 150 | } 151 | 152 | // ServiceSecretPrefix indicates an expected call of ServiceSecretPrefix. 153 | func (mr *MockVaultClientMockRecorder) ServiceSecretPrefix(configVersion interface{}) *gomock.Call { 154 | mr.mock.ctrl.T.Helper() 155 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServiceSecretPrefix", reflect.TypeOf((*MockVaultClient)(nil).ServiceSecretPrefix), configVersion) 156 | } 157 | 158 | // SetToken mocks base method. 159 | func (m *MockVaultClient) SetToken(token string) { 160 | m.ctrl.T.Helper() 161 | m.ctrl.Call(m, "SetToken", token) 162 | } 163 | 164 | // SetToken indicates an expected call of SetToken. 165 | func (mr *MockVaultClientMockRecorder) SetToken(token interface{}) *gomock.Call { 166 | mr.mock.ctrl.T.Helper() 167 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetToken", reflect.TypeOf((*MockVaultClient)(nil).SetToken), token) 168 | } 169 | 170 | // VerifyVaultToken mocks base method. 171 | func (m *MockVaultClient) VerifyVaultToken(vaultToken string) (*api.Secret, error) { 172 | m.ctrl.T.Helper() 173 | ret := m.ctrl.Call(m, "VerifyVaultToken", vaultToken) 174 | ret0, _ := ret[0].(*api.Secret) 175 | ret1, _ := ret[1].(error) 176 | return ret0, ret1 177 | } 178 | 179 | // VerifyVaultToken indicates an expected call of VerifyVaultToken. 180 | func (mr *MockVaultClientMockRecorder) VerifyVaultToken(vaultToken interface{}) *gomock.Call { 181 | mr.mock.ctrl.T.Helper() 182 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VerifyVaultToken", reflect.TypeOf((*MockVaultClient)(nil).VerifyVaultToken), vaultToken) 183 | } 184 | -------------------------------------------------------------------------------- /vaultclient/vaultclient.go: -------------------------------------------------------------------------------- 1 | package vaultclient 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/hashicorp/vault/api" 8 | "github.com/hootsuite/vault-ctrl-tool/v2/config" 9 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 10 | "github.com/rs/zerolog" 11 | zlog "github.com/rs/zerolog/log" 12 | ) 13 | 14 | const SecretsServicePathV1 = "/secret/application-config/services/" 15 | const SecretsServicePathV2 = "/kv/data/application-config/services/" 16 | 17 | type VaultClient interface { 18 | VerifyVaultToken(vaultToken string) (*api.Secret, error) 19 | Delegate() *api.Client 20 | FetchAWSSTSCredential(awsConfig config.AWSType, stsTTL time.Duration) (*AWSSTSCredential, *util.WrappedToken, error) 21 | CreateSSHCertificate(sshConfig config.SSHCertificateType) error 22 | RefreshVaultToken() (*api.Secret, error) 23 | ServiceSecretPrefix(configVersion int) string 24 | 25 | Address() string 26 | ReadWithData(string, map[string][]string) (*api.Secret, error) 27 | Read(string) (*api.Secret, error) 28 | SetToken(token string) 29 | } 30 | 31 | type wrappedVaultClient struct { 32 | delegate *api.Client 33 | secretsPrefix string 34 | log zerolog.Logger 35 | } 36 | 37 | // NewVaultClient constructs a new VaultClient implementation. 38 | func NewVaultClient(secretsPrefix string, clientTimeout time.Duration, clientRetries int) (VaultClient, error) { 39 | conf := api.DefaultConfig() 40 | // Requests are twice with a jitter backoff. 41 | conf.MaxRetries = clientRetries 42 | conf.Timeout = clientTimeout 43 | 44 | zlog.Debug(). 45 | Int("vaultClientMaxRetries", clientRetries). 46 | Int("vaultClientTimeoutSeconds", int(clientTimeout.Seconds())). 47 | Str("secretsPrefix", secretsPrefix). 48 | Msg("creating Vault client") 49 | 50 | client, err := api.NewClient(conf) 51 | if err != nil { 52 | return nil, fmt.Errorf("could not create Vault client: %w", err) 53 | } 54 | 55 | log := zlog.With().Str("vaultAddr", client.Address()).Logger() 56 | 57 | return &wrappedVaultClient{ 58 | secretsPrefix: secretsPrefix, 59 | delegate: client, 60 | log: log, 61 | }, nil 62 | } 63 | 64 | func (vc *wrappedVaultClient) Delegate() *api.Client { 65 | return vc.delegate 66 | } 67 | 68 | func (vc *wrappedVaultClient) Address() string { 69 | return vc.delegate.Address() 70 | } 71 | 72 | func (vc *wrappedVaultClient) SetToken(token string) { 73 | vc.delegate.SetToken(token) 74 | } 75 | func (vc *wrappedVaultClient) ReadWithData(path string, data map[string][]string) (*api.Secret, error) { 76 | return vc.delegate.Logical().ReadWithData(path, data) 77 | } 78 | 79 | func (vc *wrappedVaultClient) Read(path string) (*api.Secret, error) { 80 | return vc.delegate.Logical().Read(path) 81 | } 82 | 83 | func (vc *wrappedVaultClient) VerifyVaultToken(vaultToken string) (*api.Secret, error) { 84 | vc.log.Debug().Msg("verifying vault token") 85 | oldToken := vc.delegate.Token() 86 | defer vc.delegate.SetToken(oldToken) 87 | 88 | vc.delegate.SetToken(vaultToken) 89 | secret, err := vc.delegate.Auth().Token().LookupSelf() 90 | if err != nil { 91 | vc.log.Debug().Err(err).Msg("verification failed") 92 | return nil, err 93 | } 94 | vc.log.Debug().Msg("verification successful") 95 | return secret, nil 96 | } 97 | 98 | func (vc *wrappedVaultClient) RefreshVaultToken() (*api.Secret, error) { 99 | return vc.Delegate().Auth().Token().RenewSelf(86400) // this value is basically ignored by the server 100 | } 101 | 102 | func (vc *wrappedVaultClient) ServiceSecretPrefix(configVersion int) string { 103 | 104 | if vc.secretsPrefix != "" { 105 | return vc.secretsPrefix 106 | } 107 | 108 | if configVersion < 2 { 109 | return SecretsServicePathV1 110 | } else { 111 | return SecretsServicePathV2 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /vaulttoken/vault_token.go: -------------------------------------------------------------------------------- 1 | package vaulttoken 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | 9 | "github.com/hashicorp/vault/api" 10 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 11 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 12 | "github.com/hootsuite/vault-ctrl-tool/v2/vaultclient" 13 | "github.com/rs/zerolog" 14 | zlog "github.com/rs/zerolog/log" 15 | ) 16 | 17 | var ErrNoValidVaultTokenAvailable = errors.New("no currently valid valid token") 18 | 19 | // VaultToken contains actions used for checking Vault tokens and accessing underlying 20 | // fields. 21 | type VaultToken interface { 22 | CheckAndRefresh() error 23 | Set(token *util.WrappedToken) error 24 | Accessor() string 25 | TokenID() string 26 | Secret() *api.Secret 27 | Wrapped() *util.WrappedToken 28 | } 29 | 30 | type vaultTokenManager struct { 31 | log zerolog.Logger 32 | validToken *util.WrappedToken 33 | 34 | accessor string 35 | tokenID string 36 | 37 | vaultClient vaultclient.VaultClient 38 | briefcase *briefcase.Briefcase 39 | vaultTokenCliArg string 40 | tokenRenewableCliArg bool 41 | } 42 | 43 | // NewVaultToken constructs an implementation of VaultToken which uses provided parameters. 44 | func NewVaultToken(briefcase *briefcase.Briefcase, vaultClient vaultclient.VaultClient, vaultTokenCliArg string, tokenRenewableCliArg bool) VaultToken { 45 | log := zlog.With().Str("vaultAddr", vaultClient.Address()).Logger() 46 | 47 | return &vaultTokenManager{ 48 | log: log, 49 | briefcase: briefcase, 50 | vaultClient: vaultClient, 51 | vaultTokenCliArg: vaultTokenCliArg, 52 | tokenRenewableCliArg: tokenRenewableCliArg, 53 | } 54 | } 55 | 56 | // Secret returns the underlying token secret. 57 | func (vt *vaultTokenManager) Secret() *api.Secret { 58 | return vt.validToken.Secret 59 | } 60 | 61 | // Wrapped returns the wrapped token. 62 | func (vt *vaultTokenManager) Wrapped() *util.WrappedToken { 63 | return vt.validToken 64 | } 65 | 66 | // Accessor returns the underlying token accessor. 67 | func (vt *vaultTokenManager) Accessor() string { 68 | return vt.accessor 69 | } 70 | 71 | // TokenID returns the underlying token ID. 72 | func (vt *vaultTokenManager) TokenID() string { 73 | return vt.tokenID 74 | } 75 | 76 | // Set sets a VaultTokens token using a wrapped token. 77 | func (vt *vaultTokenManager) Set(authToken *util.WrappedToken) error { 78 | 79 | token, err := authToken.TokenID() 80 | if err != nil { 81 | return err 82 | } 83 | 84 | accessor, err := authToken.TokenAccessor() 85 | if err != nil { 86 | return fmt.Errorf("could not determine token's accessor: %w", err) 87 | } 88 | 89 | vt.tokenID = token 90 | vt.accessor = accessor 91 | vt.validToken = authToken 92 | 93 | return nil 94 | } 95 | 96 | // CheckAndRefresh looks for a valid Vault token and will extend it out if it's going to expire soon. The extension is just long 97 | // enough to use it for things. Returns ErrNoValidVaultTokenAvailable if none is available, or different errors 98 | // if something goes wrong along the way. 99 | func (vt *vaultTokenManager) CheckAndRefresh() error { 100 | 101 | secret, err := vt.determineVaultToken() 102 | if err != nil { 103 | return err 104 | } 105 | 106 | if err := vt.Set(secret); err != nil { 107 | return err 108 | } 109 | 110 | return nil 111 | } 112 | 113 | // determineVaultToken looks through the various ways a vault token may already exist (briefcase, flag, env variable), 114 | // and checks with the vault server if the token is still good, optionally refreshing it. If there isn't a vault 115 | // token around, it returns ErrNoValidVaultTokenAvailable. 116 | func (vt *vaultTokenManager) determineVaultToken() (*util.WrappedToken, error) { 117 | if vt.briefcase != nil && vt.briefcase.AuthTokenLease.Token != "" { 118 | log := vt.log.With().Str("source", "briefcase").Logger() 119 | 120 | log.Info().Str("accessor", vt.briefcase.AuthTokenLease.Accessor).Msg("testing if token is usable") 121 | 122 | secret, err := vt.tryToken(log, vt.briefcase.AuthTokenLease.Token) 123 | if err != nil { 124 | log.Warn().Str("accessor", vt.briefcase.AuthTokenLease.Accessor).Err(err).Msg("current briefcase token is not usable") 125 | } else { 126 | accessor, _ := secret.TokenAccessor() 127 | log.Debug().Str("accessor", accessor).Msg("current briefcase token is usable") 128 | return util.NewWrappedToken(secret, vt.briefcase.AuthTokenLease.Renewable), nil 129 | } 130 | } 131 | 132 | if vt.vaultTokenCliArg != "" { 133 | log := zlog.With().Str("source", "cli-arg").Logger() 134 | log.Info().Msg("testing if --vault-token is usable") 135 | 136 | secret, err := vt.tryToken(log, vt.vaultTokenCliArg) 137 | if err != nil { 138 | log.Info().Err(err).Msg("current cli token is not usable") 139 | } else { 140 | accessor, _ := secret.TokenAccessor() 141 | log.Debug().Str("accessor", accessor).Bool("tokenRenewableCliArg", vt.tokenRenewableCliArg).Msg("current cli token is usable") 142 | return util.NewWrappedToken(secret, vt.tokenRenewableCliArg), nil 143 | } 144 | } 145 | 146 | envVaultToken, ok := os.LookupEnv(api.EnvVaultToken) 147 | if ok { 148 | log := zlog.With().Str("source", "env").Logger() 149 | log.Info().Msg("testing if VAULT_TOKEN is usable") 150 | 151 | secret, err := vt.tryToken(log, envVaultToken) 152 | if err != nil { 153 | log.Info().Err(err).Msg("current VAULT_TOKEN is not usable") 154 | } else { 155 | accessor, _ := secret.TokenAccessor() 156 | log.Debug().Str("accessor", accessor).Msg("current VAULT_TOKEN is usable") 157 | 158 | renewable := true 159 | 160 | if renewableOverride, ok := os.LookupEnv("TOKEN_RENEWABLE"); ok { 161 | renewable, err = strconv.ParseBool(renewableOverride) 162 | if err != nil { 163 | log.Warn().Err(err). 164 | Str("TOKEN_RENEWABLE", renewableOverride). 165 | Msg("environment variable TOKEN_RENEWABLE is not parsable as boolean - ignoring") 166 | } else { 167 | renewable = true 168 | } 169 | } 170 | 171 | return util.NewWrappedToken(secret, renewable), nil 172 | } 173 | } 174 | 175 | vt.log.Debug().Msg("no current vault token available") 176 | return nil, ErrNoValidVaultTokenAvailable 177 | } 178 | 179 | func (vt *vaultTokenManager) tryToken(log zerolog.Logger, token string) (*api.Secret, error) { 180 | secret, err := vt.vaultClient.VerifyVaultToken(token) 181 | if err != nil { 182 | return nil, err 183 | } 184 | 185 | if secret == nil { 186 | return nil, fmt.Errorf("server did not return an error, nor a secret") 187 | } 188 | 189 | ttl, err := secret.TokenTTL() 190 | if err != nil { 191 | return nil, err 192 | } 193 | 194 | log.Debug().Str("ttl", ttl.String()).Msg("checking token ttl") 195 | 196 | if ttl.Seconds() > 2 && ttl.Seconds() < 60 { 197 | renewedSecret, err := vt.vaultClient.Delegate().Auth().Token().RenewTokenAsSelf(token, 3600) 198 | if err != nil { 199 | log.Warn().Err(err).Str("ttl", ttl.String()).Msg("failed to renew token") 200 | return nil, err 201 | } 202 | 203 | ttl, err := renewedSecret.TokenTTL() 204 | if err != nil { 205 | log.Error().Err(err).Msg("could not get TTL of renewed token") 206 | return nil, err 207 | } 208 | 209 | if ttl.Seconds() <= 60 { 210 | log.Error().Str("ttl", ttl.String()).Msg("new token was given a TTL that is too short") 211 | return nil, fmt.Errorf("could not renew existing token to make it viable") 212 | } 213 | return renewedSecret, nil 214 | } 215 | return secret, nil 216 | } 217 | -------------------------------------------------------------------------------- /vaulttoken/vault_token_test.go: -------------------------------------------------------------------------------- 1 | package vaulttoken 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "os" 8 | "testing" 9 | 10 | "github.com/golang/mock/gomock" 11 | "github.com/hashicorp/vault/api" 12 | "github.com/hootsuite/vault-ctrl-tool/v2/briefcase" 13 | "github.com/hootsuite/vault-ctrl-tool/v2/util" 14 | mock_vaultclient "github.com/hootsuite/vault-ctrl-tool/v2/vaultclient/mocks" 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | const exampleToken1 = `{ 19 | "request_id": "cd08f32e-f7be-60e9-4af7-d76929bd2a14", 20 | "lease_id": "", 21 | "lease_duration": 0, 22 | "renewable": false, 23 | "data": { 24 | "accessor": "8FvDM61Vc23jht83if5bFWlC", 25 | "creation_time": 1598732682, 26 | "creation_ttl": 32400, 27 | "display_name": "ldap-test1", 28 | "entity_id": "07ef9c85-9f14-1272-eb56-92b7bfd21500", 29 | "expire_time": "2020-07-29T22:24:42.348650363-07:00", 30 | "explicit_max_ttl": 0, 31 | "id": "s.eD8onDKEpvQqNCrSZDwxPLld", 32 | "issue_time": "2020-07-29T13:24:42.348663695-07:00", 33 | "meta": { 34 | "username": "test1" 35 | }, 36 | "num_uses": 0, 37 | "orphan": true, 38 | "path": "auth/ldap/login/test1", 39 | "policies": [ 40 | "default" 41 | ], 42 | "renewable": true, 43 | "ttl": 32382, 44 | "type": "service" 45 | }, 46 | "warnings": null 47 | }` 48 | 49 | func makeToken(t *testing.T, id string) api.Secret { 50 | var secret api.Secret 51 | err := json.Unmarshal([]byte(exampleToken1), &secret) 52 | assert.NoError(t, err, "could not deserialize example token: %v", err) 53 | 54 | secret.Data["id"] = id 55 | secret.Data["accessor"] = "accessor:" + id 56 | 57 | return secret 58 | } 59 | 60 | func TestEmptyVaultToken(t *testing.T) { 61 | ctrl := gomock.NewController(t) 62 | 63 | vaultClient := mock_vaultclient.NewMockVaultClient(ctrl) 64 | vaultClient.EXPECT().Address().Return("unit-test") 65 | 66 | bc := briefcase.NewBriefcase(nil) 67 | vaultToken := NewVaultToken(bc, vaultClient, "", true) 68 | 69 | assert.Equal(t, "", vaultToken.Accessor(), "freshly created vault token must not have a token in it") 70 | assert.Equal(t, "", vaultToken.TokenID(), "freshly crated vault token must not have a token in it") 71 | } 72 | 73 | func TestGetAndSetVaultToken(t *testing.T) { 74 | ctrl := gomock.NewController(t) 75 | 76 | vaultClient := mock_vaultclient.NewMockVaultClient(ctrl) 77 | vaultClient.EXPECT().Address().Return("unit-test") 78 | 79 | bc := briefcase.NewBriefcase(nil) 80 | vaultToken := NewVaultToken(bc, vaultClient, "", true) 81 | 82 | token := makeToken(t, "token-1") 83 | err := vaultToken.Set(util.NewWrappedToken(&token, true)) 84 | assert.NoError(t, err, "must be able to set token") 85 | 86 | assert.Equal(t, "accessor:token-1", vaultToken.Accessor(), "accessor must match value in token") 87 | assert.Equal(t, "token-1", vaultToken.TokenID(), "token must match value in token") 88 | } 89 | 90 | // TestVerifyBriefcaseIfSet ensures that if there's a valid vault token in the briefcase, it is used. 91 | func TestVerifyBriefcaseIfSet(t *testing.T) { 92 | ctrl := gomock.NewController(t) 93 | 94 | vaultClient := mock_vaultclient.NewMockVaultClient(ctrl) 95 | vaultClient.EXPECT().Address().Return("unit-test") 96 | 97 | bc := briefcase.NewBriefcase(nil) 98 | vaultToken := NewVaultToken(bc, vaultClient, "", true) 99 | 100 | token := makeToken(t, "token-1") 101 | assert.NoError(t, bc.EnrollVaultToken(context.TODO(), util.NewWrappedToken(&token, true)), "must be able to enroll example vault token in briefcase") 102 | 103 | vaultClient.EXPECT().VerifyVaultToken("token-1").Return(&token, nil) 104 | 105 | err := vaultToken.CheckAndRefresh() 106 | assert.NoError(t, err, "must accept valid briefcase token if set") 107 | 108 | assert.Equal(t, "accessor:token-1", vaultToken.Accessor(), "accessor must match value in token") 109 | assert.Equal(t, "token-1", vaultToken.TokenID(), "token must match value in token") 110 | } 111 | 112 | func TestBadBriefcaseGoodCLI(t *testing.T) { 113 | ctrl := gomock.NewController(t) 114 | 115 | vaultClient := mock_vaultclient.NewMockVaultClient(ctrl) 116 | vaultClient.EXPECT().Address().Return("unit-test") 117 | 118 | bc := briefcase.NewBriefcase(nil) 119 | briefcaseToken := makeToken(t, "token-1") 120 | assert.NoError(t, bc.EnrollVaultToken(context.TODO(), util.NewWrappedToken(&briefcaseToken, true)), "must be able to enroll example vault token in briefcase") 121 | 122 | cliToken := makeToken(t, "token-2") 123 | vaultToken := NewVaultToken(bc, vaultClient, "token-2", true) 124 | 125 | vaultClient.EXPECT().VerifyVaultToken(gomock.Any()).DoAndReturn( 126 | func(token string) (*api.Secret, error) { 127 | if token == "token-2" { 128 | return &cliToken, nil 129 | } 130 | return nil, errors.New("any error goes here") 131 | }).Times(2) 132 | 133 | err := vaultToken.CheckAndRefresh() 134 | assert.NoError(t, err, "must accept valid CLI token if set") 135 | 136 | assert.Equal(t, "token-2", vaultToken.TokenID(), "token must match value in token") 137 | assert.Equal(t, "accessor:token-2", vaultToken.Accessor(), "accessor must match value in token") 138 | } 139 | 140 | // TestVerifySequence ensures that first the briefcase is checked, then the CLI arg, then env vars, ultimately 141 | // returning ErrNoValidVaultTokenAvailable. 142 | func TestVerifySequence(t *testing.T) { 143 | ctrl := gomock.NewController(t) 144 | 145 | vaultClient := mock_vaultclient.NewMockVaultClient(ctrl) 146 | vaultClient.EXPECT().Address().Return("unit-test") 147 | 148 | bc := briefcase.NewBriefcase(nil) 149 | briefcaseToken := makeToken(t, "token-1") 150 | assert.NoError(t, bc.EnrollVaultToken(context.TODO(), util.NewWrappedToken(&briefcaseToken, true)), "must be able to enroll example vault token in briefcase") 151 | 152 | vaultToken := NewVaultToken(bc, vaultClient, "token-2", true) 153 | os.Setenv("VAULT_TOKEN", "token-3") 154 | 155 | gomock.InOrder( 156 | vaultClient.EXPECT().VerifyVaultToken("token-1").Return(nil, errors.New("some error 1")), 157 | vaultClient.EXPECT().VerifyVaultToken("token-2").Return(nil, errors.New("some error 2")), 158 | vaultClient.EXPECT().VerifyVaultToken("token-3").Return(nil, errors.New("some error 3")), 159 | ) 160 | 161 | err := vaultToken.CheckAndRefresh() 162 | assert.True(t, errors.Is(err, ErrNoValidVaultTokenAvailable), "CheckAndReturn must return ErrNoValidVaultTokenAvailable if there isn't") 163 | } 164 | 165 | // TODO if token is valid, but TTL is less than 3 seconds, then it's not valid 166 | // TODO if token is valid for 3..59 seconds, then ensure it's refreshed 167 | // if refresh fails, then token is no good 168 | // TODO if refresh succeeds, but only gets 30 seconds left, then token is no good 169 | --------------------------------------------------------------------------------