├── .github └── workflows │ └── tag.yml ├── .gitignore ├── CONTRIBUTING.md ├── CONTRIBUTORS.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── cmd ├── mlock-linux.go ├── mlock-non-linux.go └── root.go ├── cram_tests ├── codec.t ├── inject.t ├── regress.t ├── run_option.t └── setup.sh ├── docker-compose.yml ├── go.mod ├── go.sum ├── main.go ├── no_secrets.yml ├── reader ├── reader.go └── reader_test.go └── variables.yml /.github/workflows/tag.yml: -------------------------------------------------------------------------------- 1 | name: Tag, Release and Upload 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | Release: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* 12 | - name: Bump version and push tag 13 | id: tag_release 14 | uses: anothrNick/github-tag-action@1.67.0 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | DEFAULT_BUMP: patch 18 | - name: Create Release 19 | id: create_release 20 | uses: actions/create-release@latest 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | with: 24 | tag_name: ${{ steps.tag_release.outputs.tag }} 25 | release_name: Release ${{ steps.tag_release.outputs.tag }} 26 | body: GitHub Actions Release 27 | draft: false 28 | prerelease: false 29 | - name: Set up Go 1.21 30 | uses: actions/setup-go@v1 31 | with: 32 | go-version: 1.21 33 | id: go 34 | - name: Set up Python 3.13 35 | uses: actions/setup-python@v3 36 | with: 37 | python-version: "3.13" 38 | - name: Install cram 39 | run: | 40 | python -m pip install --upgrade pip 41 | pip install cram 42 | - name: Check out new tag into the Go module directory 43 | uses: actions/checkout@v2 44 | with: 45 | ref: ${{ steps.tag_release.outputs.tag }} 46 | - name: Deps & Build 47 | shell: bash 48 | run: | 49 | export PATH=${PATH}:`go env GOPATH`/bin 50 | make build-deps 51 | make build 52 | - name: Upload binaries to release 53 | uses: svenstaro/upload-release-action@v1-release 54 | with: 55 | repo_token: ${{ secrets.GITHUB_TOKEN }} 56 | file: buildenv-* 57 | tag: ${{ steps.tag_release.outputs.tag }} 58 | overwrite: true 59 | file_glob: true 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | buildenv 2 | pkg/ 3 | buildenv-*.tar.gz -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## CLA 2 | 3 | - If you would like to contribute code to this project you can do so through GitHub by forking the repository and sending a pull request. 4 | - Before Comcast merges your code into the project you must sign the [Comcast Contributor License Agreement (CLA)]( https://gist.github.com/ComcastOSS/a7b8933dd8e368535378cda25c92d19a). 5 | - If you haven't previously signed a Comcast CLA, you'll automatically be asked to when you open a pull request. Alternatively, we can send you a PDF that you can sign and scan back to us. Please create a new GitHub issue to request a PDF version of the CLA. 6 | 7 | ## Developer 8 | 9 | - Fork the codebase on github 10 | - Clone your fork to your machine 11 | - Create a branch describing your change 12 | - Develop your feature and push changes to your branch in your fork 13 | - Add your name to the `CONTRIBUTORS.md` file 14 | - Open a pull request from your fork and branch to master on [https://github.com/Comcast/Buildenv-Tool](https://github.com/Comcast/Buildenv-Tool) 15 | 16 | 17 | ## General Requirements 18 | 19 | - The code must be formatted with `go fmt` 20 | - The change must pass all tests 21 | - The change must include tests for new functionality created 22 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | - [Lew Goettner](mailto:Lewis_Goettner@comcast.com) 2 | - [Tyler Rivera](mailto:Tyler_Rivera@comcast.com) 3 | - [Ryan Eskin](mailto:Ryan_Eskin@comcast.com) 4 | - [Peter Shrom](mailto:Peter_Shrom@comcast.com) 5 | - [Terrance Miller](mailto:Terrance_Miller2@comcast.com) 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell git describe --tags `git rev-list --tags --max-count=1`) 2 | PROJECT_NAME := buildenv 3 | 4 | .phony: all build-deps build clean 5 | 6 | all: clean build-deps build 7 | 8 | test: build-local 9 | go test ./... 10 | cram cram_tests 11 | 12 | build-deps: 13 | go install github.com/mitchellh/gox@latest 14 | 15 | build: test 16 | CGO_ENABLED=0 gox -ldflags "-X main.version=$(VERSION)" -osarch="darwin/amd64 darwin/arm64 linux/386 linux/amd64 linux/arm linux/arm64 windows/386 windows/amd64" -output "pkg/{{.OS}}_{{.Arch}}/$(PROJECT_NAME)" 17 | for pkg in $$(ls pkg/); do cp CONTRIBUTING.md CONTRIBUTORS.md LICENSE NOTICE pkg/$${pkg}; done 18 | for pkg in $$(ls pkg/); do cd pkg/$${pkg}; tar cvzf "../../$(PROJECT_NAME)-$${pkg}-$(VERSION).tar.gz" *; cd ../..; done 19 | 20 | build-local: 21 | CGO_ENABLED=0 go build -ldflags "-X main.version=$(VERSION)" -o $(PROJECT_NAME) 22 | 23 | clean: 24 | rm -rf buildenv 25 | rm -f *.tar.gz 26 | rm -rf pkg 27 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2020 Comcast Cable Communications Management, LLC 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | This product includes software developed at Comcast (http://www.comcast.com/). 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | buildenv 2 | ======== 3 | 4 | A tool for generating environment exports from a YAML file. Variables can be set in plain test, or by specifying vault key-value (version 2) paths and keys (`kv_secrets`) or the older generic / kv paths (`secrets`) where the key name "value" is assumed. Buildenv will autodetect between version 2 and version 1 `kv_secret` paths _unless it can't read the mount details_. For that case, `kv_secrets` will assume version 2, and `kv1_secrets` will use version 1. 5 | 6 | Usage 7 | ----- 8 | 9 | Given a `variables.yml` file like this: 10 | ```yaml 11 | --- 12 | vars: 13 | GLOBAL: "global" 14 | 15 | secrets: 16 | GENERIC_SECRET: "gen/test" 17 | KV_SECRET: "old/test" 18 | KV2_SECRET: "secret/oldstyle" 19 | 20 | kv_secrets: 21 | - path: "secret/test" 22 | vars: 23 | KV2_ONE: "one" 24 | KV2_TWO: "two" 25 | - path: "old/test" 26 | vars: 27 | KV1: "value" 28 | - path: "gen/test" 29 | vars: 30 | KV_GENERIC: "value" 31 | 32 | kv1_secrets: 33 | - path: "old/test" 34 | vars: 35 | KV1SPECIFIC: "value" 36 | 37 | environments: 38 | stage: 39 | vars: 40 | ENVIRONMENT: "stage" 41 | 42 | secrets: 43 | ANOTHER_SECRET: "secret/oldstyle" 44 | 45 | dcs: 46 | ndc_one: 47 | vars: 48 | DC: "one" 49 | kv_secrets: 50 | - path: "old/test" 51 | vars: 52 | KV2_THREE: "three" 53 | ``` 54 | 55 | Output would look like this: 56 | 57 | ```bash 58 | % buildenv -c -e stage -d ndc_one 59 | # Global Variables 60 | export GLOBAL="global" 61 | export KV2_ONE="1" # Path: secret/test, Key: one 62 | export KV2_TWO="2" # Path: secret/test, Key: two 63 | export KV1="old" # Path: old/test, Key: value 64 | export KV_GENERIC="generic" # Path: gen/test, Key: value 65 | export KV1SPECIFIC="old" # Path: old/test, Key: value 66 | export GENERIC_SECRET="generic" # Path: gen/test, Key: value 67 | export KV_SECRET="old" # Path: old/test, Key: value 68 | export KV2_SECRET="default" # Path: secret/oldstyle, Key: value 69 | # Environment: stage 70 | export ENVIRONMENT="stage" 71 | export ANOTHER_SECRET="default" # Path: secret/oldstyle, Key: value 72 | # Datacenter: ndc_one 73 | export DC="one" 74 | export KV2_THREE="3" # Path: old/test, Key: three 75 | ``` 76 | 77 | Another mode uses -r to run a command. All exports will be provided directly to a subshell invoked with the command. This is especially useful in the context of a Makefile where it's very awkward to export lists of environment variables. An added benefit is it's now trivial to set environment variables just for a single command without causing any side-effects for subsequent commands. 78 | 79 | Example Makefile: 80 | 81 | ``` 82 | list-buckets: creds.yml 83 | buildenv -e stage -f $< -r "aws s3 ls" 84 | ``` 85 | 86 | If it's necessary to merge or save a set of variables (for example, so that vault does not need to be called repeatedly), the -u option allows for saving and using a set of variables from the environment without writing possibly sensitive data out to a file: 87 | 88 | ```bash 89 | % export SAVED_ENV=`echo '{"example_var": "the value"}' | base64` 90 | % buildenv -u SAVED_ENV -f /dev/null 91 | export example_var="the value" 92 | ``` 93 | 94 | This takes a base64 encoded json object with key-value pairs and treats them as additional input variables. The corresponding flag for export in the same format is -x: 95 | 96 | ```bash 97 | % buildenv -u SAVED_ENV -f /dev/null -x | base64 -d 98 | {"example_var":"the value"} 99 | ``` 100 | 101 | Multiple -u options can be used as well as combined with -f to combine multiple sources. Given the above variables.yml: 102 | 103 | ```bash 104 | % export SAVED_ENV=`echo '{"example_var": "the value"}' | base64` 105 | % export SAVED_ENV2=`echo '{"another_var": "another value"}' | base64` 106 | % buildenv -u SAVED_ENV -u SAVED_ENV2 -v 107 | export GLOBAL="global" 108 | export example_var="the value" 109 | export another_var="another value" 110 | ``` 111 | 112 | *A Note About Vault:* If you have `secrets` or `kv_secrets` defined in either the global or environment scope, it's a mapping from environment variable to the path & key in vault. Buildenv uses all the standard vault environment variables to communicate with vault (`VAULT_ADDR` and `VAULT_TOKEN` being the two you're most likely to use.) You can find the complete list [in the vault client docs](https://pkg.go.dev/github.com/hashicorp/vault-client-go@v0.4.2#WithEnvironment). 113 | 114 | Running on Linux or in Docker container 115 | ---------- 116 | 117 | It is recommended to use the flag `-m` when running on linux or docker container with swap enabled. This will attempt to lock memory and prevent secrets from being written to swap space. If running on a docker container it may be necessary to add `--cap-add=IPC_LOCK` to the `docker run` command or in the `docker-compose` file to allow this. More info can be found at https://hub.docker.com/_/vault under Memory Locking and 'setcap'. 118 | 119 | Developing 120 | ---------- 121 | 122 | To test with vault, run: 123 | 124 | ```bash 125 | docker-compose up vault -d 126 | export VAULT_ADDR="http://localhost:8200" 127 | export VAULT_TOKEN="test" 128 | vault secrets enable -path gen generic 129 | vault secrets enable -version=1 -path old kv 130 | vault kv put secret/test "one=1" "two=2" 131 | vault kv put secret/oldstyle "value=default" 132 | vault kv put old/test "value=old" "three=3" 133 | vault write gen/test "value=generic" 134 | 135 | buildenv -c -e stage -d ndc_one 136 | docker-compose down 137 | ``` 138 | -------------------------------------------------------------------------------- /cmd/mlock-linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package cmd 5 | 6 | import ( 7 | "fmt" 8 | "syscall" 9 | 10 | "golang.org/x/sys/unix" 11 | ) 12 | 13 | func EnableMlock() error { 14 | mlockError := unix.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE) 15 | if mlockError != nil { 16 | return fmt.Errorf("mlock error: %w", mlockError) 17 | } 18 | return nil 19 | } 20 | -------------------------------------------------------------------------------- /cmd/mlock-non-linux.go: -------------------------------------------------------------------------------- 1 | //go:build !linux 2 | // +build !linux 3 | 4 | package cmd 5 | 6 | func EnableMlock() error { 7 | // NOTE: Not required for windows 8 | return nil 9 | } 10 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2023 Comcast Cable Communications Management, LLC 3 | */ 4 | package cmd 5 | 6 | import ( 7 | "context" 8 | "encoding/base64" 9 | "encoding/json" 10 | "fmt" 11 | "os" 12 | 13 | "github.com/Comcast/Buildenv-Tool/reader" 14 | "github.com/spf13/cobra" 15 | "github.com/spf13/viper" 16 | "gopkg.in/yaml.v3" 17 | ) 18 | 19 | const ( 20 | // ErrorCodeMLock Exit Code for MLock Errors 21 | ErrorCodeMLock = 1 22 | // ErrorCodeEnv Exit Code for Missing Environment 23 | ErrorCodeEnv = 2 24 | // ErrorCodeYaml Exit Code for YAML Errors 25 | ErrorCodeYaml = 5 26 | // ErrorCodeVault Exit Code for Vault Errors 27 | ErrorCodeVault = 6 28 | // ErrorCodeInput Exit Code for Bad Input 29 | ErrorCodeInput = 7 30 | // ErrorCodeOutput Exit Code for Failed Serialization/Output 31 | ErrorCodeOutput = 8 32 | ) 33 | 34 | var cfgFile string 35 | 36 | var Version string 37 | 38 | // rootCmd represents the base command when called without any subcommands 39 | var rootCmd = &cobra.Command{ 40 | Use: "buildenv", 41 | Short: "Set environment variables from a configuation file", 42 | Long: `Set environment variables based on environment and datacenter. 43 | Values can be specified in plain text, or set from a vault server.`, 44 | // Uncomment the following line if your bare application 45 | // has an action associated with it: 46 | Run: func(cmd *cobra.Command, args []string) { 47 | version, _ := cmd.Flags().GetBool("version") 48 | if version { 49 | fmt.Printf("buildenv version %s\n", Version) 50 | os.Exit(0) 51 | } 52 | 53 | ctx := context.Background() 54 | debug, _ := cmd.Flags().GetBool("debug") 55 | 56 | enableMlock, _ := cmd.Flags().GetBool("mlock") 57 | if enableMlock { 58 | err := EnableMlock() 59 | if err != nil { 60 | fmt.Printf("Failure locking memory: %v", err) 61 | os.Exit(ErrorCodeMLock) 62 | } 63 | } 64 | 65 | // Read the Data File 66 | variablesFile, _ := cmd.Flags().GetString("variables_file") 67 | var data reader.Variables 68 | 69 | yamlFile, err := os.ReadFile(variablesFile) 70 | if err != nil { 71 | fmt.Printf("Unable to read file %s: %v", variablesFile, err) 72 | os.Exit(ErrorCodeYaml) 73 | } 74 | err = yaml.Unmarshal(yamlFile, &data) 75 | if err != nil { 76 | fmt.Printf("Unable to parse YAML file %s: %v", variablesFile, err) 77 | } 78 | if debug { 79 | inData, _ := json.MarshalIndent(data, "", " ") 80 | fmt.Printf("Data:\n%s\n\n", inData) 81 | } 82 | 83 | skip_vault, _ := cmd.Flags().GetBool("skip-vault") 84 | 85 | // Setup the Reader 86 | rdr, err := reader.NewReader(reader.WithSkipVault(skip_vault)) 87 | if err != nil { 88 | fmt.Printf("Failure creating Reader: %v", err) 89 | os.Exit(ErrorCodeVault) 90 | } 91 | 92 | // Get the Output 93 | env, _ := cmd.Flags().GetString("environment") 94 | run, _ := cmd.Flags().GetString("run") 95 | dc, _ := cmd.Flags().GetString("datacenter") 96 | 97 | out, err := rdr.Read(ctx, &data, env, dc) 98 | if err != nil { 99 | fmt.Printf("Failure reading data: %v", err) 100 | os.Exit(ErrorCodeVault) 101 | } 102 | 103 | if debug { 104 | outData, _ := json.MarshalIndent(out, "", " ") 105 | fmt.Printf("Output:\n%s\n\n", outData) 106 | } 107 | 108 | var use_vars reader.EnvVars 109 | 110 | use, err := cmd.Flags().GetStringArray("use") 111 | if err != nil { 112 | fmt.Printf("Could not get \"use\" (-u) flag values: %v", err) 113 | os.Exit(ErrorCodeInput) 114 | } 115 | 116 | for _, use_inst := range use { 117 | blob := os.Getenv(use_inst) 118 | decoded := make([]byte, base64.StdEncoding.DecodedLen(len(blob))) 119 | len, err := base64.StdEncoding.Decode(decoded, []byte(blob)) 120 | if err != nil { 121 | fmt.Printf("Could not decode input to flag \"use\" (-u): %v", err) 122 | os.Exit(ErrorCodeInput) 123 | } 124 | decoded = decoded[:len] 125 | /* It adds to the structure, merging matching keys */ 126 | err = json.Unmarshal([]byte(decoded), &use_vars) 127 | if err != nil { 128 | fmt.Printf("Could not decode input to flag \"use\" (-u): %v", err) 129 | os.Exit(ErrorCodeInput) 130 | } 131 | } 132 | 133 | vars_out := use_vars.GetOutput() 134 | out = append(out, vars_out...) 135 | 136 | // Output the Exports 137 | comments, _ := cmd.Flags().GetBool("comments") 138 | if cmd.Flags().Lookup("run").Changed { 139 | os.Exit(out.Exec(run)) 140 | } else { 141 | encoded_export, err := cmd.Flags().GetBool("export") 142 | if err != nil { 143 | fmt.Printf("Failure reading export flag: %v", err) 144 | os.Exit(ErrorCodeInput) 145 | } 146 | 147 | if encoded_export { 148 | err = out.PrintB64Json() 149 | if err != nil { 150 | fmt.Printf("Failure printing output: %v", err) 151 | os.Exit(ErrorCodeOutput) 152 | } 153 | } else { 154 | out.Print(comments) 155 | } 156 | } 157 | }, 158 | } 159 | 160 | // Execute adds all child commands to the root command and sets flags appropriately. 161 | // This is called by main.main(). It only needs to happen once to the rootCmd. 162 | func Execute(version string) { 163 | Version = version 164 | err := rootCmd.Execute() 165 | if err != nil { 166 | os.Exit(1) 167 | } 168 | } 169 | 170 | func init() { 171 | cobra.OnInitialize(initConfig) 172 | 173 | // Here you will define your flags and configuration settings. 174 | // Cobra supports persistent flags, which, if defined here, 175 | // will be global for your application. 176 | 177 | rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.buildenv.yaml)") 178 | 179 | // Cobra also supports local flags, which will only run 180 | // when this action is called directly. 181 | rootCmd.Flags().StringP("environment", "e", "", "Environment (qa, dev, stage, prod, etc)") 182 | rootCmd.Flags().StringP("run", "r", "", "Shell command to execute with environment") 183 | rootCmd.Flags().StringP("datacenter", "d", "", "Datacenter (ndc_as_a, us-east-1 etc)") 184 | rootCmd.Flags().StringP("variables_file", "f", "variables.yml", "Variables Source YAML file") 185 | 186 | rootCmd.Flags().BoolP("skip-vault", "v", false, "Skip Vault and use only variables file") 187 | rootCmd.Flags().BoolP("mlock", "m", false, "Will enable system mlock if set (prevent write to swap on linux)") 188 | rootCmd.Flags().BoolP("comments", "c", false, "Comments will be included in output") 189 | rootCmd.Flags().Bool("debug", false, "Turn on debugging output") 190 | rootCmd.Flags().Bool("version", false, "Print the version number") 191 | rootCmd.Flags().StringArrayP("use", "u", []string{}, "Use Stored Vars from named environment variable. Contents should be base64 encoded JSON.") 192 | rootCmd.Flags().BoolP("export", "x", false, "Print Vars as base64 encoded json") 193 | } 194 | 195 | // initConfig reads in config file and ENV variables if set. 196 | func initConfig() { 197 | if cfgFile != "" { 198 | // Use config file from the flag. 199 | viper.SetConfigFile(cfgFile) 200 | } else { 201 | // Find home directory. 202 | home, err := os.UserHomeDir() 203 | cobra.CheckErr(err) 204 | 205 | // Search config in home directory with name ".buildenv" (without extension). 206 | viper.AddConfigPath(home) 207 | viper.SetConfigType("yaml") 208 | viper.SetConfigName(".buildenv") 209 | } 210 | 211 | viper.AutomaticEnv() // read in environment variables that match 212 | 213 | // If a config file is found, read it in. 214 | if err := viper.ReadInConfig(); err == nil { 215 | fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /cram_tests/codec.t: -------------------------------------------------------------------------------- 1 | Setup 2 | 3 | $ . "$TESTDIR"/setup.sh 4 | 5 | Make env vars 6 | 7 | $ export INP=$(printf '{"VAR1": "VAL1", "VAR2": "VAL2"}' | base64) 8 | $ be -v -f /dev/null -u INP -x > codec_test.blob 9 | $ export BLOB=`cat codec_test.blob` 10 | $ echo "$BLOB" 11 | eyJWQVIxIjoiVkFMMSIsIlZBUjIiOiJWQUwyIn0= 12 | $ be -v -f /dev/null -u BLOB 13 | export VAR1="VAL1" 14 | export VAR2="VAL2" 15 | 16 | -------------------------------------------------------------------------------- /cram_tests/inject.t: -------------------------------------------------------------------------------- 1 | Setup 2 | 3 | $ . "$TESTDIR"/setup.sh 4 | 5 | Try injecting quote 6 | 7 | $ echo '{"vars":{"Q": "\"; echo bad \""}}' > test.yml 8 | $ be -f test.yml 9 | export Q="\"; echo bad \"" 10 | 11 | Bad keys 12 | 13 | $ echo '{"vars":{"export hi=there; dosomethingevil && ": "hi"}}' > test2.yml 14 | $ be -f test2.yml 15 | -------------------------------------------------------------------------------- /cram_tests/regress.t: -------------------------------------------------------------------------------- 1 | Setup 2 | 3 | $ . "$TESTDIR"/setup.sh 4 | 5 | Regular run 6 | 7 | $ be -cf "$TESTDIR"/../no_secrets.yml 8 | # Global Variables 9 | export TEST="no secrets" 10 | 11 | -------------------------------------------------------------------------------- /cram_tests/run_option.t: -------------------------------------------------------------------------------- 1 | Setup 2 | 3 | $ . "$TESTDIR"/setup.sh 4 | 5 | Run buildenv with -r 6 | 7 | $ be -r "echo -n hi" 8 | hi (no-eol) 9 | 10 | Bad command 11 | 12 | $ be -r "./notacommand" 2>/dev/null 13 | [127] 14 | 15 | Vars are there 16 | 17 | $ be -r "echo \${TEST}" 18 | no secrets 19 | -------------------------------------------------------------------------------- /cram_tests/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | alias be="${TESTDIR}/../buildenv -f ${TESTDIR}/../no_secrets.yml" 3 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | build: 4 | image: golang:1.21-alpine 5 | volumes: 6 | - ./:/go/src/buildenv 7 | vault: 8 | image: hashicorp/vault 9 | cap_add: 10 | - IPC_LOCK 11 | environment: 12 | VAULT_DEV_ROOT_TOKEN_ID: test 13 | ports: 14 | - 8200:8200 15 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Comcast/Buildenv-Tool 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/hashicorp/vault-client-go v0.4.2 7 | github.com/spf13/cobra v1.8.0 8 | github.com/spf13/viper v1.18.2 9 | golang.org/x/sys v0.15.0 10 | gopkg.in/yaml.v3 v3.0.1 11 | ) 12 | 13 | require ( 14 | github.com/fsnotify/fsnotify v1.7.0 // indirect 15 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 16 | github.com/hashicorp/go-retryablehttp v0.7.1 // indirect 17 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 18 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 19 | github.com/hashicorp/hcl v1.0.0 // indirect 20 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 21 | github.com/magiconair/properties v1.8.7 // indirect 22 | github.com/mitchellh/go-homedir v1.1.0 // indirect 23 | github.com/mitchellh/mapstructure v1.5.0 // indirect 24 | github.com/pelletier/go-toml/v2 v2.1.0 // indirect 25 | github.com/ryanuber/go-glob v1.0.0 // indirect 26 | github.com/sagikazarmark/locafero v0.4.0 // indirect 27 | github.com/sagikazarmark/slog-shim v0.1.0 // indirect 28 | github.com/sourcegraph/conc v0.3.0 // indirect 29 | github.com/spf13/afero v1.11.0 // indirect 30 | github.com/spf13/cast v1.6.0 // indirect 31 | github.com/spf13/pflag v1.0.5 // indirect 32 | github.com/subosito/gotenv v1.6.0 // indirect 33 | go.uber.org/atomic v1.9.0 // indirect 34 | go.uber.org/multierr v1.9.0 // indirect 35 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect 36 | golang.org/x/text v0.14.0 // indirect 37 | golang.org/x/time v0.5.0 // indirect 38 | gopkg.in/ini.v1 v1.67.0 // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 5 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= 7 | github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= 8 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 9 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 10 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= 11 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= 12 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 13 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 14 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 15 | github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= 16 | github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= 17 | github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 18 | github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= 19 | github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= 20 | github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= 21 | github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= 22 | github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= 23 | github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= 24 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= 25 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= 26 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 27 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 28 | github.com/hashicorp/vault-client-go v0.4.2 h1:XeUXb5jnDuCUhC8HRpkdGPLh1XtzXmiOnF0mXEbARxI= 29 | github.com/hashicorp/vault-client-go v0.4.2/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY= 30 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 31 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 32 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 33 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 34 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 35 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 36 | github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= 37 | github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 38 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 39 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 40 | github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= 41 | github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 42 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 43 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 44 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 45 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 46 | github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= 47 | github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= 48 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 49 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 50 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 51 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 52 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 53 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 54 | github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= 55 | github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 56 | github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= 57 | github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= 58 | github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= 59 | github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= 60 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 61 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 62 | github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= 63 | github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= 64 | github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= 65 | github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 66 | github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= 67 | github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= 68 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 69 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 70 | github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= 71 | github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= 72 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 73 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 74 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 75 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 76 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 77 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 78 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 79 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 80 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 81 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 82 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 83 | go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= 84 | go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 85 | go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= 86 | go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= 87 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= 88 | golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= 89 | golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= 90 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 91 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 92 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 93 | golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= 94 | golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 95 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 96 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 97 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 98 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 99 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 100 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 101 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 102 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 103 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2023 Comcast Cable Communications Management, LLC 3 | */ 4 | package main 5 | 6 | import "github.com/Comcast/Buildenv-Tool/cmd" 7 | 8 | var version string = "development" 9 | 10 | func main() { 11 | cmd.Execute(version) 12 | } 13 | -------------------------------------------------------------------------------- /no_secrets.yml: -------------------------------------------------------------------------------- 1 | --- 2 | vars: 3 | TEST: no secrets -------------------------------------------------------------------------------- /reader/reader.go: -------------------------------------------------------------------------------- 1 | package reader 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "net/http" 10 | "os" 11 | "os/exec" 12 | "regexp" 13 | "slices" 14 | "strings" 15 | 16 | "github.com/hashicorp/vault-client-go" 17 | ) 18 | 19 | type Reader struct { 20 | client *vault.Client 21 | skipVault bool 22 | canDetectMounts bool 23 | mounts Mounts 24 | } 25 | 26 | type ReaderOptFunc func(*Reader) 27 | 28 | func WithSkipVault(skip bool) ReaderOptFunc { 29 | return func(r *Reader) { 30 | r.skipVault = skip 31 | } 32 | } 33 | 34 | type EnvVars map[string]string 35 | 36 | func (e EnvVars) GetOutput() OutputList { 37 | output := OutputList{} 38 | for k, v := range e { 39 | output = append(output, Output{ 40 | Key: k, 41 | Value: v, 42 | }) 43 | } 44 | return output 45 | } 46 | 47 | type Secrets map[string]string 48 | 49 | var shellvar_regexp = regexp.MustCompile("^[_A-Za-z][A-Za-z0-9_]*$") 50 | 51 | func (s Secrets) GetOutput(ctx context.Context, r *Reader) (OutputList, error) { 52 | // Read it like a kv secrets where all keys are "value" 53 | kvSecrets := KVSecrets{} 54 | for outVar, path := range s { 55 | kvSecret := KVSecretBlock{ 56 | Path: path, 57 | Vars: KVSecret{ 58 | outVar: "value", 59 | }, 60 | } 61 | kvSecrets = append(kvSecrets, kvSecret) 62 | } 63 | return kvSecrets.GetOutput(ctx, r) 64 | } 65 | 66 | type KVSecret map[string]string 67 | 68 | type KVSecretBlock struct { 69 | Path string 70 | Vars KVSecret 71 | } 72 | 73 | type KVSecrets []KVSecretBlock 74 | 75 | func (s KVSecretBlock) GetOutput(ctx context.Context, r *Reader) (OutputList, error) { 76 | output := OutputList{} 77 | 78 | // Initialize the Vault Client if Necessary 79 | if r.client == nil { 80 | err := r.InitVault() 81 | if err != nil { 82 | return nil, err 83 | } 84 | } 85 | 86 | // Get the Mount Point for the Secret 87 | mountPoint, secretPath := r.MountAndPath(s.Path) 88 | if mountPoint == "" { 89 | return nil, fmt.Errorf("no mount point found for path %s", s.Path) 90 | } 91 | 92 | // Assume v2 if we can detect mounts and it's a KV engine, or if it's explicitly v2 93 | if !r.canDetectMounts || (r.mounts[mountPoint].Type == "kv" && r.mounts[mountPoint].Version == "2") { 94 | // Get Secret 95 | resp, err := r.client.Secrets.KvV2Read(ctx, secretPath, vault.WithMountPath(mountPoint)) 96 | if err != nil { 97 | if vault.IsErrorStatus(err, http.StatusNotFound) { 98 | return nil, fmt.Errorf("kv2 secret does not exist: '%s'", s.Path) 99 | } 100 | return nil, fmt.Errorf("error reading kv2 path '%s': %w", s.Path, err) 101 | } 102 | // For testing purposes, we want to order this 103 | envVars := []string{} 104 | for varName := range s.Vars { 105 | envVars = append(envVars, varName) 106 | } 107 | slices.Sort(envVars) 108 | for _, varName := range envVars { 109 | varKey := s.Vars[varName] 110 | if _, hasValue := resp.Data.Data[varKey]; !hasValue { 111 | return nil, fmt.Errorf("key %s not found in path %s", varKey, s.Path) 112 | } 113 | val := fmt.Sprintf("%s", resp.Data.Data[varKey]) 114 | output = append(output, Output{ 115 | Key: varName, 116 | Value: val, 117 | Comment: fmt.Sprintf("Path: %s, Key: %s", s.Path, varKey), 118 | }) 119 | } 120 | } else { 121 | // Treat it as a KVv1 secret 122 | resp, err := r.client.Secrets.KvV1Read(ctx, secretPath, vault.WithMountPath(mountPoint)) 123 | if err != nil { 124 | return nil, fmt.Errorf("error reading kv1 path %s: %w", s.Path, err) 125 | } 126 | for varName, varKey := range s.Vars { 127 | if _, hasValue := resp.Data[varKey]; !hasValue { 128 | return nil, fmt.Errorf("key %s not found in path %s", varKey, s.Path) 129 | } 130 | val := fmt.Sprintf("%s", resp.Data[varKey]) 131 | output = append(output, Output{ 132 | Key: varName, 133 | Value: val, 134 | Comment: fmt.Sprintf("Path: %s, Key: %s", s.Path, varKey), 135 | }) 136 | } 137 | } 138 | 139 | return output, nil 140 | } 141 | 142 | func (s KVSecrets) GetOutput(ctx context.Context, r *Reader) (OutputList, error) { 143 | output := OutputList{} 144 | for _, block := range s { 145 | blockOutput, err := block.GetOutput(ctx, r) 146 | if err != nil { 147 | return nil, err 148 | } 149 | output = append(output, blockOutput...) 150 | } 151 | return output, nil 152 | } 153 | 154 | // KV1Secrets is a list of Key-Value Version 1 Secrets 155 | type KV1Secrets []KV1SecretBlock 156 | 157 | type KV1SecretBlock struct { 158 | Path string 159 | Vars KVSecret 160 | } 161 | 162 | func (s KV1SecretBlock) GetOutput(ctx context.Context, r *Reader) (OutputList, error) { 163 | output := OutputList{} 164 | 165 | // Initialize the Vault Client if Necessary 166 | if r.client == nil { 167 | err := r.InitVault() 168 | if err != nil { 169 | return nil, err 170 | } 171 | } 172 | 173 | // The first thing we need to do is get the mount point for the KV engine 174 | mountPoint, secretPath := r.MountAndPath(s.Path) 175 | if mountPoint == "" { 176 | return nil, fmt.Errorf("no mount point found for path %s", s.Path) 177 | } 178 | 179 | // Treat it as a KVv1 secret 180 | resp, err := r.client.Secrets.KvV1Read(ctx, secretPath, vault.WithMountPath(mountPoint)) 181 | if err != nil { 182 | return nil, fmt.Errorf("error reading kv1 path %s: %w", s.Path, err) 183 | } 184 | for varName, varKey := range s.Vars { 185 | if _, hasValue := resp.Data[varKey]; !hasValue { 186 | return nil, fmt.Errorf("key %s not found in path %s", varKey, s.Path) 187 | } 188 | val := fmt.Sprintf("%s", resp.Data[varKey]) 189 | output = append(output, Output{ 190 | Key: varName, 191 | Value: val, 192 | Comment: fmt.Sprintf("Path: %s, Key: %s", s.Path, varKey), 193 | }) 194 | } 195 | 196 | return output, nil 197 | } 198 | 199 | func (s KV1Secrets) GetOutput(ctx context.Context, r *Reader) (OutputList, error) { 200 | output := OutputList{} 201 | for _, block := range s { 202 | blockOutput, err := block.GetOutput(ctx, r) 203 | if err != nil { 204 | return nil, err 205 | } 206 | output = append(output, blockOutput...) 207 | } 208 | return output, nil 209 | } 210 | 211 | type DC struct { 212 | Vars EnvVars `yaml:"vars,omitempty"` 213 | Secrets Secrets `yaml:"secrets,omitempty"` 214 | KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` 215 | KV1Secrets KVSecrets `yaml:"kv1_secrets,omitempty"` 216 | } 217 | 218 | type Environment struct { 219 | Vars EnvVars `yaml:"vars,omitempty"` 220 | Secrets Secrets `yaml:"secrets,omitempty"` 221 | KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` 222 | KV1Secrets KVSecrets `yaml:"kv1_secrets,omitempty"` 223 | Dcs map[string]DC `yaml:"dcs,omitempty"` 224 | } 225 | 226 | type Variables struct { 227 | Vars EnvVars `yaml:"vars,omitempty"` 228 | Secrets Secrets `yaml:"secrets,omitempty"` 229 | KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` 230 | KV1Secrets KVSecrets `yaml:"kv1_secrets,omitempty"` 231 | Environments map[string]Environment `yaml:"environments,omitempty"` 232 | } 233 | 234 | type Output struct { 235 | Key string 236 | Value string 237 | Comment string 238 | } 239 | type OutputList []Output 240 | 241 | func (o OutputList) Exec(shell_cmd string) int { 242 | shell, shell_isset := os.LookupEnv("SHELL") 243 | 244 | var cmd *exec.Cmd 245 | 246 | if shell_isset { 247 | cmd = exec.Command(shell, "-c", shell_cmd) 248 | } else { 249 | cmd = exec.Command("/usr/bin/env", "bash", "-c", shell_cmd) 250 | } 251 | 252 | for _, out := range o { 253 | if shellvar_regexp.MatchString(out.Key) { 254 | s := fmt.Sprintf("%s=%s", out.Key, out.Value) 255 | cmd.Env = append(cmd.Environ(), s) 256 | } 257 | } 258 | 259 | cmd.Stdin = os.Stdin 260 | cmd.Stdout = os.Stdout 261 | cmd.Stderr = os.Stderr 262 | 263 | if err := cmd.Run(); err != nil { 264 | var exitError *exec.ExitError 265 | if errors.As(err, &exitError) { 266 | return exitError.ExitCode() 267 | } 268 | return -1 269 | } 270 | return 0 271 | } 272 | 273 | func (ol OutputList) PrintB64Json() error { 274 | envs := EnvVars{} 275 | 276 | for _, o := range ol { 277 | if o.Key != "" { 278 | envs[o.Key] = o.Value 279 | } 280 | } 281 | 282 | serialized, err := json.Marshal(envs) 283 | if err != nil { 284 | return err 285 | } 286 | encoded := base64.StdEncoding.EncodeToString(serialized) 287 | 288 | fmt.Print(encoded) 289 | 290 | return nil 291 | } 292 | 293 | func (o OutputList) Print(showComments bool) { 294 | for _, out := range o { 295 | if out.Key == "" { 296 | if showComments && out.Comment != "" { 297 | fmt.Printf("# %s\n", out.Comment) 298 | } 299 | } else { 300 | /* silently discards variable names that are not shell safe */ 301 | if shellvar_regexp.MatchString(out.Key) { 302 | fmt.Printf("export %s=%q", out.Key, out.Value) 303 | if out.Comment != "" && showComments { 304 | fmt.Printf(" # %s", out.Comment) 305 | } 306 | fmt.Println() 307 | } 308 | } 309 | } 310 | } 311 | 312 | type MountInfo struct { 313 | Type string 314 | Version string 315 | } 316 | 317 | type Mounts map[string]MountInfo 318 | 319 | func (r *Reader) InitVault() error { 320 | if r.skipVault { 321 | return nil 322 | } 323 | 324 | vaultClient, err := vault.New(vault.WithEnvironment()) 325 | if err != nil { 326 | return err 327 | } 328 | r.client = vaultClient 329 | r.canDetectMounts = false 330 | 331 | // Get mount info 332 | resp, err := vaultClient.System.MountsListSecretsEngines(context.Background()) 333 | if err == nil { 334 | r.canDetectMounts = true 335 | mounts := Mounts{} 336 | for mount, details := range resp.Data { 337 | detailMap := details.(map[string]interface{}) 338 | thisMount := MountInfo{ 339 | Type: detailMap["type"].(string), 340 | } 341 | if options, hasOptions := detailMap["options"]; hasOptions && options != nil { 342 | optionMap := options.(map[string]interface{}) 343 | if version, hasVersion := optionMap["version"]; hasVersion { 344 | thisMount.Version = version.(string) 345 | } 346 | } 347 | mounts[mount] = thisMount 348 | } 349 | 350 | r.mounts = mounts 351 | } 352 | 353 | return nil 354 | } 355 | 356 | func NewReader(opts ...ReaderOptFunc) (*Reader, error) { 357 | r := &Reader{} 358 | for _, opt := range opts { 359 | opt(r) 360 | } 361 | return r, nil 362 | } 363 | 364 | func (r *Reader) MountAndPath(path string) (string, string) { 365 | if r.canDetectMounts { 366 | for mount := range r.mounts { 367 | if strings.HasPrefix(path, mount) { 368 | return mount, strings.TrimPrefix(path, mount) 369 | } 370 | } 371 | } else { 372 | // Take the first part of the path 373 | parts := strings.SplitN(path, "/", 2) 374 | return parts[0], parts[1] 375 | } 376 | return "", "" 377 | } 378 | 379 | func (r *Reader) Read(ctx context.Context, input *Variables, env string, dc string) (OutputList, error) { 380 | output := OutputList{} 381 | 382 | // Global Variables 383 | output = append(output, Output{ 384 | Comment: "Global Variables", 385 | }) 386 | output = append(output, input.Vars.GetOutput()...) 387 | 388 | if !r.skipVault { 389 | // Global Secrets 390 | kvOut, err := input.KVSecrets.GetOutput(ctx, r) 391 | if err != nil { 392 | return nil, fmt.Errorf("kv secret error: %w", err) 393 | } 394 | output = append(output, kvOut...) 395 | kv1Out, err := input.KV1Secrets.GetOutput(ctx, r) 396 | if err != nil { 397 | return nil, fmt.Errorf("kv1 secret error: %w", err) 398 | } 399 | output = append(output, kv1Out...) 400 | secretOut, err := input.Secrets.GetOutput(ctx, r) 401 | if err != nil { 402 | return nil, fmt.Errorf("secret error: %w", err) 403 | } 404 | output = append(output, secretOut...) 405 | } 406 | 407 | // Environment Variablers 408 | if env != "" { 409 | output = append(output, Output{ 410 | Comment: fmt.Sprintf("Environment: %s", env), 411 | }) 412 | output = append(output, input.Environments[env].Vars.GetOutput()...) 413 | // KV (autodetect or v2) 414 | if !r.skipVault { 415 | kvOut, err := input.Environments[env].KVSecrets.GetOutput(ctx, r) 416 | if err != nil { 417 | return nil, fmt.Errorf("kv secret error: %w", err) 418 | } 419 | output = append(output, kvOut...) 420 | // KV1 421 | kv1Out, err := input.Environments[env].KV1Secrets.GetOutput(ctx, r) 422 | if err != nil { 423 | return nil, fmt.Errorf("kv1 secret error: %w", err) 424 | } 425 | output = append(output, kv1Out...) 426 | // Secrets 427 | secretOut, err := input.Environments[env].Secrets.GetOutput(ctx, r) 428 | if err != nil { 429 | return nil, fmt.Errorf("secret error: %w", err) 430 | } 431 | output = append(output, secretOut...) 432 | } 433 | } 434 | 435 | // DC Variables 436 | if dc != "" { 437 | output = append(output, Output{ 438 | Comment: fmt.Sprintf("Datacenter: %s", dc), 439 | }) 440 | output = append(output, input.Environments[env].Dcs[dc].Vars.GetOutput()...) 441 | 442 | if !r.skipVault { 443 | // KV (autodetect or v2) 444 | kvOut, err := input.Environments[env].Dcs[dc].KVSecrets.GetOutput(ctx, r) 445 | if err != nil { 446 | return nil, fmt.Errorf("kv secret error: %w", err) 447 | } 448 | output = append(output, kvOut...) 449 | // KV1 450 | kv1Out, err := input.Environments[env].Dcs[dc].KV1Secrets.GetOutput(ctx, r) 451 | if err != nil { 452 | return nil, fmt.Errorf("kv1 secret error: %w", err) 453 | } 454 | output = append(output, kv1Out...) 455 | // Secrets 456 | secretOut, err := input.Environments[env].Dcs[dc].Secrets.GetOutput(ctx, r) 457 | if err != nil { 458 | return nil, fmt.Errorf("secret error: %w", err) 459 | } 460 | output = append(output, secretOut...) 461 | } 462 | } 463 | 464 | return output, nil 465 | } 466 | -------------------------------------------------------------------------------- /reader/reader_test.go: -------------------------------------------------------------------------------- 1 | package reader 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/base64" 7 | "encoding/json" 8 | "io" 9 | "log" 10 | "net/http" 11 | "net/http/httptest" 12 | "os" 13 | "reflect" 14 | "testing" 15 | 16 | "github.com/hashicorp/vault-client-go" 17 | ) 18 | 19 | func TestEnvVars_GetOutput(t *testing.T) { 20 | tests := []struct { 21 | name string 22 | e EnvVars 23 | want OutputList 24 | }{ 25 | { 26 | name: "Test Output", 27 | e: EnvVars{ 28 | "a": "b", 29 | }, 30 | want: OutputList{ 31 | { 32 | Key: "a", 33 | Value: "b", 34 | }, 35 | }, 36 | }, 37 | { 38 | name: "Empty Output", 39 | e: EnvVars{}, 40 | want: OutputList{}, 41 | }, 42 | } 43 | for _, tt := range tests { 44 | t.Run(tt.name, func(t *testing.T) { 45 | if got := tt.e.GetOutput(); !reflect.DeepEqual(got, tt.want) { 46 | t.Errorf("EnvVars.GetOutput() = %v, want %v", got, tt.want) 47 | } 48 | }) 49 | } 50 | } 51 | 52 | func TestReader_Read(t *testing.T) { 53 | type fields struct { 54 | client *vault.Client 55 | } 56 | type args struct { 57 | input *Variables 58 | env string 59 | dc string 60 | } 61 | tests := []struct { 62 | name string 63 | fields fields 64 | args args 65 | want OutputList 66 | wantErr bool 67 | }{ 68 | { 69 | name: "Just Plain Variables", 70 | fields: fields{}, 71 | args: args{ 72 | env: "dev", 73 | dc: "us-least-1", 74 | input: &Variables{ 75 | Vars: EnvVars{ 76 | "FOO": "bar", 77 | }, 78 | Environments: map[string]Environment{ 79 | "dev": { 80 | Vars: EnvVars{ 81 | "ENV": "dev", 82 | }, 83 | Dcs: map[string]DC{ 84 | "us-least-1": { 85 | Vars: EnvVars{ 86 | "DC": "us-least-1", 87 | }, 88 | }, 89 | }, 90 | }, 91 | "stage": { 92 | Vars: EnvVars{ 93 | "env": "stage", 94 | }, 95 | }, 96 | }, 97 | }, 98 | }, 99 | want: OutputList{ 100 | { 101 | Comment: "Global Variables", 102 | }, 103 | { 104 | Key: "FOO", 105 | Value: "bar", 106 | }, 107 | { 108 | Comment: "Environment: dev", 109 | }, 110 | { 111 | Key: "ENV", 112 | Value: "dev", 113 | }, 114 | { 115 | Comment: "Datacenter: us-least-1", 116 | }, 117 | { 118 | Key: "DC", 119 | Value: "us-least-1", 120 | }, 121 | }, 122 | wantErr: false, 123 | }, 124 | } 125 | for _, tt := range tests { 126 | t.Run(tt.name, func(t *testing.T) { 127 | ctx := context.Background() 128 | r := &Reader{ 129 | client: tt.fields.client, 130 | } 131 | got, err := r.Read(ctx, tt.args.input, tt.args.env, tt.args.dc) 132 | if (err != nil) != tt.wantErr { 133 | t.Errorf("Reader.Read() error = %v, wantErr %v", err, tt.wantErr) 134 | return 135 | } 136 | if !reflect.DeepEqual(got, tt.want) { 137 | t.Errorf("Reader.Read() = %+v, want %+v", got, tt.want) 138 | } 139 | }) 140 | } 141 | } 142 | 143 | func TestKVSecretBlock_GetOutputNoDetect(t *testing.T) { 144 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 145 | log.Printf("%+v", r) 146 | var resp []byte 147 | status := http.StatusOK 148 | 149 | // KV Data 150 | switch r.URL.Path { 151 | case "/v1/kv2/data/test": 152 | resp = []byte(`{"request_id":"bf3b02c0-096e-84d3-dad7-196aa9f112ed","lease_id":"","renewable":false,"lease_duration":0,"data":{"data":{"one":"1","two":"2","three":"3"},"metadata":{"created_time":"2023-12-20T15:32:32.814115685Z","custom_metadata":null,"deletion_time":"","destroyed":false,"version":1}},"wrap_info":null,"warnings":null,"auth":null}`) 153 | case "/v1/kv/test": 154 | resp = []byte(`{"request_id":"63c8c31b-f03f-81ac-cfaa-324239789c3f","lease_id":"","renewable":false,"lease_duration":2764800,"data":{"value":"old"},"wrap_info":null,"warnings":null,"auth":null}`) 155 | default: 156 | status = http.StatusNotFound 157 | resp = []byte(`{"errors":[]}`) 158 | } 159 | 160 | w.WriteHeader(status) 161 | w.Write(resp) 162 | })) 163 | defer server.Close() 164 | 165 | client, _ := vault.New(vault.WithAddress(server.URL)) 166 | reader := &Reader{ 167 | client: client, 168 | canDetectMounts: false, 169 | } 170 | 171 | type fields struct { 172 | Path string 173 | Vars KVSecret 174 | } 175 | type args struct { 176 | r *Reader 177 | } 178 | tests := []struct { 179 | name string 180 | fields fields 181 | args args 182 | want OutputList 183 | wantErr bool 184 | }{ 185 | { 186 | name: "No KV Path", 187 | args: args{ 188 | r: reader, 189 | }, 190 | fields: fields{ 191 | Path: "kv2/path", 192 | Vars: KVSecret{ 193 | "NOT": "here", 194 | }, 195 | }, 196 | wantErr: true, 197 | want: nil, 198 | }, 199 | { 200 | name: "No KV2 Key", 201 | args: args{ 202 | r: reader, 203 | }, 204 | fields: fields{ 205 | Path: "kv2/test", 206 | Vars: KVSecret{ 207 | "THREE": "nope", 208 | }, 209 | }, 210 | wantErr: true, 211 | want: nil, 212 | }, 213 | { 214 | name: "With no autodection, KV Read Fails", 215 | args: args{ 216 | r: reader, 217 | }, 218 | fields: fields{ 219 | Path: "kv/test", 220 | Vars: KVSecret{ 221 | "VALUE": "value", 222 | }, 223 | }, 224 | wantErr: true, 225 | }, 226 | { 227 | name: "Test KV2 Read", 228 | args: args{ 229 | r: reader, 230 | }, 231 | fields: fields{ 232 | Path: "kv2/test", 233 | Vars: KVSecret{ 234 | "ONE": "one", 235 | "TWO": "two", 236 | "THREE": "three", 237 | }, 238 | }, 239 | want: OutputList{ 240 | { 241 | Key: "ONE", 242 | Value: "1", 243 | Comment: "Path: kv2/test, Key: one", 244 | }, 245 | { 246 | Key: "THREE", 247 | Value: "3", 248 | Comment: "Path: kv2/test, Key: three", 249 | }, 250 | { 251 | Key: "TWO", 252 | Value: "2", 253 | Comment: "Path: kv2/test, Key: two", 254 | }, 255 | }, 256 | wantErr: false, 257 | }, 258 | } 259 | for _, tt := range tests { 260 | t.Run(tt.name, func(t *testing.T) { 261 | ctx := context.Background() 262 | s := KVSecretBlock{ 263 | Path: tt.fields.Path, 264 | Vars: tt.fields.Vars, 265 | } 266 | got, err := s.GetOutput(ctx, tt.args.r) 267 | if (err != nil) != tt.wantErr { 268 | t.Errorf("KVSecretBlock.GetOutput() error = %v, wantErr %v", err, tt.wantErr) 269 | return 270 | } 271 | if !reflect.DeepEqual(got, tt.want) { 272 | t.Errorf("KVSecretBlock.GetOutput() = %v, want %v", got, tt.want) 273 | } 274 | }) 275 | } 276 | } 277 | 278 | func TestKVSecretBlock_GetOutput(t *testing.T) { 279 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 280 | log.Printf("%+v", r) 281 | var resp []byte 282 | status := http.StatusOK 283 | 284 | // KV Data 285 | switch r.URL.Path { 286 | case "/v1/kv2/data/test": 287 | resp = []byte(`{"request_id":"bf3b02c0-096e-84d3-dad7-196aa9f112ed","lease_id":"","renewable":false,"lease_duration":0,"data":{"data":{"one":"1","two":"2","three":"3"},"metadata":{"created_time":"2023-12-20T15:32:32.814115685Z","custom_metadata":null,"deletion_time":"","destroyed":false,"version":1}},"wrap_info":null,"warnings":null,"auth":null}`) 288 | case "/v1/kv/test": 289 | resp = []byte(`{"request_id":"63c8c31b-f03f-81ac-cfaa-324239789c3f","lease_id":"","renewable":false,"lease_duration":2764800,"data":{"value":"old"},"wrap_info":null,"warnings":null,"auth":null}`) 290 | default: 291 | status = http.StatusNotFound 292 | resp = []byte(`{"errors":[]}`) 293 | } 294 | 295 | w.WriteHeader(status) 296 | w.Write(resp) 297 | })) 298 | defer server.Close() 299 | 300 | client, _ := vault.New(vault.WithAddress(server.URL)) 301 | reader := &Reader{ 302 | client: client, 303 | canDetectMounts: true, 304 | mounts: Mounts{ 305 | "kv2/": { 306 | Type: "kv", 307 | Version: "2", 308 | }, 309 | "kv/": { 310 | Type: "kv", 311 | }, 312 | "generic/": { 313 | Type: "generic", 314 | }, 315 | }, 316 | } 317 | 318 | type fields struct { 319 | Path string 320 | Vars KVSecret 321 | } 322 | type args struct { 323 | r *Reader 324 | } 325 | tests := []struct { 326 | name string 327 | fields fields 328 | args args 329 | want OutputList 330 | wantErr bool 331 | }{ 332 | { 333 | name: "No Mount", 334 | args: args{ 335 | r: reader, 336 | }, 337 | fields: fields{ 338 | Path: "secret/test", 339 | Vars: KVSecret{ 340 | "should": "fail", 341 | }, 342 | }, 343 | want: nil, 344 | wantErr: true, 345 | }, 346 | { 347 | name: "No KV Path", 348 | args: args{ 349 | r: reader, 350 | }, 351 | fields: fields{ 352 | Path: "kv2/path", 353 | Vars: KVSecret{ 354 | "NOT": "here", 355 | }, 356 | }, 357 | wantErr: true, 358 | want: nil, 359 | }, 360 | { 361 | name: "No KV2 Key", 362 | args: args{ 363 | r: reader, 364 | }, 365 | fields: fields{ 366 | Path: "kv2/test", 367 | Vars: KVSecret{ 368 | "THREE": "nope", 369 | }, 370 | }, 371 | wantErr: true, 372 | want: nil, 373 | }, 374 | { 375 | name: "Test KV Read", 376 | args: args{ 377 | r: reader, 378 | }, 379 | fields: fields{ 380 | Path: "kv/test", 381 | Vars: KVSecret{ 382 | "VALUE": "value", 383 | }, 384 | }, 385 | want: OutputList{ 386 | { 387 | Key: "VALUE", 388 | Value: "old", 389 | Comment: "Path: kv/test, Key: value", 390 | }, 391 | }, 392 | }, 393 | { 394 | name: "Test KV2 Read", 395 | args: args{ 396 | r: reader, 397 | }, 398 | fields: fields{ 399 | Path: "kv2/test", 400 | Vars: KVSecret{ 401 | "ONE": "one", 402 | "TWO": "two", 403 | "THREE": "three", 404 | }, 405 | }, 406 | want: OutputList{ 407 | { 408 | Key: "ONE", 409 | Value: "1", 410 | Comment: "Path: kv2/test, Key: one", 411 | }, 412 | { 413 | Key: "THREE", 414 | Value: "3", 415 | Comment: "Path: kv2/test, Key: three", 416 | }, 417 | { 418 | Key: "TWO", 419 | Value: "2", 420 | Comment: "Path: kv2/test, Key: two", 421 | }, 422 | }, 423 | wantErr: false, 424 | }, 425 | } 426 | for _, tt := range tests { 427 | t.Run(tt.name, func(t *testing.T) { 428 | ctx := context.Background() 429 | s := KVSecretBlock{ 430 | Path: tt.fields.Path, 431 | Vars: tt.fields.Vars, 432 | } 433 | got, err := s.GetOutput(ctx, tt.args.r) 434 | if (err != nil) != tt.wantErr { 435 | t.Errorf("KVSecretBlock.GetOutput() error = %v, wantErr %v", err, tt.wantErr) 436 | return 437 | } 438 | if !reflect.DeepEqual(got, tt.want) { 439 | t.Errorf("KVSecretBlock.GetOutput() = %v, want %v", got, tt.want) 440 | } 441 | }) 442 | } 443 | } 444 | 445 | func TestKV1SecretBlock_GetOutput(t *testing.T) { 446 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 447 | log.Printf("%+v", r) 448 | var resp []byte 449 | status := http.StatusOK 450 | 451 | // KV Data 452 | switch r.URL.Path { 453 | case "/v1/kv/test": 454 | resp = []byte(`{"request_id":"63c8c31b-f03f-81ac-cfaa-324239789c3f","lease_id":"","renewable":false,"lease_duration":2764800,"data":{"value":"old"},"wrap_info":null,"warnings":null,"auth":null}`) 455 | default: 456 | status = http.StatusNotFound 457 | resp = []byte(`{"errors":[]}`) 458 | } 459 | 460 | w.WriteHeader(status) 461 | w.Write(resp) 462 | })) 463 | defer server.Close() 464 | 465 | client, _ := vault.New(vault.WithAddress(server.URL)) 466 | reader := &Reader{ 467 | client: client, 468 | canDetectMounts: true, 469 | mounts: Mounts{ 470 | "kv2/": { 471 | Type: "kv", 472 | Version: "2", 473 | }, 474 | "kv/": { 475 | Type: "kv", 476 | }, 477 | "generic/": { 478 | Type: "generic", 479 | }, 480 | }, 481 | } 482 | 483 | type fields struct { 484 | Path string 485 | Vars KVSecret 486 | } 487 | type args struct { 488 | r *Reader 489 | } 490 | tests := []struct { 491 | name string 492 | fields fields 493 | args args 494 | want OutputList 495 | wantErr bool 496 | }{ 497 | { 498 | name: "No Mount", 499 | args: args{ 500 | r: reader, 501 | }, 502 | fields: fields{ 503 | Path: "secret/test", 504 | Vars: KVSecret{ 505 | "should": "fail", 506 | }, 507 | }, 508 | want: nil, 509 | wantErr: true, 510 | }, 511 | { 512 | name: "No KV Path", 513 | args: args{ 514 | r: reader, 515 | }, 516 | fields: fields{ 517 | Path: "kv/path", 518 | Vars: KVSecret{ 519 | "NOT": "here", 520 | }, 521 | }, 522 | wantErr: true, 523 | want: nil, 524 | }, 525 | { 526 | name: "Test KV Read", 527 | args: args{ 528 | r: reader, 529 | }, 530 | fields: fields{ 531 | Path: "kv/test", 532 | Vars: KVSecret{ 533 | "VALUE": "value", 534 | }, 535 | }, 536 | want: OutputList{ 537 | { 538 | Key: "VALUE", 539 | Value: "old", 540 | Comment: "Path: kv/test, Key: value", 541 | }, 542 | }, 543 | }, 544 | } 545 | for _, tt := range tests { 546 | t.Run(tt.name, func(t *testing.T) { 547 | ctx := context.Background() 548 | s := KV1SecretBlock{ 549 | Path: tt.fields.Path, 550 | Vars: tt.fields.Vars, 551 | } 552 | got, err := s.GetOutput(ctx, tt.args.r) 553 | if (err != nil) != tt.wantErr { 554 | t.Errorf("KVSecretBlock.GetOutput() error = %v, wantErr %v", err, tt.wantErr) 555 | return 556 | } 557 | if !reflect.DeepEqual(got, tt.want) { 558 | t.Errorf("KVSecretBlock.GetOutput() = %v, want %v", got, tt.want) 559 | } 560 | }) 561 | } 562 | } 563 | 564 | func TestSkipVault_Reader(t *testing.T) { 565 | reader, _ := NewReader(WithSkipVault(true)) 566 | 567 | type args struct { 568 | r *Reader 569 | i *Variables 570 | env string 571 | dc string 572 | skip bool 573 | } 574 | 575 | tests := []struct { 576 | name string 577 | args args 578 | want OutputList 579 | wantErr bool 580 | }{ 581 | { 582 | name: "Has Secrets", 583 | args: args{ 584 | skip: true, 585 | env: "dev", 586 | dc: "us-least-1", 587 | r: reader, 588 | i: &Variables{ 589 | Vars: EnvVars{ 590 | "XYZ": "yep", 591 | }, 592 | Secrets: Secrets{ 593 | "Secret1": "it's here", 594 | }, 595 | KVSecrets: KVSecrets{{ 596 | Path: "path/test", 597 | Vars: KVSecret{"KVSecret1": "kvsecret1"}, 598 | }}, 599 | KV1Secrets: KVSecrets{{ 600 | Path: "path2/test", 601 | Vars: KVSecret{ 602 | "KV1Secret1": "another one", 603 | }, 604 | }}, 605 | }, 606 | }, 607 | want: OutputList{ 608 | { 609 | Comment: "Global Variables", 610 | }, 611 | {Key: "XYZ", Value: "yep", Comment: ""}, 612 | { 613 | Comment: "Environment: dev", 614 | }, 615 | { 616 | Comment: "Datacenter: us-least-1", 617 | }, 618 | }, 619 | wantErr: false, 620 | }, 621 | } 622 | for _, tt := range tests { 623 | t.Run(tt.name, func(t *testing.T) { 624 | ctx := context.Background() 625 | got, err := tt.args.r.Read(ctx, tt.args.i, tt.args.env, tt.args.dc) 626 | if (err != nil) != tt.wantErr { 627 | t.Errorf("Reader.Read() error = %v, wantErr %v", err, tt.wantErr) 628 | return 629 | } 630 | if !reflect.DeepEqual(got, tt.want) { 631 | t.Errorf("Reader.Read() = %v, want %v", got, tt.want) 632 | } 633 | }) 634 | } 635 | } 636 | 637 | func TestOutputList_Exec(t *testing.T) { 638 | key := "BuildEnvTestKey" 639 | val := "BuildEnvTestVal" 640 | outputList := OutputList{ 641 | Output{key, val, "acomment"}, 642 | } 643 | 644 | type fields struct { 645 | Key string 646 | Val string 647 | Out OutputList 648 | } 649 | 650 | type args struct { 651 | cmd string 652 | } 653 | 654 | tests := []struct { 655 | name string 656 | fields fields 657 | args args 658 | want string 659 | wantErr int 660 | }{ 661 | { 662 | name: "Good Simple Cmd", 663 | args: args{ 664 | cmd: "echo -n hi", 665 | }, 666 | fields: fields{ 667 | Key: key, 668 | Val: val, 669 | Out: outputList, 670 | }, 671 | want: "hi", 672 | wantErr: 0, 673 | }, 674 | { 675 | name: "Bad Cmd", 676 | args: args{ 677 | cmd: "./nosuchcommandprobably", 678 | }, 679 | fields: fields{ 680 | Key: key, 681 | Val: val, 682 | Out: outputList, 683 | }, 684 | want: "", 685 | wantErr: 127, 686 | }, 687 | { 688 | name: "Has Env Var val", 689 | args: args{ 690 | cmd: "echo -n ${" + key + "}", 691 | }, 692 | fields: fields{ 693 | Key: key, 694 | Val: val, 695 | Out: outputList, 696 | }, 697 | want: val, 698 | wantErr: 0, 699 | }, 700 | } 701 | 702 | for _, tt := range tests { 703 | t.Run(tt.name, func(t *testing.T) { 704 | oldstdout := os.Stdout 705 | r, w, _ := os.Pipe() 706 | os.Stdout = w 707 | got := tt.fields.Out.Exec(tt.args.cmd) 708 | 709 | outC := make(chan string) 710 | 711 | go func() { 712 | var buf bytes.Buffer 713 | io.Copy(&buf, r) 714 | outC <- buf.String() 715 | }() 716 | 717 | w.Close() 718 | 719 | out := <-outC 720 | 721 | os.Stdout = oldstdout 722 | 723 | if out != tt.want { 724 | t.Errorf("OutputList.Exec() output = %v, want %v", out, tt.want) 725 | } 726 | if got != tt.wantErr { 727 | t.Errorf("OutputList.Exec() err = %v, wantErr %v", got, tt.wantErr) 728 | } 729 | }) 730 | } 731 | } 732 | 733 | func TestOutputList_PrintB64Json(t *testing.T) { 734 | envVars := EnvVars{ 735 | "BuildEnvTestKey1": "BuildEnvTestVal1", 736 | "BuildEnvTestKey2": "BuildEnvTestVal2", 737 | } 738 | 739 | type fields struct { 740 | Out OutputList 741 | } 742 | 743 | tests := []struct { 744 | name string 745 | fields fields 746 | want EnvVars 747 | wantErr bool 748 | }{ 749 | { 750 | name: "Basic 2 vars", 751 | fields: fields{Out: envVars.GetOutput()}, 752 | want: envVars, 753 | }, 754 | } 755 | 756 | for _, tt := range tests { 757 | t.Run(tt.name, func(t *testing.T) { 758 | oldstdout := os.Stdout 759 | r, w, _ := os.Pipe() 760 | os.Stdout = w 761 | 762 | outC := make(chan string) 763 | 764 | go func() { 765 | var buf bytes.Buffer 766 | io.Copy(&buf, r) 767 | outC <- buf.String() 768 | }() 769 | 770 | err := tt.fields.Out.PrintB64Json() 771 | if err != nil { 772 | t.Errorf("PrintB64Json() error = %v", err) 773 | } 774 | 775 | os.Stdout = oldstdout 776 | 777 | w.Close() 778 | 779 | out := <-outC 780 | 781 | decoded := make([]byte, base64.StdEncoding.DecodedLen(len(out))) 782 | len, err := base64.StdEncoding.Decode(decoded, []byte(out)) 783 | 784 | var out_vars EnvVars 785 | 786 | if err != nil { 787 | t.Errorf("Exception parsing output: %v", err) 788 | return 789 | } 790 | 791 | decoded = decoded[:len] 792 | 793 | err = json.Unmarshal([]byte(decoded), &out_vars) 794 | if err != nil { 795 | t.Errorf("Exception parsing output: %v", err) 796 | return 797 | } 798 | 799 | if !reflect.DeepEqual(out_vars, tt.want) { 800 | t.Errorf("PrintB64Json() output = %v, want %v", out_vars, tt.want) 801 | } 802 | }) 803 | } 804 | } 805 | -------------------------------------------------------------------------------- /variables.yml: -------------------------------------------------------------------------------- 1 | --- 2 | vars: 3 | GLOBAL: "global" 4 | 5 | secrets: 6 | GENERIC_SECRET: "gen/test" 7 | KV_SECRET: "old/test" 8 | KV2_SECRET: "secret/oldstyle" 9 | 10 | kv_secrets: 11 | - path: "secret/test" 12 | vars: 13 | KV2_ONE: "one" 14 | KV2_TWO: "two" 15 | - path: "old/test" 16 | vars: 17 | KV1: "value" 18 | - path: "gen/test" 19 | vars: 20 | KV_GENERIC: "value" 21 | 22 | kv1_secrets: 23 | - path: "old/test" 24 | vars: 25 | KV1SPECIFIC: "value" 26 | 27 | environments: 28 | stage: 29 | vars: 30 | ENVIRONMENT: "stage" 31 | 32 | secrets: 33 | ANOTHER_SECRET: "secret/oldstyle" 34 | 35 | dcs: 36 | ndc_one: 37 | vars: 38 | DC: "one" 39 | kv_secrets: 40 | - path: "old/test" 41 | vars: 42 | KV2_THREE: "three" 43 | 44 | 45 | --------------------------------------------------------------------------------