├── .gitignore ├── .semaphore ├── install-nix.sha256 └── semaphore.yml ├── LICENSE ├── README.md ├── RELEASE.md ├── app └── Main.hs ├── default.nix ├── nix ├── haskell-dependencies.nix ├── haskell-overlay.nix ├── nixpkgs-pinned.nix ├── overlay.nix ├── release.nix ├── sources.json ├── sources.nix └── stack-shell.nix ├── package.yaml ├── package ├── build_package.sh ├── deb_control └── deb_postinst ├── src ├── Config.hs ├── KeyMap.hs ├── Response.hs └── SecretsFile.hs ├── stack.yaml ├── test ├── .gitignore ├── ConfigSpec.hs ├── KeyMapSpec.hs ├── ResponseSpec.hs ├── SecretFileSpec.hs ├── Spec.hs ├── default.nix ├── golden │ ├── empty-block.secrets │ ├── empty-v1.secrets │ ├── empty-v2.secrets │ ├── special.secrets │ ├── v1-version.secrets │ ├── v1.secrets │ ├── v2.secrets │ ├── variables.secrets │ └── whitespace.secrets ├── integration │ ├── bad_token.sh │ ├── duplicate_env_var.sh │ ├── environment_inheritance.sh │ ├── happy_path.sh │ ├── inherit-env-blacklist-both.sh │ ├── inherit-env-blacklist-existing.sh │ ├── inherit-env-blacklist-secret.sh │ ├── invalid_addr.sh │ ├── kubernetes_auth.py │ ├── kubernetes_auth_container.nix │ ├── no_token_configured.sh │ ├── retry_tests.py │ └── tap.py ├── integration_test.sh └── invalid │ ├── ambiguous.secrets │ ├── v1.secrets │ ├── whitespace-wrong-mount.secrets │ ├── whitespace-wrong-version.secrets │ └── wrong_envvar_names.secrets └── vaultenv.nix /.gitignore: -------------------------------------------------------------------------------- 1 | # Build output 2 | /.stack-work 3 | *.deb 4 | /vaultenv-*-linux-musl 5 | /result 6 | 7 | # auto-generated files 8 | *.cabal 9 | -------------------------------------------------------------------------------- /.semaphore/install-nix.sha256: -------------------------------------------------------------------------------- 1 | a2d0e4f6954a6295664994dc4e5492843b7de3e7e23e89a1df9e0820975d2fde install-nix-2.24.12 2 | -------------------------------------------------------------------------------- /.semaphore/semaphore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: "v1.0" 3 | name: "Vaultenv CI Pipeline" 4 | 5 | agent: 6 | machine: 7 | type: "f1-standard-2" 8 | os_image: "ubuntu2204" 9 | 10 | blocks: 11 | - name: "Run checks" 12 | task: 13 | secrets: 14 | # Keys needed to access our cachix cache. 15 | - name: "cachix-channable-public" 16 | 17 | jobs: 18 | - name: "Run tests" 19 | commands: 20 | - "checkout" 21 | 22 | # Download and verify Nix installation script. 23 | - curl -o install-nix-2.24.12 https://releases.nixos.org/nix/nix-2.24.12/install 24 | - sha256sum --check .semaphore/install-nix.sha256 25 | 26 | # Restore `/nix` cache. Create the directory first, otherwise we encounter 27 | # permission errors. We do this because the Semaphore cache is faster than 28 | # both Cachix and cache.nixos.org. 29 | - "sudo mkdir /nix" 30 | - "sudo chown -R semaphore:semaphore /nix" 31 | - "cache restore nix-store-" 32 | 33 | # For some reason, Semaphore CI sets this variable, but it causes the nix installation to fail 34 | - unset LD_LIBRARY_PATH 35 | 36 | # Install Nix and source the shell configuration immediately. 37 | - "sh ./install-nix-2.24.12 --no-daemon" 38 | # Enable `nix-command` feature, which `nix build` needs to build 39 | - "sudo mkdir /etc/nix" 40 | - "echo 'experimental-features = nix-command' | sudo tee -a /etc/nix/nix.conf" 41 | 42 | - ". $HOME/.nix-profile/etc/profile.d/nix.sh" 43 | 44 | # Install Cachix and use the Channable cache 45 | - "nix-env -iA nixpkgs.cachix" 46 | - "cachix use channable-public" 47 | 48 | # Build the devenv and the static package 49 | - "nix-build --no-out-link default.nix nix/release.nix > nix-store-locations" 50 | 51 | # push the result to Cachix. 52 | - "cat nix-store-locations | cachix push channable-public" 53 | 54 | # Run the build and tests with Stack 55 | - "nix shell -f default.nix -c stack build --only-dependencies --test" 56 | - "nix shell -f default.nix -c stack test" 57 | 58 | # Run the integration tests 59 | - "(cd test && nix shell -f default.nix -c ./integration_test.sh)" 60 | 61 | # Build the Debian package 62 | - "nix shell -f default.nix -c ./package/build_package.sh" 63 | 64 | # Store a copy of the nix store. This will be refreshed daily, which 65 | # is more than sufficient for this repo. 66 | - "cache store nix-store-$(date -u -Idate) /nix" 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017, Channable 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above 12 | copyright notice, this list of conditions and the following 13 | disclaimer in the documentation and/or other materials provided 14 | with the distribution. 15 | 16 | * Neither the name of Channable nor the names of other 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vaultenv 2 | 3 | [![Build Status](https://channable.semaphoreci.com/badges/vaultenv/branches/master.svg?style=shields)](https://channable.semaphoreci.com/projects/vaultenv) 4 | 5 | Run processes with secrets from [HashiCorp Vault]. It: 6 | 7 | 1. Reads a list of required secrets 8 | 2. Fetches them from Vault 9 | 4. Calls `exec` with the secrets in the process environment 10 | 11 | There is nothing else going on. 12 | 13 | `vaultenv` supports the Vault KV API. It supports both version 1 and version 2. 14 | This support is automatic; but you do need a token which has read access on 15 | the `/sys/mounts` endpoint. 16 | 17 | ## Comparison to alternatives 18 | 19 | The only alternative to this tool that we are aware of is [envconsul], also by 20 | HashiCorp. Unlike envconsul, `vaultenv` does not: 21 | 22 | - daemonize 23 | - spawn any child processes 24 | - manage the lifecycle of the process it provides the secrets for 25 | 26 | All of the above should not be done by a secret fetching tool. This should be 27 | left to a service manager, like systemd. 28 | 29 | `vaultenv` calls a syscall from the `exec` family after fetching secrets for 30 | you. This means that `vaultenv` replaces its own process with whatever you want. 31 | After your service has started, `vaultenv` is not running anymore. 32 | 33 | This approach does mean that we cannot automatically restart services if 34 | secrets in Vault have changed. 35 | 36 | ## Terminology 37 | 38 | A brief summary of Vault terminology: 39 | 40 | A Vault **secret** consists of multiple **key/value pairs**, which are stored 41 | under a **path** in a **backend**. 42 | 43 | Let's use the Vault CLI to write a secret to see all the concepts in action: 44 | 45 | ```shell 46 | $ vault write secret/production/third-party api-key=fecb0f6e97c5b37b3a814107682cf68416f072a8 47 | ^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 48 | backend path key value 49 | ``` 50 | 51 | Note: this will fail without a running Vault instance. Use `vault server -dev` 52 | to get one up and running locally. 53 | 54 | ## Tutorial 55 | 56 | Before we can start, [build vaultenv locally][build-vaultenv] or [download a 57 | binary][download-vaultenv]. 58 | 59 | The following program depends on the secret that we stored in Vault in the 60 | previous section: 61 | 62 | ```bash 63 | #!/bin/bash 64 | 65 | # Fail when referencing an unbound variable 66 | set -u 67 | 68 | # Mentally substitute the call to echo with something like: 69 | # curl -H "Content-Type: application/json" -H "X-API-Key: ${PRODUCTION_THIRD_PARTY_API_KEY}" -X POST -d '{"my": "payload"}' https://example.com/ 70 | echo "${PRODUCTION_THIRD_PARTY_API_KEY}" 71 | ``` 72 | 73 | This program will fail without `PRODUCTION_THIRD_PARTY_API_KEY` in its 74 | environment: 75 | 76 | ``` 77 | $ ./tutorial.sh 78 | ./tutorial.sh: line 8: PRODUCTION_THIRD_PARTY_API_KEY: unbound variable 79 | ``` 80 | 81 | We can use `vaultenv` to fetch the required secret before running `tutorial.sh`. 82 | Create a file `tutorial.secrets` with the following content: 83 | 84 | ``` 85 | production/third-party#api-key 86 | ``` 87 | 88 | And run `vaultenv` like so: 89 | 90 | ``` 91 | $ vaultenv --token --no-connect-tls --secrets-file ./tutorial.secrets ./tutorial.sh 92 | fecb0f6e97c5b37b3a814107682cf68416f072a8 93 | ``` 94 | 95 | This instructs `vaultenv` to fetch `secret/production/third-party` and load the 96 | contents of `api-key` under `PRODUCTION_THIRD_PARTY_API_KEY` in the environment 97 | of `tutorial.sh`. 98 | 99 | ## Usage 100 | 101 | ``` 102 | vaultenv 0.19.0 - run programs with secrets from HashiCorp Vault 103 | 104 | Usage: vaultenv [--version] [--host HOST] [--port PORT] [--addr ADDR] 105 | [--auth-backend AUTH_BACKEND] 106 | [--token TOKEN | --kubernetes-role ROLE | --github-token TOKEN] 107 | [--secrets-file FILENAME | --secret-file FILENAME] [CMD] 108 | [ARGS...] [--no-connect-tls | --connect-tls] 109 | [--no-validate-certs | --validate-certs] 110 | [--no-inherit-env | --inherit-env] 111 | [--inherit-env-blacklist COMMA_SEPARATED_NAMES] 112 | [--retry-base-delay-milliseconds MILLISECONDS] 113 | [--retry-attempts NUM] [--log-level error | info] [--use-path] 114 | [--max-concurrent-requests NUM] 115 | [--duplicate-variable-behavior error | keep | overwrite] 116 | 117 | Available options: 118 | -h,--help Show this help text 119 | --version Show version 120 | --host HOST Vault host, either an IP address or DNS name. 121 | Defaults to localhost. Also configurable via 122 | VAULT_HOST. 123 | --port PORT Vault port. Defaults to 8200. Also configurable via 124 | VAULT_PORT. 125 | --addr ADDR Vault address, the scheme, either http:// or 126 | https://, the ip-address or DNS name, followed by the 127 | port, separated with a ':'. Cannot be combined with 128 | either VAULT_PORT or VAULT_HOST 129 | --auth-backend AUTH_BACKEND 130 | Name of Vault authentication backend. Defaults to 131 | 'kubernetes' or 'github' depending on the type of the 132 | chosen authentication method. Also configurable via 133 | VAULT_AUTH_BACKEND. 134 | --token TOKEN Token to authenticate to Vault with. Also 135 | configurable via VAULT_TOKEN. 136 | --kubernetes-role ROLE Authenticate using Kubernetes service account in 137 | /var/run/secrets/kubernetes.io, with the given role. 138 | Also configurable via VAULTENV_KUBERNETES_ROLE. 139 | --github-token TOKEN Authenticate using a GitHub personal access 140 | token.Also configurable via VAULTENV_GITHUB_TOKEN. 141 | --secrets-file FILENAME Config file specifying which secrets to request. Also 142 | configurable via VAULTENV_SECRETS_FILE. 143 | --secret-file FILENAME alias for `--secrets-file` 144 | CMD command to run after fetching secrets 145 | ARGS... Arguments to pass to CMD, defaults to nothing 146 | --no-connect-tls Don't use TLS when connecting to Vault. Default: use 147 | TLS. Also configurable via VAULTENV_CONNECT_TLS. 148 | --connect-tls Always connect to Vault via TLS. Default: use TLS. 149 | Can be used to override VAULTENV_CONNECT_TLS. 150 | --no-validate-certs Don't validate TLS certificates when connecting to 151 | Vault. Default: validate certs. Also configurable via 152 | VAULTENV_VALIDATE_CERTS. 153 | --validate-certs Always validate TLS certificates when connecting to 154 | Vault. Default: validate certs. Can be used to 155 | override VAULTENV_CONNECT_TLS. 156 | --no-inherit-env Don't merge the parent environment with the secrets 157 | file. Default: merge environments. Also configurable 158 | via VAULTENV_INHERIT_ENV. 159 | --inherit-env Always merge the parent environment with the secrets 160 | file. Default: merge environments. Can be used to 161 | override VAULTENV_INHERIT_ENV. 162 | --inherit-env-blacklist COMMA_SEPARATED_NAMES 163 | Comma-separated list of environment variable names to 164 | remove from the environment before executing CMD. 165 | Also configurable via VAULTENV_INHERIT_ENV_BLACKLIST. 166 | Has no effect if no-inherit-env is set! 167 | --retry-base-delay-milliseconds MILLISECONDS 168 | Base delay for vault connection retrying. Defaults to 169 | 40ms. Also configurable via 170 | VAULTENV_RETRY_BASE_DELAY_MS. 171 | --retry-attempts NUM Maximum number of vault connection retries. Defaults 172 | to 9. Also configurable through 173 | VAULTENV_RETRY_ATTEMPTS. 174 | --log-level error | info Log-level to run vaultenv under. Options: 'error' or 175 | 'info'. Defaults to 'error'. Also configurable via 176 | VAULTENV_LOG_LEVEL 177 | --use-path Use PATH for finding the executable that vaultenv 178 | should call. Default: don't search PATH. Also 179 | configurable via VAULTENV_USE_PATH. 180 | --max-concurrent-requests NUM 181 | Maximum number of concurrent requests to vault. 182 | Defaults to 8. Pass 0 to disable the limit. Also 183 | configurable through 184 | VAULTENV_MAX_CONCURRENT_REQUESTS. 185 | --duplicate-variable-behavior error | keep | overwrite 186 | Changes the behavior of duplicate variables. 'error` 187 | produces an error for duplicate variables. 'keep' 188 | keeps the value already in the environment , ignoring 189 | the secret. 'overwrite' overwrite the environment 190 | variable in favor of the secret. Default to 'error'. 191 | Also configurable through 192 | VAULTENV_DUPLICATE_VARIABLE_BEHAVIOR. 193 | ``` 194 | 195 | ## Configuration 196 | 197 | Vaultenv reads configuration from two types of files: 198 | 199 | - A specification of secrets to fetch. 200 | - Configuration options for `vaultenv` itself, mostly connection related. 201 | 202 | Decoupling these is useful, because this allows for e.g. changing which secrets 203 | are fetched on a per project basis, while the connection options stay the same. 204 | Let's first discuss secrets files. 205 | 206 | ### Secret specification 207 | 208 | There are two versions of the secret specification format. The first version shipped 209 | with the initial version of Vaultenv, but doesn't allow users to specify custom 210 | mountpoints for backends. Vaultenv would always fetch from the generic secret 211 | backend mounted at `secret/`. Version 2 of the format supports custom mount 212 | points. 213 | 214 | Example (version 1, implicit `secret/` path prepended): 215 | 216 | ``` 217 | production/third-party#api-key 218 | production/another-third-party#refresh-token 219 | FOOBAR=production/third-party#foobar 220 | ``` 221 | 222 | Vaultenv will make the following environment variables available: 223 | 224 | - `PRODUCTION_THIRD_PARTY_API_KEY`: Contents of the `api-key` field of the 225 | secret at `secret/production/third-party`. 226 | - `PRODUCTION_ANOTHER_THIRD_PARTY_REFRESH_TOKEN`: Contents of the 227 | `refresh-token` field of the secret at 228 | `secret/production/another-third-party#refresh-token`. 229 | - `FOOBAR`: Contents of the `foobar` field in the secret at 230 | `secret/production/third-party`. 231 | 232 | The `FOOBAR=` syntax means: make this secret available under the FOOBAR 233 | environment variable. 234 | 235 | Example (version 2, explicit mount paths): 236 | 237 | ``` 238 | VERSION 2 239 | 240 | MOUNT secret 241 | third-party#api-key 242 | 243 | MOUNT production 244 | third-party#refresh-token 245 | FOOBAR=third-party#foobar 246 | ``` 247 | 248 | Vaultenv will make the following environment variables available: 249 | 250 | - `SECRET_THIRD_PARTY_API_KEY` with the contents of the `api-key` field of the 251 | secret at `secret/third-party`. 252 | - `PRODUCTION_THIRD_PARTY_REFRESH_TOKEN` with the contents of the 253 | `refresh-token` field from the secret at `production/third-party` 254 | - `FOOBAR` with the contents of the `foobar` field of the secret at 255 | `production/third-party`. 256 | 257 | ## Authentication 258 | 259 | Vaultenv fetches these secrets from Hashicorp Vault. 260 | Vaultenv supports multiple ways to authenticate to Vault: 261 | 262 | - Token-based authentication 263 | - Kubernetes authentication 264 | - GitHub authentication 265 | 266 | The token-based authentication relies on the root token, and passing this to 267 | vaultenv in either the `--token` flag or the `VAULTENV_TOKEN` environment 268 | variable. 269 | 270 | The Kubernetes authentication is useful when running Vaultenv in a Kubernetes 271 | cluster. In order to use this method, it should be enabled in Vault: 272 | https://www.vaultproject.io/docs/auth/kubernetes The role should be specified 273 | with either the `kubernetes-role` flag or the `VAULTENV_KUBERNETES_ROLE` 274 | environment variable. The Kubernetes authentication token is read from 275 | `/var/run/secrets/kubernetes.io/serviceaccount/token` 276 | 277 | The GitHub authentication is recommended when running Vaultenv on a development 278 | machine. In order to use this method, it should be enabled in Vault: 279 | https://www.vaultproject.io/docs/auth/github The user should create a GitHub 280 | personal access token with the `read:org` scope on the configured organization. 281 | This token can then be supplied to either the `github-token` flag or the 282 | `VAULTENV_GITHUB_TOKEN` environment variable. 283 | 284 | Sometimes the name of the authentication backend configured in vault may be different 285 | than the default one. Vaultenv supports non-default auth backend names by using the 286 | `--auth-backend` parameter. For example if your kubernetes auth backend is called 287 | `k8s-dev` then you can run vaultenv with `--auth-backend k8s-dev` or set the 288 | `VAULT_AUTH_BACKEND` environment variable to `k8s-dev`. 289 | 290 | ### Behavior configuration 291 | 292 | Vaultenv supports loading behavior configuration from files (in addition to the 293 | CLI flags and environment variable lookups). Vaultenv currently looks for these 294 | files in the following places: 295 | 296 | - `/etc/vaultenv.conf` 297 | - `$HOME/.config/vaultenv/vaultenv.conf` 298 | - `$CWD/.env` 299 | 300 | These config files support the exact same syntax as the environment variables 301 | that you can use to configure Vaultenv. See the `--help` output for a list of 302 | what's available. 303 | 304 | These files follow the `.env` format (as popularized by [this Ruby 305 | gem](https://github.com/bkeepers/dotenv)). An example: 306 | 307 | ``` 308 | # /etc/vaultenv.conf 309 | # Also: comments are allowed if they start with `#`. 310 | VAULT_TOKEN="your-vault-token" 311 | VAULT_PORT="8200" 312 | VAULTENV_INHERIT_ENV="yes" 313 | ``` 314 | 315 | All while running Vaultenv without any CLI args. 316 | 317 | #### Different levels of configuration 318 | 319 | It can happen that conflicting configuration values are send to Vaultenv. An example 320 | would be a global secret file, which is overwritten by a project specific configuration file. 321 | The order of precedence (from least precedence to most precedence) is as follows: 322 | - `/etc/vaultenv.conf` 323 | - `$HOME/.config/vaultenv/vaultenv.conf` 324 | - `$CWD/.env` 325 | - environment variables 326 | - command line options 327 | 328 | This is useful on development machines. It allows you to: 329 | 330 | - Set global connection options on a per-machine basis. Useful if you run a 331 | Vault instance in your VPN. 332 | - Set per-user tokens. 333 | - Set per-project secrets files. 334 | 335 | This means that any command line option that is present would overwrite any other configuration. 336 | If an option is not specified, the default is used. The defaults are as follows: 337 | ``` 338 | VAULT_HOST: localhost 339 | VAULT_PORT: 8200 340 | VAULT_ADDR: https://localhost:8200 341 | VAULT_TOKEN: Unspecified 342 | VAULTENV_GITHUB_TOKEN: Unspecified 343 | VAULTENV_KUBERNETES_ROLE: Unspecified 344 | VAULTENV_SECRETS_FILE: Unspecified 345 | CMD: Unspecified 346 | ARGS: [] 347 | VAULTENV_CONNECT_TLS: True 348 | VAULTENV_VALIDATE_CERTS: True 349 | VAULTENV_INHERIT_ENV: True 350 | VAULTENV_INHERIT_ENV_BLACKLIST: [] 351 | VAULTENV_RETRY_BASE_DELAY: 40 352 | VAULTENV_RETRY_ATTEMPTS: 9 353 | VAULTENV_LOG_LEVEL: Error 354 | VAULTENV_USE_PATH: True 355 | VAULTENV_MAX_CONCURRENT_REQUESTS: 8 356 | VAULTENV_DUPLICATE_VARIABLE_BEHAVIOR: error 357 | ``` 358 | In cases where no default nor any value is specified, which is possible for `Token`, `Secret file` and 359 | `Command`, Vaultenv will give an error that it requires these values to operate. 360 | 361 | #### Connection options 362 | 363 | Vaultenv also supports the `VAULT_ADDR` configuration. In such a case, without specifying separate 364 | parameters for host, port and whether to use TLS or not, one can specify these values in a single value. 365 | The address always starts with either `http://` or `https://`, followed by a either a DNS name 366 | or an ip-address. The port is specified at the end of the address, using a `:` to separate the host 367 | and the port. For example: `https://example.com:42` would create a TLS enabled connection 368 | to the host `example.com` on port `42`. 369 | 370 | Other errors can happen with the address configuration. There are two ways of specifying 371 | what the connection options are, either via the address of via a combination of 372 | the port, the host and whether to use TLS. As there are two ways of specifying this, 373 | it is also possible for these values to conflict. Consider the situation where 374 | `VAULT_ADDR` is `http://example.com:8200` and `VAULT_PORT` is set to `42`. There 375 | are two ways Vaultenv can resolve this. In the case of the address and the port 376 | being specified in the same configuration, like the same file or both as command 377 | line options, Vaultenv will not choose either way and will report an error. 378 | 379 | In the case they are specified in different configuration levels, like the address in a file 380 | and the port in the command line options, the higher precedence (as defined 381 | [above](#Different-levels-of-configuration)) is used for that specific value. 382 | In this case, this would result in a host of `example.com`, no usage of TLS, 383 | due to the `http://` scheme, and a port of `42`. 384 | 385 | Other errors than the mismatch address error that can happen during parsing are: 386 | - A non-numeric port in the address, like `http://localhost:my_port` 387 | - A non-supported scheme in the address, like `ftp://example.com:42` 388 | 389 | By default, `vaultenv` will open at most 8 concurrent HTTP connections to the vault server. 390 | This limit can be changed using the `VAULTENV_MAX_CONCURRENT_REQUESTS` setting, and it can 391 | be disabled by choosing the limit `0`. 392 | 393 | ## Allowed characters in environment variables 394 | 395 | We disallow the following in any path to keep the parser and format simple and 396 | unambiguous: 397 | 398 | - Whitespace 399 | - The `#` and `=` characters 400 | - Control characters 401 | 402 | Everything else is allowed. 403 | 404 | **N.B.:** Be careful with special characters in path components. While 405 | vault supports them, and vaultenv parses them from the secrets file just fine, 406 | you MUST specify an environment variable to put them in, otherwise you may run 407 | into unexpected behavior. 408 | 409 | ## Environment variable names 410 | 411 | The secret `path` and `key` determine the name of the environment variable 412 | that the secret contents will be available under. `path` and `key` are 413 | uppercased and concatenated by a `_`. All `/` and `-` characters are also 414 | replaced by underscores. 415 | 416 | Example: the contents of `production/third-party#api-key` will be available 417 | as `PRODUCTION_THIRD_PARTY_API_KEY`. 418 | 419 | ## Building 420 | 421 | Vaultenv is written in Haskell and builds with [Stack][stack]: 422 | 423 | stack setup 424 | stack build 425 | 426 | The binary can then be found at `$(stack path 427 | --local-install-root)/bin/vaultenv`. You can also run it directly 428 | with `stack exec`: 429 | 430 | ``` 431 | stack exec vaultenv -- --token SECRET --secrets-file foo.env /usr/bin/env 432 | ``` 433 | 434 | It is possible to `stack build` with `--split-objs` to produce a smaller binary. 435 | To take full advantage of this, the Stackage snapshot has to be rebuilt. 436 | 437 | If you want a fully static executable without a runtime dependency on `libc` 438 | and run GNU/Linux, you can install [Nix](https://nixos.org/nix/) and run: 439 | 440 | ``` 441 | nix-build --no-out-link nix/release.nix -A vaultenvStatic 442 | ``` 443 | 444 | This has not been tested on any other platform. 445 | 446 | That will build vaultenv (and a bunch of dependencies). The final line of the 447 | output should be a path in `/nix/store` which contains the final vaultenv 448 | binary. 449 | 450 | It is possible to speed up the compilation process by copying some dependencies 451 | from a Nix cache, to not have to recompile all of Haskell and its dependencies. 452 | To set up the cache, execute these commands once before building: 453 | 454 | ```console 455 | $ nix shell --file default.nix -c cachix use static-haskell-nix 456 | $ nix shell --file default.nix -c cachix use channable-public 457 | ``` 458 | 459 | Note that the build process via Nix is not (yet) reproducible, which means that 460 | different builds of the same source code may result in different Nix derivation 461 | hashes. 462 | 463 | ## Development 464 | 465 | If you want a convenient way to gather the development dependencies of 466 | `vaultenv`, you can use `nix`. 467 | 468 | The repository contains a `default.nix` which will get you `stack` and `vault`. 469 | You can then use this to get a shell with the tools in scope to work on and 470 | test `vaultenv`. 471 | 472 | Get this shell with: 473 | 474 | ``` 475 | $ nix shell --file default.nix 476 | ``` 477 | 478 | ## Future work 479 | 480 | - Support DNS `SRV` record lookups, so users only need to specify the host 481 | Vault runs on. This integrates `vaultenv` nicely with Vaults HA setup. 482 | - Certificate pinning/validation 483 | 484 | ## License 485 | 486 | 3-clause BSD. See `LICENSE` for details. 487 | 488 | [HashiCorp Vault]: https://www.vaultproject.io/ 489 | [envconsul]: https://github.com/hashicorp/envconsul 490 | [stack]: https://haskellstack.org 491 | [build-vaultenv]:#building 492 | [download-vaultenv]:https://github.com/channable/vaultenv/releases 493 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # How to release `vaultenv` 2 | 3 | 1. Build `vaultenv` with `stack`. Run the tests. 4 | 1. Increment the `version` field in `package.yaml` 5 | 1. Create a git commit 6 | 1. Tag: `git tag -a v`. Write a changelog. 7 | 1. `git push origin master --tags` 8 | 1. Build the static version of `vaultenv` and the Debian package by running 9 | `nix shell --file default.nix -c ./package/build_package.sh`. 10 | 1. Go to https://github.com/channable/vaultenv/releases 11 | 1. Click "Draft a new release". Add the binary from the Nix output and the 12 | .deb package. 13 | -------------------------------------------------------------------------------- /app/Main.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE FlexibleContexts #-} 2 | {-# LANGUAGE LambdaCase #-} 3 | {-# LANGUAGE OverloadedStrings #-} 4 | {-# LANGUAGE ScopedTypeVariables #-} 5 | 6 | import Control.Applicative ((<|>)) 7 | import Control.Concurrent.QSem (newQSem, waitQSem, signalQSem) 8 | import Control.Exception (Handler (..), bracket_, catch, catches) 9 | import Control.Monad (forM, when) 10 | import Control.Monad.IO.Class (MonadIO, liftIO) 11 | import Data.Aeson (FromJSON, (.:)) 12 | import Data.Aeson.Types (parseMaybe) 13 | import Data.Bifunctor (first) 14 | import Data.HashMap.Strict (HashMap) 15 | import Data.List (nubBy) 16 | import Data.Text (Text, unpack) 17 | import Network.HTTP.Client (ManagerSettings (managerConnCount)) 18 | import Network.HTTP.Conduit (Manager, newManager) 19 | import Network.HTTP.Simple (HttpException(..), Request, Response, 20 | defaultRequest, setRequestBodyJSON, setRequestHeader, 21 | setRequestMethod, setRequestPort, 22 | setRequestPath, setRequestHost, setRequestManager, 23 | setRequestSecure, httpLBS, getResponseBody, 24 | getResponseStatusCode) 25 | import Network.HTTP.Client.OpenSSL (defaultMakeContext, defaultOpenSSLSettings, 26 | opensslManagerSettings, osslSettingsVerifyMode) 27 | import OpenSSL.Session (contextSetDefaultVerifyPaths, 28 | VerificationMode (VerifyNone, VerifyPeer), 29 | vpFailIfNoPeerCert, vpClientOnce, vpCallback) 30 | import System.Environment (getEnvironment) 31 | import System.IO (BufferMode (LineBuffering), hSetBuffering, stderr, stdout) 32 | import System.Posix.Process (executeFile) 33 | 34 | import qualified Control.Concurrent.Async as Async 35 | import qualified Control.Retry as Retry 36 | import qualified Data.Aeson as Aeson 37 | import qualified Data.ByteString as ByteString 38 | import qualified Data.ByteString.Char8 as SBS 39 | import qualified Data.ByteString.Lazy as LBS hiding (unpack) 40 | import qualified Data.ByteString.Lazy.Char8 as LBS 41 | import qualified Data.Foldable as Foldable 42 | import qualified Data.HashMap.Strict as HashMap 43 | import qualified Data.Map as Map 44 | import qualified Data.Text.Encoding as Text 45 | import qualified Data.Text.Encoding.Error as Text 46 | import qualified System.Exit as Exit 47 | 48 | import Config (AuthMethod (..), Options(..), parseOptions, unMilliSeconds, 49 | LogLevel(..), readConfigFromEnvFiles, getOptionsValue, 50 | Validated, Completed, DuplicateVariableBehavior (..)) 51 | import KeyMap (KeyMap) 52 | import SecretsFile (Secret(..), SFError(..), readSecretsFile) 53 | import Response (ClientToken (..)) 54 | 55 | import qualified KeyMap as KM 56 | 57 | -- | Make a HTTP URL path from a secret. This is the path that Vault expects. 58 | secretRequestPath :: MountInfo -> Secret -> String 59 | secretRequestPath (MountInfo mountInfo) secret = "/v1/" <> sMount secret <> foo <> sPath secret 60 | where 61 | foo = case KM.lookupDefault KV1 (KM.fromString $ sMount secret <> "/") mountInfo of 62 | KV1 -> "/" 63 | KV2 -> "/data/" 64 | 65 | type EnvVar = (String, String) 66 | 67 | data MountInfo = MountInfo (KeyMap EngineType) 68 | deriving (Show) 69 | 70 | data Context 71 | = Context 72 | { cLocalEnvVars :: [EnvVar] 73 | , cCliOptions :: Options Validated Completed 74 | , cHttpManager :: Manager 75 | , cExtraEnvVars :: [EnvVar] 76 | -- ^ Variables we want to inject into the command's environment that were not 77 | -- in the local environment when vaultenv was called, nor fetched via vault 78 | } 79 | 80 | -- | The different types of Engine that Vautlenv supports 81 | data EngineType = KV1 | KV2 82 | deriving (Show) 83 | 84 | data VaultData = VaultData (HashMap String Aeson.Value) 85 | 86 | -- Vault has two versions of the Key/Value backend. We try to parse both 87 | -- response formats here and return the one which we can parse correctly. 88 | -- 89 | -- This means there is some inference going on by vaultenv, but since 90 | -- we can find out which format we're parsing unambiguously, we just go 91 | -- with the inference instead of making the user specify this. The benefit 92 | -- here is that users can upgrade the version of the KV secret engine 93 | -- without having to change their config files. 94 | -- 95 | -- KV version 1 looks like this: 96 | -- 97 | -- @ 98 | -- { 99 | -- "auth": null, 100 | -- "data": { 101 | -- "foo": "bar" 102 | -- }, 103 | -- "lease_duration": 2764800, 104 | -- "lease_id": "", 105 | -- "renewable": false 106 | -- } 107 | -- @ 108 | -- 109 | -- And version 2 looks like this: 110 | -- 111 | -- @ 112 | -- { 113 | -- "data": { 114 | -- "data": { 115 | -- "foo": "bar" 116 | -- }, 117 | -- "metadata": { 118 | -- "created_time": "2018-03-22T02:24:06.945319214Z", 119 | -- "deletion_time": "", 120 | -- "destroyed": false, 121 | -- "version": 1 122 | -- } 123 | -- }, 124 | -- } 125 | -- @ 126 | instance FromJSON VaultData where 127 | parseJSON = 128 | let 129 | parseV1 obj = do 130 | keyValuePairs <- obj .: "data" 131 | VaultData <$> Aeson.parseJSON keyValuePairs 132 | parseV2 obj = do 133 | nested <- obj .: "data" 134 | flip (Aeson.withObject "nested") nested $ \obj' -> do 135 | keyValuePairs <- obj' .: "data" 136 | VaultData <$> Aeson.parseJSON keyValuePairs 137 | in Aeson.withObject "VaultData" $ \obj -> parseV2 obj <|> parseV1 obj 138 | 139 | -- Parses a very mixed type of response that Vault gives back when you 140 | -- request /sys/mounts. It has a garbage format. We primarily care about 141 | -- this information: 142 | -- 143 | -- { 144 | -- "secret/": { 145 | -- "options": { 146 | -- "version": "1" 147 | -- }, 148 | -- "type": "kv" 149 | -- }, 150 | -- } 151 | -- 152 | -- But there is a whole load of other stuff in the top level object. 153 | -- Integers, dates, strings, null values. Get the stuff we need 154 | -- and get out. 155 | instance FromJSON MountInfo where 156 | parseJSON = 157 | let 158 | getType = Aeson.withObject "MountSpec" $ \o -> 159 | o .: "type" >>= (Aeson.withText "mount type" $ (\case 160 | "kv" -> do 161 | options <- o .: "options" 162 | Aeson.withObject "Options" (\opts -> do 163 | version <- opts .: "version" 164 | case version of 165 | Aeson.String "1" -> pure KV1 166 | Aeson.String "2" -> pure KV2 167 | _ -> fail "unknown version number") options 168 | _ -> fail "expected a KV type")) 169 | in 170 | Aeson.withObject "MountResp" $ \obj -> 171 | pure $ MountInfo (KM.mapMaybe (\v -> parseMaybe getType v) obj) 172 | 173 | -- | Error modes of this program. 174 | -- 175 | -- Every part of the program that can fail has an error type. These can bubble 176 | -- up the call stack and end up as a value of this type. We then have a single 177 | -- function which is responsible for printing an error message and exiting. 178 | data VaultError 179 | = SecretNotFound String 180 | | SecretFileError SFError 181 | | KeyNotFound Secret 182 | | WrongType Secret 183 | | BadRequest LBS.ByteString 184 | | Forbidden 185 | | BadJSONResp LBS.ByteString String -- Body and error 186 | | ServerError LBS.ByteString 187 | | ServerUnavailable LBS.ByteString 188 | | ServerUnreachable HttpException 189 | | InvalidUrl String 190 | | DuplicateVar String 191 | | Unspecified Int LBS.ByteString 192 | | KubernetesJwtInvalidUtf8 193 | | KubernetesJwtFailedRead 194 | deriving Show 195 | 196 | -- | "Handle" a HttpException by wrapping it in a Left VaultError. 197 | -- We also edit the Request contained in the exception to remove the 198 | -- Vault token, as it would otherwise get printed to stderr if we error 199 | -- out. 200 | httpErrorHandler :: HttpException -> IO (Either VaultError b) 201 | httpErrorHandler (e :: HttpException) = case e of 202 | (HttpExceptionRequest request reason) -> 203 | let sanitizedRequest = sanitizeRequest request 204 | in pure $ Left $ ServerUnreachable (HttpExceptionRequest sanitizedRequest reason) 205 | (InvalidUrlException url _reason) -> pure $ Left $ InvalidUrl url 206 | 207 | where 208 | sanitizeRequest :: Request -> Request 209 | sanitizeRequest = setRequestHeader "x-vault-token" ["**removed**"] 210 | 211 | -- | Retry configuration to use for network requests to Vault. 212 | -- We use a limited exponential backoff with the policy 213 | -- fullJitterBackoff that comes with the Retry package. 214 | vaultRetryPolicy :: (MonadIO m) => Options Validated Completed -> Retry.RetryPolicyM m 215 | vaultRetryPolicy opts = Retry.fullJitterBackoff (unMilliSeconds 216 | (getOptionsValue oRetryBaseDelay opts) * 1000 217 | ) 218 | <> Retry.limitRetries ( 219 | getOptionsValue oRetryAttempts opts 220 | ) 221 | 222 | -- | Perform the given action according to our retry policy. 223 | doWithRetries :: Retry.RetryPolicyM IO -> (Retry.RetryStatus -> IO (Either VaultError a)) -> IO (Either VaultError a) 224 | doWithRetries retryPolicy = Retry.retrying retryPolicy isRetryableFailure 225 | where 226 | -- | Indicator function for retrying to retry on VaultErrors (Lefts) that 227 | -- shouldRetry thinks we should retry on. Needs to be in IO because the 228 | -- actions to perform are in IO as well. 229 | isRetryableFailure :: Retry.RetryStatus -> Either VaultError a -> IO Bool 230 | isRetryableFailure _retryStatus (Right _) = pure False 231 | isRetryableFailure _retryStatus (Left err) = pure $ case err of 232 | ServerError _ -> True 233 | ServerUnavailable _ -> True 234 | ServerUnreachable _ -> True 235 | Unspecified _ _ -> True 236 | BadJSONResp _ _ -> True 237 | 238 | -- Errors where we don't retry 239 | BadRequest _ -> False 240 | Forbidden -> False 241 | InvalidUrl _ -> False 242 | SecretNotFound _ -> False 243 | KubernetesJwtInvalidUtf8 -> False 244 | KubernetesJwtFailedRead -> False 245 | 246 | -- Errors that cannot occur at this point, but we list for 247 | -- exhaustiveness checking. 248 | KeyNotFound _ -> False 249 | DuplicateVar _ -> False 250 | SecretFileError _ -> False 251 | WrongType _ -> False 252 | 253 | -- 254 | -- IO 255 | -- 256 | 257 | main :: IO () 258 | main = do 259 | -- When the runtime detects that stdout is not connected to a console, it 260 | -- defaults to block buffering instead of line buffering. When running under 261 | -- systemd or a container manager, this prevents log messages from showing up 262 | -- until the buffer is flushed, and it breaks interleaving of stdout and 263 | -- stderr print. Therefore, explicitly select line buffering, to enforce a 264 | -- flush after every newline. 265 | hSetBuffering stdout LineBuffering 266 | hSetBuffering stderr LineBuffering 267 | 268 | localEnvVars <- getEnvironment 269 | envFileSettings <- readConfigFromEnvFiles 270 | 271 | cliAndEnvAndEnvFileOptions <- parseOptions localEnvVars envFileSettings 272 | 273 | let envAndEnvFileConfig = nubBy (\(x, _) (y, _) -> x == y) 274 | (localEnvVars ++ concat (reverse envFileSettings)) 275 | 276 | if getOptionsValue oLogLevel cliAndEnvAndEnvFileOptions <= Info 277 | then print cliAndEnvAndEnvFileOptions 278 | else pure () 279 | 280 | httpManager <- getHttpManager cliAndEnvAndEnvFileOptions 281 | 282 | let context = Context { cLocalEnvVars = envAndEnvFileConfig 283 | , cCliOptions = cliAndEnvAndEnvFileOptions 284 | , cHttpManager = httpManager 285 | , cExtraEnvVars = [] 286 | } 287 | 288 | vaultEnv context >>= \case 289 | Left err -> Exit.die (vaultErrorLogMessage err) 290 | Right newEnv -> runCommand cliAndEnvAndEnvFileOptions newEnv 291 | 292 | 293 | -- | This function returns either a manager for plain HTTP or 294 | -- for HTTPS connections. If TLS is wanted, we also check if the 295 | -- user specified an option to disable the certificate check. 296 | getHttpManager :: Options Validated Completed -> IO Manager 297 | getHttpManager opts = newManager $ applyConfig basicManagerSettings 298 | where 299 | maxConnections = getOptionsValue oMaxConcurrentRequests opts 300 | applyConfig settings = settings 301 | -- Allow the manager to keep as many connections live as were requested. 302 | -- Unless we use the unlimited flag, in that case, use the default value. 303 | { managerConnCount = if maxConnections > 0 then maxConnections else managerConnCount settings 304 | } 305 | basicManagerSettings = 306 | (opensslManagerSettings makeContext) 307 | { managerConnCount = maxConnections } 308 | makeContext = do 309 | context <- defaultMakeContext opensslSettings 310 | contextSetDefaultVerifyPaths context 311 | pure context 312 | opensslSettings = defaultOpenSSLSettings 313 | { osslSettingsVerifyMode = 314 | if not $ getOptionsValue oValidateCerts opts 315 | then VerifyNone 316 | else VerifyPeer 317 | { 318 | vpFailIfNoPeerCert = True, 319 | vpClientOnce = True, 320 | vpCallback = Nothing 321 | } 322 | } 323 | 324 | -- | Main logic of our application. 325 | -- 326 | -- We first retrieve the mount information from Vault, as this is needed to 327 | -- construct the URLs of the secrets to fetch. 328 | -- With this information we fetch the secrets from Vault, check for duplicates, 329 | -- and then yield the list of environment variables to make available to the 330 | -- process we want to run eventually. 331 | -- Based on the settings in the context we either scrub the environment that 332 | -- already existed or keep it (applying the inheritance blacklist if it exists). 333 | -- 334 | -- Signals failure through a value of type VaultError. 335 | vaultEnv :: Context -> IO (Either VaultError [EnvVar]) 336 | vaultEnv originalContext = 337 | readSecretsFile secretFile >>= \case 338 | Left sfError -> pure $ Left $ SecretFileError sfError 339 | Right secrets -> 340 | doWithRetries retryPolicy (authenticate originalContext) >>= \case 341 | Left vaultError -> pure $ Left vaultError 342 | Right authenticatedContext -> 343 | doWithRetries retryPolicy (getMountInfo authenticatedContext) >>= \case 344 | Left vaultError -> pure $ Left vaultError 345 | Right mountInfo -> 346 | requestSecrets authenticatedContext mountInfo secrets >>= \case 347 | Left vaultError -> pure $ Left vaultError 348 | Right secretEnv -> pure $ handleDuplicates $ 349 | buildEnv (cExtraEnvVars authenticatedContext ++ secretEnv) 350 | where 351 | retryPolicy = vaultRetryPolicy (cCliOptions originalContext) 352 | 353 | authenticate :: Context -> Retry.RetryStatus -> IO (Either VaultError Context) 354 | authenticate context _retryStatus = case oAuthMethod (cCliOptions context) of 355 | -- If we have a token already, or if we should not authenticate at all, 356 | -- then there is nothing to be done here. But if Kubernetes or Github 357 | -- auth is enabled, then we do that here and then fill out the token. 358 | AuthVaultToken _ -> pure $ Right context 359 | AuthNone -> pure $ Right context 360 | AuthGitHub token -> handleVaultAuthResponse context (requestGitHubVaultToken context token) 361 | AuthKubernetes role -> handleVaultAuthResponse context (requestKubernetesVaultToken context role) 362 | 363 | handleVaultAuthResponse :: Context -> IO (Either VaultError ClientToken) -> IO (Either VaultError Context) 364 | handleVaultAuthResponse context f = 365 | catch f httpErrorHandler >>= \case 366 | Left vaultError -> pure $ Left vaultError 367 | Right (ClientToken token) -> do 368 | when (getOptionsValue oLogLevel (cCliOptions context) <= Info) $ 369 | putStrLn "[INFO] Authentication successful, we have a VAULT_TOKEN now." 370 | pure $ Right context 371 | { cCliOptions = (cCliOptions context) 372 | { oAuthMethod = AuthVaultToken token 373 | } 374 | , cExtraEnvVars = [("VAULT_TOKEN", unpack token)] 375 | } 376 | 377 | getMountInfo :: Context -> Retry.RetryStatus -> IO (Either VaultError MountInfo) 378 | getMountInfo context _retryStatus = catch (requestMountInfo context) httpErrorHandler 379 | 380 | secretFile = getOptionsValue oSecretFile (cCliOptions originalContext) 381 | 382 | allowDuplicates = getOptionsValue oDuplicateVariablesBehavior (cCliOptions originalContext) 383 | 384 | -- Handle duplicates baseup 385 | handleDuplicates = case allowDuplicates of 386 | DuplicateError -> checkNoDuplicates 387 | DuplicateKeep -> Right . preferLast 388 | DuplicateOverwrite -> Right . preferFirst 389 | 390 | -- | Check that the given list of EnvVars contains no duplicate 391 | -- variables, return a DuplicateVar error if it does. 392 | checkNoDuplicates :: [EnvVar] -> Either VaultError [EnvVar] 393 | checkNoDuplicates vars = case dups (map fst vars) of 394 | Right () -> Right vars 395 | Left var -> Left $ DuplicateVar var 396 | 397 | -- Prefer the last element in the list with environment variables. This 398 | -- is a somewhat inefficient way, but the list here is small, so performance shouldn't 399 | -- be a problem. 400 | preferLast :: [EnvVar] -> [EnvVar] 401 | preferLast = reverse . preferFirst . reverse 402 | 403 | preferFirst :: [EnvVar] -> [EnvVar] 404 | preferFirst = nubBy (\(x, _) (y, _) -> x == y) 405 | 406 | -- We need to check duplicates in the environment and fail if 407 | -- there are any. `dups` runs in O(n^2), 408 | -- but this shouldn't matter for our small lists. 409 | -- 410 | -- Equality is determined on the first element of the env var 411 | -- tuples. 412 | dups :: Eq a => [a] -> Either a () 413 | dups [] = Right () 414 | dups (x:xs) | isDup x xs = Left x 415 | | otherwise = dups xs 416 | 417 | isDup x = foldr (\y acc -> acc || x == y) False 418 | 419 | -- | Build the resulting environment for the process to start, given the 420 | -- list of environment variables that were retrieved from Vault. Return 421 | -- either only the retrieved secrets (if --no-inherit-env is used), or 422 | -- merge the retrieved variables with the environment where Vaultenv was 423 | -- called and apply the blacklist. 424 | buildEnv :: [EnvVar] -> [EnvVar] 425 | buildEnv secretsEnv = 426 | if getOptionsValue oInheritEnv . cCliOptions $ originalContext 427 | then removeBlacklistedVars $ secretsEnv ++ cLocalEnvVars originalContext 428 | else secretsEnv 429 | 430 | where 431 | inheritEnvBlacklist = getOptionsValue oInheritEnvBlacklist . cCliOptions $ originalContext 432 | removeBlacklistedVars = filter (not . flip elem inheritEnvBlacklist . fst) 433 | 434 | 435 | runCommand :: Options Validated Completed -> [EnvVar] -> IO a 436 | runCommand options env = 437 | let 438 | command = getOptionsValue oCmd options 439 | searchPath = getOptionsValue oUsePath options 440 | args = getOptionsValue oArgs options 441 | env' = Just env 442 | in 443 | -- `executeFile` calls one of the syscalls in the execv* family, which 444 | -- replaces the current process with `command`. It does not return. 445 | executeFile command searchPath args env' 446 | 447 | 448 | -- | Add Vault authentication token to a request, if set in the options. If 449 | -- not, the request is returned unmodified. 450 | addVaultToken :: Options Validated Completed -> Request -> Request 451 | addVaultToken options request = case oAuthMethod options of 452 | AuthVaultToken token -> setRequestHeader "x-vault-token" [Text.encodeUtf8 token] request 453 | AuthGitHub _ -> error "GitHub auth method should have been resolved to token by now." 454 | AuthKubernetes _ -> error "Kubernetes auth method should have been resolved to token by now." 455 | AuthNone -> request 456 | 457 | unauthenticatedVaultRequest :: Context -> String -> Request 458 | unauthenticatedVaultRequest context path = 459 | let 460 | cliOptions = cCliOptions context 461 | in 462 | setRequestManager (cHttpManager context) 463 | $ setRequestHeader "x-vault-request" ["true"] 464 | $ setRequestPath (SBS.pack path) 465 | $ setRequestPort (getOptionsValue oVaultPort cliOptions) 466 | $ setRequestHost (SBS.pack (getOptionsValue oVaultHost cliOptions)) 467 | $ setRequestSecure (getOptionsValue oConnectTls cliOptions) 468 | $ defaultRequest 469 | 470 | kubernetesJwtPath :: FilePath 471 | kubernetesJwtPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" 472 | 473 | -- | Read the contents of `/var/run/secrets/kubernetes.io/serviceaccount/token`. 474 | readKubernetesJwt :: IO (Either VaultError Text) 475 | readKubernetesJwt = 476 | let 477 | readJwtFile = do 478 | contentsUtf8 <- ByteString.readFile kubernetesJwtPath 479 | pure $ Right $ Text.decodeUtf8With Text.strictDecode contentsUtf8 480 | in 481 | readJwtFile `catches` 482 | [ Handler $ \(_ :: Text.UnicodeException) -> pure $ Left KubernetesJwtInvalidUtf8 483 | , Handler $ \(_ :: IOError) -> pure $ Left KubernetesJwtFailedRead 484 | ] 485 | 486 | -- | Authenticate using GitHub auth, see https://www.vaultproject.io/docs/auth/github. 487 | requestGitHubVaultToken :: Context -> Text -> IO (Either VaultError ClientToken) 488 | requestGitHubVaultToken context ghtoken = let 489 | bodyJson = Aeson.Object $ KM.singleton "token" (Aeson.String ghtoken) 490 | backendName = getOptionsValue oVaultAuthBackend $ cCliOptions context 491 | request = setRequestBodyJSON bodyJson 492 | $ setRequestMethod "POST" 493 | $ unauthenticatedVaultRequest context 494 | $ "/v1/auth/" ++ backendName ++ "/login" 495 | in decodeVaultAuthResponse <$> httpLBS request 496 | 497 | -- | Authenticate using Kubernetes auth, see https://www.vaultproject.io/docs/auth/kubernetes. 498 | requestKubernetesVaultToken :: Context -> Text -> IO (Either VaultError ClientToken) 499 | requestKubernetesVaultToken context role = do 500 | when (getOptionsValue oLogLevel (cCliOptions context) <= Info) $ 501 | putStrLn "[INFO] Authenticating with Kubernetes ..." 502 | jwtResult <- readKubernetesJwt 503 | case jwtResult of 504 | Left err -> pure $ Left err 505 | Right jwt -> 506 | let 507 | bodyJson = Aeson.Object $ KM.fromList 508 | [ ("jwt", Aeson.String jwt) 509 | , ("role", Aeson.String role) 510 | ] 511 | backendName = getOptionsValue oVaultAuthBackend $ cCliOptions context 512 | request = 513 | setRequestBodyJSON bodyJson 514 | $ setRequestMethod "POST" 515 | $ unauthenticatedVaultRequest context 516 | $ "/v1/auth/" ++ backendName ++ "/login" 517 | in decodeVaultAuthResponse <$> httpLBS request 518 | 519 | decodeVaultAuthResponse :: Response LBS.ByteString -> Either VaultError ClientToken 520 | decodeVaultAuthResponse response = 521 | case getResponseStatusCode response of 522 | 200 -> case Aeson.eitherDecode' (getResponseBody response) of 523 | Left err -> Left $ BadJSONResp (getResponseBody response) err 524 | Right token -> Right token 525 | _notOk -> Left $ ServerError $ getResponseBody response 526 | 527 | -- | Look up what mounts are available and what type they have. 528 | requestMountInfo :: Context -> IO (Either VaultError MountInfo) 529 | requestMountInfo context = 530 | let 531 | cliOptions = cCliOptions context 532 | request = addVaultToken cliOptions $ unauthenticatedVaultRequest context "/v1/sys/mounts" 533 | in do 534 | -- 'httpLBS' throws an IO Exception ('HttpException') if it fails to complete the request. 535 | -- We intentionally don't capture this here, but handle it with the retries in 'vaultEnv' instead. 536 | resp <- httpLBS request 537 | let decodeResult = Aeson.eitherDecode' (getResponseBody resp) :: Either String MountInfo 538 | 539 | case decodeResult of 540 | Left errorMsg -> pure $ Left $ BadJSONResp (getResponseBody resp) errorMsg 541 | Right result -> pure $ Right result 542 | 543 | -- | Request all the data associated with a secret from the vault. 544 | -- 545 | -- This function automatically retries the request if it fails according to the 546 | -- retryPolicy set in the given context. 547 | requestSecret :: Context -> String -> IO (Either VaultError VaultData) 548 | requestSecret context secretPath = 549 | let 550 | cliOptions = cCliOptions context 551 | retryPolicy = vaultRetryPolicy cliOptions 552 | request = addVaultToken cliOptions $ unauthenticatedVaultRequest context secretPath 553 | 554 | getSecret :: Retry.RetryStatus -> IO (Either VaultError VaultData) 555 | getSecret _retryStatus = catch (doRequest secretPath request) httpErrorHandler 556 | 557 | in do 558 | when (getOptionsValue oLogLevel (cCliOptions context) <= Info) $ 559 | putStrLn $ "[INFO] Getting " <> secretPath <> " ..." 560 | doWithRetries retryPolicy getSecret 561 | 562 | -- | Request all the supplied secrets from the vault, but just once, even if 563 | -- multiple keys are specified for a single secret. This is an optimization in 564 | -- order to avoid unnecessary round trips and DNS requests. 565 | requestSecrets :: Context -> MountInfo -> [Secret] -> IO (Either VaultError [EnvVar]) 566 | requestSecrets context mountInfo secrets = do 567 | let 568 | secretPaths = Foldable.foldMap (\x -> Map.singleton x x) $ fmap (secretRequestPath mountInfo) secrets 569 | concurrentRequests = getOptionsValue oMaxConcurrentRequests (cCliOptions context) 570 | 571 | -- Limit the number of concurrent requests with a semaphore 572 | requestSemaphore <- newQSem concurrentRequests 573 | let 574 | withSemaphore 575 | | concurrentRequests == 0 = id 576 | | otherwise = bracket_ (waitQSem requestSemaphore) (signalQSem requestSemaphore) 577 | 578 | secretData <- liftIO (Async.mapConcurrently (withSemaphore . requestSecret context) secretPaths) 579 | pure $ sequence secretData >>= lookupSecrets mountInfo secrets 580 | 581 | -- | Look for the requested keys in the secret data that has been previously fetched. 582 | lookupSecrets :: MountInfo -> [Secret] -> Map.Map String VaultData -> Either VaultError [EnvVar] 583 | lookupSecrets mountInfo secrets vaultData = forM secrets $ \secret -> 584 | let secretData = Map.lookup (secretRequestPath mountInfo secret) vaultData 585 | secretValue = secretData >>= (\(VaultData vd) -> HashMap.lookup (sKey secret) vd) 586 | toEnvVar (Aeson.String val) = Right (sVarName secret, unpack val) 587 | toEnvVar _ = Left (WrongType secret) 588 | in maybe (Left $ KeyNotFound secret) toEnvVar $ secretValue 589 | 590 | -- | Send a request for secrets to the vault and parse the response. 591 | doRequest :: String -> Request -> IO (Either VaultError VaultData) 592 | doRequest secretPath request = do 593 | -- As in 'requestMountInfo': 'httpLBS' throws a 'HttpException' in IO if it 594 | -- fails to complete the request, this is handled in 'vaultEnv' by the retry logic. 595 | resp <- httpLBS request 596 | pure $ parseResponse secretPath resp 597 | 598 | -- 599 | -- HTTP response handling 600 | -- 601 | 602 | parseResponse :: String -> Response LBS.ByteString -> Either VaultError VaultData 603 | parseResponse secretPath response = 604 | let 605 | responseBody = getResponseBody response 606 | statusCode = getResponseStatusCode response 607 | in case statusCode of 608 | 200 -> parseSuccessResponse responseBody 609 | 400 -> Left $ BadRequest responseBody 610 | 403 -> Left Forbidden 611 | 404 -> Left $ SecretNotFound secretPath 612 | 500 -> Left $ ServerError responseBody 613 | 503 -> Left $ ServerUnavailable responseBody 614 | _ -> Left $ Unspecified statusCode responseBody 615 | 616 | 617 | parseSuccessResponse :: LBS.ByteString -> Either VaultError VaultData 618 | parseSuccessResponse responseBody = first 619 | (BadJSONResp responseBody) 620 | (Aeson.eitherDecode' responseBody) 621 | 622 | -- 623 | -- Utility functions 624 | -- 625 | 626 | vaultErrorLogMessage :: VaultError -> String 627 | vaultErrorLogMessage vaultError = 628 | let 629 | description = case vaultError of 630 | SecretNotFound secretPath -> 631 | "Secret not found: " <> secretPath 632 | SecretFileError sfe -> show sfe 633 | KeyNotFound secret -> 634 | "Key " <> (sKey secret) <> " not found for path " <> (sPath secret) 635 | WrongType secret -> 636 | "Key " <> (sKey secret) <> " in path " <> (sPath secret) <> " is not a String" 637 | DuplicateVar varName -> 638 | "Found duplicate environment variable \"" ++ varName ++ "\"" 639 | BadRequest resp -> 640 | "Made a bad request: " <> (LBS.unpack resp) 641 | Forbidden -> 642 | "Forbidden: The provided token is not valid or expired. If you're not \ 643 | \explictly passing a token, check if the configured token in \ 644 | \`~/.config/vaultenv/vaultenv.conf` is still valid" 645 | InvalidUrl secretPath -> 646 | "Secret " <> secretPath <> " contains characters that are illegal in URLs" 647 | BadJSONResp body msg -> 648 | "Received bad JSON from Vault: " <> msg <> ". Response was: " <> (LBS.unpack body) 649 | ServerError resp -> 650 | "Internal Vault error: " <> (LBS.unpack resp) 651 | ServerUnavailable resp -> 652 | "Vault is unavailable for requests. It can be sealed, " <> 653 | "under maintenance or enduring heavy load: " <> (LBS.unpack resp) 654 | ServerUnreachable exception -> 655 | "ServerUnreachable error: " <> show exception 656 | Unspecified status resp -> 657 | "Received an error that I don't know about (" <> show status 658 | <> "): " <> (LBS.unpack resp) 659 | KubernetesJwtFailedRead -> 660 | "Failed to read '" <> kubernetesJwtPath <> "'." 661 | KubernetesJwtInvalidUtf8 -> 662 | "Contents of '" <> kubernetesJwtPath <> "' is not valid UTF-8." 663 | in 664 | "[ERROR] " <> description 665 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import ./nix/nixpkgs-pinned.nix 2 | # Allow vault to be used only as a part of this development shell. 3 | # We are not allowing the use of vault as a part of our final package 4 | # because vault-1.16.1 is lisenced under BSL-1.1 5 | { config.allowUnfreePredicate = pkg: 6 | (pkgs.lib.getName pkg) == "vault" && 7 | (pkgs.lib.getVersion pkg) == "1.17.2"; 8 | } 9 | }: 10 | with pkgs; buildEnv { 11 | name = "vaultenv-devenv"; 12 | paths = [ 13 | glibcLocales # So you can export LOCALE_ARCHIVE=$(nix path-info)/lib/locale/locale-archive. 14 | niv 15 | perl # For "prove" 16 | python3 17 | stack 18 | vault 19 | ]; 20 | } 21 | -------------------------------------------------------------------------------- /nix/haskell-dependencies.nix: -------------------------------------------------------------------------------- 1 | 2 | haskellPackages: 3 | with haskellPackages; 4 | [ 5 | QuickCheck 6 | aeson 7 | async 8 | base 9 | bytestring 10 | crypton-connection 11 | containers 12 | directory 13 | dotenv 14 | hspec 15 | hspec-discover 16 | hspec-expectations 17 | http-client 18 | http-client-openssl 19 | http-conduit 20 | megaparsec 21 | optparse-applicative 22 | parser-combinators 23 | quickcheck-instances 24 | retry 25 | text 26 | unix 27 | unordered-containers 28 | utf8-string 29 | ] 30 | -------------------------------------------------------------------------------- /nix/haskell-overlay.nix: -------------------------------------------------------------------------------- 1 | self: super: { 2 | } 3 | -------------------------------------------------------------------------------- /nix/nixpkgs-pinned.nix: -------------------------------------------------------------------------------- 1 | # Provide almost the same arguments as the actual nixpkgs. 2 | # This allows us to further configure this nixpkgs instantiation in places where we need it. 3 | # In particular, `stack` needs this to be a function. 4 | { overlays ? [ ] # additional overlays 5 | , config ? { } # Imported configuration 6 | }: 7 | let 8 | sources = import ./sources.nix; 9 | 10 | nixpkgs = import sources.nixpkgs { 11 | overlays = [(import ./overlay.nix)] ++ overlays; 12 | inherit config; 13 | }; 14 | in 15 | nixpkgs 16 | -------------------------------------------------------------------------------- /nix/overlay.nix: -------------------------------------------------------------------------------- 1 | self: super: 2 | let 3 | haskellOverlay = import ./haskell-overlay.nix; 4 | in { 5 | vaultenvHaskellPackages = super.haskell.packages.ghc964.extend haskellOverlay; 6 | } 7 | -------------------------------------------------------------------------------- /nix/release.nix: -------------------------------------------------------------------------------- 1 | let 2 | pkgs = import ./nixpkgs-pinned.nix {}; 3 | in { 4 | # Normal cabal build where Nix handles dependencies. 5 | vaultenv = pkgs.haskellPackages.callPackage ../vaultenv.nix {}; 6 | 7 | # Static package build 8 | vaultenvStatic = (pkgs.pkgsStatic.haskellPackages.callPackage ../vaultenv.nix { 9 | # Use non-static version of glibc locales, as the static version fails to build. 10 | glibcLocales = pkgs.glibcLocales; 11 | }).overrideAttrs (prev: { 12 | # Disable the check phase, as hspec-discover will not run. 13 | # TODO: investigate what's happening with hspec-discover. 14 | doCheck = false; 15 | 16 | # Remove the fixupPhase, as for some reason it still tries to strip the statically built 17 | # executable. 18 | # TODO: Cleaner way of disabling the fixup phase. 19 | fixupPhase = " "; 20 | }); 21 | } 22 | -------------------------------------------------------------------------------- /nix/sources.json: -------------------------------------------------------------------------------- 1 | { 2 | "nixpkgs": { 3 | "branch": "nixpkgs-unstable", 4 | "description": "Nix Packages collection", 5 | "homepage": "", 6 | "owner": "NixOS", 7 | "repo": "nixpkgs", 8 | "rev": "453402b94f39f968a7c27df28e060f69e4a50c3b", 9 | "sha256": "10ipmhb34ccrbndiryzbgqfdjaw1w7c05wi22yg45m605nxsl3w9", 10 | "type": "tarball", 11 | "url": "https://github.com/NixOS/nixpkgs/archive/453402b94f39f968a7c27df28e060f69e4a50c3b.tar.gz", 12 | "url_template": "https://github.com///archive/.tar.gz" 13 | }, 14 | "static-haskell-nix": { 15 | "branch": "master", 16 | "description": "easily build most Haskell programs into fully static Linux executables", 17 | "homepage": "", 18 | "owner": "nh2", 19 | "repo": "static-haskell-nix", 20 | "rev": "481e7d73ca624278ef0f840a0a2ba09e3a583217", 21 | "sha256": "0y4hzk1jxp4fdjksg6p1q6g5i4xw7cmb50vg5np7z5ipk4y4gc2x", 22 | "type": "tarball", 23 | "url": "https://github.com/nh2/static-haskell-nix/archive/481e7d73ca624278ef0f840a0a2ba09e3a583217.tar.gz", 24 | "url_template": "https://github.com///archive/.tar.gz" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /nix/sources.nix: -------------------------------------------------------------------------------- 1 | # This file has been generated by Niv. 2 | 3 | let 4 | 5 | # 6 | # The fetchers. fetch_ fetches specs of type . 7 | # 8 | 9 | fetch_file = pkgs: name: spec: 10 | let 11 | name' = sanitizeName name + "-src"; 12 | in 13 | if spec.builtin or true then 14 | builtins_fetchurl { inherit (spec) url sha256; name = name'; } 15 | else 16 | pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; 17 | 18 | fetch_tarball = pkgs: name: spec: 19 | let 20 | name' = sanitizeName name + "-src"; 21 | in 22 | if spec.builtin or true then 23 | builtins_fetchTarball { name = name'; inherit (spec) url sha256; } 24 | else 25 | pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; 26 | 27 | fetch_git = name: spec: 28 | let 29 | ref = 30 | spec.ref or ( 31 | if spec ? branch then "refs/heads/${spec.branch}" else 32 | if spec ? tag then "refs/tags/${spec.tag}" else 33 | abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" 34 | ); 35 | submodules = spec.submodules or false; 36 | submoduleArg = 37 | let 38 | nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; 39 | emptyArgWithWarning = 40 | if submodules 41 | then 42 | builtins.trace 43 | ( 44 | "The niv input \"${name}\" uses submodules " 45 | + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " 46 | + "does not support them" 47 | ) 48 | { } 49 | else { }; 50 | in 51 | if nixSupportsSubmodules 52 | then { inherit submodules; } 53 | else emptyArgWithWarning; 54 | in 55 | builtins.fetchGit 56 | ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); 57 | 58 | fetch_local = spec: spec.path; 59 | 60 | fetch_builtin-tarball = name: throw 61 | ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. 62 | $ niv modify ${name} -a type=tarball -a builtin=true''; 63 | 64 | fetch_builtin-url = name: throw 65 | ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. 66 | $ niv modify ${name} -a type=file -a builtin=true''; 67 | 68 | # 69 | # Various helpers 70 | # 71 | 72 | # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 73 | sanitizeName = name: 74 | ( 75 | concatMapStrings (s: if builtins.isList s then "-" else s) 76 | ( 77 | builtins.split "[^[:alnum:]+._?=-]+" 78 | ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) 79 | ) 80 | ); 81 | 82 | # The set of packages used when specs are fetched using non-builtins. 83 | mkPkgs = sources: system: 84 | let 85 | sourcesNixpkgs = 86 | import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; 87 | hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; 88 | hasThisAsNixpkgsPath = == ./.; 89 | in 90 | if builtins.hasAttr "nixpkgs" sources 91 | then sourcesNixpkgs 92 | else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then 93 | import { } 94 | else 95 | abort 96 | '' 97 | Please specify either (through -I or NIX_PATH=nixpkgs=...) or 98 | add a package called "nixpkgs" to your sources.json. 99 | ''; 100 | 101 | # The actual fetching function. 102 | fetch = pkgs: name: spec: 103 | 104 | if ! builtins.hasAttr "type" spec then 105 | abort "ERROR: niv spec ${name} does not have a 'type' attribute" 106 | else if spec.type == "file" then fetch_file pkgs name spec 107 | else if spec.type == "tarball" then fetch_tarball pkgs name spec 108 | else if spec.type == "git" then fetch_git name spec 109 | else if spec.type == "local" then fetch_local spec 110 | else if spec.type == "builtin-tarball" then fetch_builtin-tarball name 111 | else if spec.type == "builtin-url" then fetch_builtin-url name 112 | else 113 | abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; 114 | 115 | # If the environment variable NIV_OVERRIDE_${name} is set, then use 116 | # the path directly as opposed to the fetched source. 117 | replace = name: drv: 118 | let 119 | saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; 120 | ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; 121 | in 122 | if ersatz == "" then drv else 123 | # this turns the string into an actual Nix path (for both absolute and 124 | # relative paths) 125 | if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; 126 | 127 | # Ports of functions for older nix versions 128 | 129 | # a Nix version of mapAttrs if the built-in doesn't exist 130 | mapAttrs = builtins.mapAttrs or ( 131 | f: set: with builtins; 132 | listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) 133 | ); 134 | 135 | # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 136 | range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); 137 | 138 | # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 139 | stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); 140 | 141 | # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 142 | stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); 143 | concatMapStrings = f: list: concatStrings (map f list); 144 | concatStrings = builtins.concatStringsSep ""; 145 | 146 | # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 147 | optionalAttrs = cond: as: if cond then as else { }; 148 | 149 | # fetchTarball version that is compatible between all the versions of Nix 150 | builtins_fetchTarball = { url, name ? null, sha256 }@attrs: 151 | let 152 | inherit (builtins) lessThan nixVersion fetchTarball; 153 | in 154 | if lessThan nixVersion "1.12" then 155 | fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) 156 | else 157 | fetchTarball attrs; 158 | 159 | # fetchurl version that is compatible between all the versions of Nix 160 | builtins_fetchurl = { url, name ? null, sha256 }@attrs: 161 | let 162 | inherit (builtins) lessThan nixVersion fetchurl; 163 | in 164 | if lessThan nixVersion "1.12" then 165 | fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) 166 | else 167 | fetchurl attrs; 168 | 169 | # Create the final "sources" from the config 170 | mkSources = config: 171 | mapAttrs 172 | ( 173 | name: spec: 174 | if builtins.hasAttr "outPath" spec 175 | then 176 | abort 177 | "The values in sources.json should not have an 'outPath' attribute" 178 | else 179 | spec // { outPath = replace name (fetch config.pkgs name spec); } 180 | ) 181 | config.sources; 182 | 183 | # The "config" used by the fetchers 184 | mkConfig = 185 | { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null 186 | , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) 187 | , system ? builtins.currentSystem 188 | , pkgs ? mkPkgs sources system 189 | }: rec { 190 | # The sources, i.e. the attribute set of spec name to spec 191 | inherit sources; 192 | 193 | # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers 194 | inherit pkgs; 195 | }; 196 | 197 | in 198 | mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } 199 | -------------------------------------------------------------------------------- /nix/stack-shell.nix: -------------------------------------------------------------------------------- 1 | # This nix expression is used by stack for setting up the build environment. 2 | # It is referenced in `stack.yaml` so that stack uses it by default. 3 | { }: 4 | let 5 | nixpkgs = import ./nixpkgs-pinned.nix {}; 6 | getDependencies = import ./haskell-dependencies.nix; 7 | in 8 | nixpkgs.haskell.lib.buildStackProject { 9 | name = "vaultenv"; 10 | # This is the GHC that will be seen by stack, and it will come 11 | # bundled with all the dependencies listed in `haskell-dependencies.nix`. 12 | # This allows us to have stack use the dependencies from nixpkgs, 13 | # instead of fetching them itself. 14 | ghc = nixpkgs.vaultenvHaskellPackages.ghcWithPackages getDependencies; 15 | buildInputs = with nixpkgs; [ 16 | glibcLocales 17 | ]; 18 | } 19 | -------------------------------------------------------------------------------- /package.yaml: -------------------------------------------------------------------------------- 1 | # Changed the name here because nixpkgs also includes a vaultenv and we haven't 2 | # figured out how to mask that when we build vaultenv with nix itself. 3 | name: vaultenv-real 4 | version: 0.19.0 5 | synopsis: Runs processes with secrets from HashiCorp Vault 6 | license: BSD3 7 | github: channable/vaultenv 8 | 9 | dependencies: 10 | - aeson 11 | - base 12 | - async 13 | - bytestring 14 | - crypton-connection 15 | - containers 16 | - dotenv 17 | - directory 18 | - HsOpenSSL 19 | - http-conduit 20 | - http-client 21 | - http-client-openssl 22 | - megaparsec 23 | - network-uri 24 | - optparse-applicative 25 | - parser-combinators 26 | - retry 27 | - text 28 | - unordered-containers 29 | - unix 30 | - utf8-string 31 | - optparse-applicative 32 | 33 | ghc-options: -threaded -Wall -Werror 34 | 35 | library: 36 | source-dirs: src 37 | 38 | executables: 39 | vaultenv: 40 | main: Main.hs 41 | source-dirs: app 42 | dependencies: 43 | - vaultenv-real 44 | 45 | tests: 46 | vaultenv-test: 47 | main: Spec.hs 48 | source-dirs: test 49 | ghc-options: 50 | - -threaded 51 | - -rtsopts 52 | - -with-rtsopts=-N 53 | dependencies: 54 | - vaultenv-real 55 | - hspec 56 | - hspec-discover 57 | - hspec-expectations 58 | - directory 59 | - QuickCheck 60 | - quickcheck-instances 61 | -------------------------------------------------------------------------------- /package/build_package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # shellcheck disable=SC2091,SC2103 3 | 4 | # This script builds a .deb package from the static binary build by Nix. 5 | # 6 | # Usage: ./scripts/build_package.sh 7 | 8 | # Safe scripting options: 9 | # - `e` for fail on nonzero exit of any command 10 | # - `u` for fail on unset variables 11 | # - `f` to disable globbing 12 | # - `o pipefail` for error on failed pipes 13 | set -euf -o pipefail 14 | 15 | # Get the package version from Stack, eliminate the single quotes. 16 | # Exported, because it is used with envsubst to write the control file. 17 | VERSION=$(stack query locals vaultenv-real version | sed -e "s/^'//" -e "s/'$//") 18 | export VERSION 19 | 20 | # The name of the .deb file to create 21 | PKGNAME="vaultenv-${VERSION}" 22 | 23 | VAULTENV_NIX_PATH=$(nix-build --no-out-link ./nix/release.nix -A vaultenvStatic) 24 | 25 | cp --no-preserve=mode,ownership "${VAULTENV_NIX_PATH}/bin/vaultenv" "vaultenv-${VERSION}-linux-musl" 26 | 27 | 28 | # Build the Debian package 29 | # We need to use `fakeroot` here to have the package files be correctly owned 30 | # by `root` without being `root` ourselves. 31 | fakeroot -- bash <<-EOFAKEROOT 32 | # Re-enable safe scripting options since this is a new shell 33 | set -euf -o pipefail 34 | 35 | # Ensure created directories are mode 0755 (default is 0775) 36 | umask 022 37 | 38 | # Recreate the file system layout as it should be on the target machine. 39 | mkdir -p "$PKGNAME/DEBIAN" 40 | mkdir -p "$PKGNAME/usr/bin" 41 | mkdir -p "$PKGNAME/etc/secrets.d" 42 | 43 | # Write the package metadata file, substituting environment variables in the 44 | # template file. 45 | cat package/deb_control | envsubst > "$PKGNAME/DEBIAN/control" 46 | 47 | cp package/deb_postinst "$PKGNAME/DEBIAN/postinst" 48 | chmod 0755 "$PKGNAME/DEBIAN/postinst" 49 | 50 | # Copy the built binary from the Nix store to the target directory 51 | cp --no-preserve=mode,ownership "${VAULTENV_NIX_PATH}/bin/vaultenv" "$PKGNAME/usr/bin/" 52 | chown root:root "$PKGNAME/usr/bin/vaultenv" 53 | chmod 0755 "$PKGNAME/usr/bin/vaultenv" 54 | dpkg-deb --build "$PKGNAME" 55 | EOFAKEROOT 56 | 57 | # Clean up temporary files 58 | rm -fr "$PKGNAME" 59 | -------------------------------------------------------------------------------- /package/deb_control: -------------------------------------------------------------------------------- 1 | Package: vaultenv 2 | Version: ${VERSION} 3 | Priority: optional 4 | Architecture: amd64 5 | Maintainer: Laurens Duijvesteijn 6 | Description: Run processes with secrets from HashiCorp Vault 7 | -------------------------------------------------------------------------------- /package/deb_postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # If upgrading from 0.13.0 or earlier the permissions on /etc/secrets.d and 6 | # /usr/bin/vaultenv will be messed up. 7 | # Fixing this is idempotent so do this here: 8 | echo "-- Fixing potentially wrong file mode, ownership of /etc/secrets.d, /usr/bin/vaultenv" 9 | chown root:root /usr/bin/vaultenv /etc/secrets.d 10 | chmod 0755 /usr/bin/vaultenv /etc/secrets.d 11 | 12 | exit 0 13 | -------------------------------------------------------------------------------- /src/Config.hs: -------------------------------------------------------------------------------- 1 | {-| 2 | Module : Config 3 | Description : Read config from CLI options and environment 4 | 5 | This module uses optparse-applicative to parse CLI options. It augments the 6 | optparse parser with a mechanism to allow for overrides in environment 7 | variables. 8 | 9 | The main entry point is @parseOptions@. 10 | -} 11 | module Config 12 | ( AuthMethod (..) 13 | , Options(..) 14 | , MilliSeconds(..) 15 | , parseOptions 16 | , LogLevel(..) 17 | , DuplicateVariableBehavior(..) 18 | , readConfigFromEnvFiles 19 | , OptionsError 20 | , ValidScheme(..) 21 | , defaultOptions, isOptionsComplete, castOptions 22 | , splitAddress, validateCopyAddr, mergeOptions, getOptionsValue 23 | , Validated(), Completed() 24 | ) where 25 | 26 | import Control.Applicative ((<|>)) 27 | import Control.Monad(when) 28 | import Data.Char (isDigit) 29 | import Data.Either (lefts, rights, isLeft) 30 | import Data.List (intercalate) 31 | import Data.Maybe (fromJust, fromMaybe, isNothing, isJust) 32 | import Data.Text (Text) 33 | import Data.Version (showVersion) 34 | import Network.URI (URI(..), URIAuth(..), parseURI) 35 | import Options.Applicative (value, long, auto, option, metavar, help, flag, 36 | str, argument, many) 37 | -- Cabal generates the @Paths_vaultenv_real@ module, which contains a @version@ 38 | -- binding with the value out of the Cabal file. This feature is documented at: 39 | -- https://www.haskell.org/cabal/users-guide/developing-packages.html#accessing-data-files-from-package-code 40 | import Paths_vaultenv_real (version) 41 | import System.IO.Error (catchIOError) 42 | import System.Exit (die) 43 | import Text.Read (readMaybe) 44 | 45 | import qualified Configuration.Dotenv as DotEnv 46 | import qualified Data.Text as Text 47 | import qualified Options.Applicative as OptParse 48 | import qualified System.Directory as Dir 49 | 50 | -- | Type alias for enviornment variables, used for readability in this module. 51 | type EnvVar = (String, String) 52 | 53 | -- | Newtype wrapper for millisecond values. 54 | newtype MilliSeconds = MilliSeconds { unMilliSeconds :: Int } 55 | deriving (Eq, Show) 56 | 57 | data AuthMethod 58 | -- Note, these options are deliberately ordered from least specific to most 59 | -- specific, so we can use `max` as a sensible merge operation. 60 | = AuthNone 61 | -- ^ Do not include an x-vault-token header at all. 62 | | AuthGitHub Text 63 | -- ^ Use Vault's Github authentication [1] to obtain a Vault token, then 64 | -- use that as the x-vault-token. The value is the github personal access 65 | -- token to use. 66 | -- 67 | -- [1]: https://www.vaultproject.io/docs/auth/github) 68 | | AuthKubernetes Text 69 | -- ^ Use Vault's Kubernetes authentication [1] to obtain a Vault token, then 70 | -- use that as the x-vault-token. The value is the role to use. 71 | -- 72 | -- [1]: https://www.vaultproject.io/docs/auth/kubernetes) 73 | | AuthVaultToken Text 74 | -- ^ Provide the x-vault-token header, with this value. 75 | deriving (Eq, Ord, Show) 76 | 77 | -- | @Options@ contains all the configuration we support in vaultenv. It is 78 | -- used in our @Main@ module to specify behavior. 79 | data Options validated completed = Options 80 | { oVaultHost :: Maybe String 81 | , oVaultPort :: Maybe Int 82 | , oVaultAddr :: Maybe URI 83 | , oVaultAuthBackend :: Maybe String 84 | , oAuthMethod :: AuthMethod 85 | , oSecretFile :: Maybe FilePath 86 | , oCmd :: Maybe String 87 | , oArgs :: Maybe [String] 88 | , oConnectTls :: Maybe Bool 89 | , oValidateCerts :: Maybe Bool 90 | , oInheritEnv :: Maybe Bool 91 | , oInheritEnvBlacklist :: Maybe [String] 92 | , oRetryBaseDelay :: Maybe MilliSeconds 93 | , oRetryAttempts :: Maybe Int 94 | , oLogLevel :: Maybe LogLevel 95 | , oUsePath :: Maybe Bool 96 | , oMaxConcurrentRequests :: Maybe Int 97 | , oDuplicateVariablesBehavior :: Maybe DuplicateVariableBehavior 98 | } deriving (Eq) 99 | 100 | -- | Phantom type that indicates that an option is 101 | -- checked to be completed by ```isOptionsComplete``` and is validated by 102 | -- ```validateCopyAddr``` 103 | data Completed 104 | 105 | -- | Phantom type that indicates that an option is not yet checked to be complete 106 | data UnCompleted 107 | 108 | -- | Phantom type that indicates that an option is 109 | -- validated by ```validateCopyAddr``` 110 | data Validated 111 | 112 | -- | Phantom type that indicates that an option is 113 | -- not validated by ```isOptionsComplete``` 114 | data UnValidated 115 | 116 | -- | The default options that should be used when no value is specified 117 | -- If this default changes, please update the function isOptionsComplete if needed. 118 | defaultOptions :: Options Validated UnCompleted 119 | defaultOptions = Options 120 | { oVaultHost = Just "localhost" 121 | , oVaultPort = Just 8200 122 | , oVaultAddr = parseURI "https://localhost:8200" 123 | , oVaultAuthBackend = Nothing 124 | , oAuthMethod = AuthNone 125 | , oSecretFile = Nothing 126 | , oCmd = Nothing 127 | , oArgs = Just [] 128 | , oConnectTls = Just True 129 | , oValidateCerts = Just True 130 | , oInheritEnv = Just True 131 | , oInheritEnvBlacklist = Just [] 132 | , oRetryBaseDelay = Just (MilliSeconds 40) 133 | , oRetryAttempts = Just 9 134 | , oLogLevel = Just Error 135 | , oUsePath = Just True 136 | 137 | -- Number of concurrent requests to send to the vault server. 138 | -- The default has been chosen by doubling the limit in combination with a suitably large 139 | -- secrets file until the gain in performance was insignificant. 140 | , oMaxConcurrentRequests = Just 8 141 | , oDuplicateVariablesBehavior = Just DuplicateError 142 | } 143 | 144 | -- | Casts one options structure into another, use only when certain that 145 | -- the validated and completed options can be given to a options file. 146 | castOptions :: Options a b -> Options c d 147 | castOptions opts = Options 148 | { oVaultHost = oVaultHost opts 149 | , oVaultPort = oVaultPort opts 150 | , oVaultAddr = oVaultAddr opts 151 | , oVaultAuthBackend = oVaultAuthBackend opts 152 | , oAuthMethod = oAuthMethod opts 153 | , oSecretFile = oSecretFile opts 154 | , oCmd = oCmd opts 155 | , oArgs = oArgs opts 156 | , oConnectTls = oConnectTls opts 157 | , oValidateCerts = oValidateCerts opts 158 | , oInheritEnv = oInheritEnv opts 159 | , oInheritEnvBlacklist = oInheritEnvBlacklist opts 160 | , oRetryBaseDelay = oRetryBaseDelay opts 161 | , oRetryAttempts = oRetryAttempts opts 162 | , oLogLevel = oLogLevel opts 163 | , oUsePath = oUsePath opts 164 | , oMaxConcurrentRequests = oMaxConcurrentRequests opts 165 | , oDuplicateVariablesBehavior = oDuplicateVariablesBehavior opts 166 | } 167 | 168 | instance Show (Options valid complete) where 169 | show opts = intercalate "\n" 170 | [ "Host: " ++ showSpecifiedString (oVaultHost opts) 171 | , "Port: " ++ showSpecified (oVaultPort opts) 172 | , "Addr: " ++ showSpecified (oVaultAddr opts) 173 | , "Authentication method: " ++ case oAuthMethod opts of 174 | AuthVaultToken _ -> "Specified X-Vault-Token" 175 | AuthGitHub _ -> "Specified GitHub personal access token" 176 | AuthKubernetes role -> "Kubernetes service account, role: " <> (Text.unpack role) 177 | AuthNone -> "None" 178 | , "Auth backend: " ++ showSpecifiedString (oVaultAuthBackend opts) 179 | , "Secret file: " ++ showSpecifiedString (oSecretFile opts) 180 | , "Command: " ++ showSpecifiedString (oCmd opts) 181 | , "Arguments: " ++ showSpecified (oArgs opts) 182 | , "Use TLS: " ++ showSpecified (oConnectTls opts) 183 | , "Validate certs: " ++ showSpecified (oValidateCerts opts) 184 | , "Inherit env: " ++ showSpecified (oInheritEnv opts) 185 | , "Inherit env blacklist: " ++ showSpecified (oInheritEnvBlacklist opts) 186 | , "Base delay: " ++ case oRetryBaseDelay opts of 187 | Just (MilliSeconds ms) -> (show ms) ++ " ms" 188 | Nothing -> "Unspecified" 189 | , "Retry attempts: " ++ showSpecified (oRetryAttempts opts) 190 | , "Log-level: " ++ showSpecified (oLogLevel opts) 191 | , "Use PATH: " ++ showSpecified (oUsePath opts) 192 | , "Concurrent requests: " ++ showSpecified (oMaxConcurrentRequests opts) 193 | ] where 194 | showSpecified :: Show a => Maybe a -> String 195 | showSpecified (Just x) = show x 196 | showSpecified Nothing = "Unspecified" 197 | showSpecifiedString = fromMaybe "Unspecified" 198 | 199 | -- | Gets the option from a Maybe, ```name``` should only be used for debugging 200 | -- purposes, as ```isOptionsComplete``` should verify that every property that 201 | -- does not have a default value, is present in the options. 202 | getOptionsValue :: (Options Validated Completed -> Maybe c) -> Options Validated Completed -> c 203 | getOptionsValue f opts= 204 | fromMaybe (error "Complete options has an unknown key") (f opts) 205 | 206 | -- | The possible errors that can occur in the construction of an Options datatype 207 | data OptionsError 208 | = UnspecifiedValue String -- ^ A value is missing and no default is specified 209 | | URIParseError URI -- ^ The URI could not be parsed 210 | | UnknownScheme URI -- ^ The scheme of the address is invalid, e.g. ftp:// 211 | | NonNumericPort String -- ^ The port of the address is not an valid integer 212 | | HostPortSchemeAddrMismatch String (Maybe Bool) (Maybe String) (Maybe Int) URI 213 | | URIPathNotEmpty String -- ^ Disallow using non-empty path at the end of a URL 214 | -- ^ The source, useTls, host, port and scheme do not match the provided address 215 | deriving (Eq) 216 | 217 | instance Show OptionsError where 218 | show (UnspecifiedValue s) = "The option " ++ s ++ " is required but not specified. Run with --help for usage." 219 | show (URIParseError uri) = "The address" ++ show uri ++ " could not be parsed properly. Maybe the schema is missing?" 220 | show (UnknownScheme uri) = "The address " ++ show uri ++ " has no recognisable scheme, " 221 | ++ " expected http:// or https:// at the beginning of the address." 222 | show (NonNumericPort s) = "The port " ++ s ++ " is not a valid int value" 223 | show (HostPortSchemeAddrMismatch source useTLs host port addr) = 224 | concat [ 225 | "Confliciting configuration values in ", source, "\n", 226 | "Vault address: ", show addr, "\n", 227 | "Vault host: ", maybe "Unspecified" show host, "\n", 228 | "Vault port: ", maybe "Unspecified" show port, "\n", 229 | "Use TLS: ", maybe "Unspecified" show useTLs, "\n", 230 | "Hint: This can happen when you provide a `addr` which ", 231 | "conflicts with `port`, `host`, or `[no-]connect-tls` in either the environment variables or the CLI.\n" 232 | ] 233 | show (URIPathNotEmpty s) = "The address contains a non-empty path \"" ++ s ++ "\". This is not supported." 234 | 235 | -- | Validates for a set of options that any provided addr is valid and that either the 236 | -- scheme, host and port or that any given addr matches the other provided information. 237 | validateCopyAddr :: String -> Options UnValidated completed -> Either OptionsError (Options Validated completed) 238 | validateCopyAddr source opts = case oVaultAddr opts of 239 | Nothing -> Right $ castOptions opts 240 | Just addr -> do 241 | (scheme, addrHost, addrPort) <- splitAddress addr 242 | let mHost = oVaultHost opts 243 | mPort = oVaultPort opts 244 | mUseTLS = oConnectTls opts 245 | addrTLS = case scheme of 246 | HTTPS -> True 247 | HTTP -> False 248 | doesAddrDiffer = 249 | (isJust mHost && Just addrHost /= mHost) -- Is the Host set and the same 250 | || 251 | (isJust mPort && Just addrPort /= mPort) -- Is the Port set and the same 252 | || 253 | (isJust mUseTLS && Just addrTLS /= mUseTLS) -- Is the UseTLS set and the same 254 | when doesAddrDiffer 255 | (Left $ HostPortSchemeAddrMismatch 256 | source 257 | mUseTLS 258 | mHost 259 | mPort 260 | addr 261 | ) 262 | pure opts 263 | { oVaultHost = Just addrHost 264 | , oVaultPort = Just addrPort 265 | , oConnectTls = Just addrTLS 266 | } 267 | 268 | -- | This functions merges two options, where every specific option in the 269 | -- second options parameter is only used if for that option no value is 270 | -- specified in the first options parameter. 271 | mergeOptions :: Options Validated UnCompleted 272 | -> Options Validated UnCompleted 273 | -> Options Validated UnCompleted 274 | mergeOptions opts1 opts2 = let 275 | combine :: (Options Validated UnCompleted -> Maybe a) -> Maybe a 276 | combine f | isJust (f opts1) = f opts1 277 | | otherwise = f opts2 278 | in 279 | Options 280 | { oVaultHost = combine oVaultHost 281 | , oVaultPort = combine oVaultPort 282 | , oVaultAddr = combine oVaultAddr 283 | , oVaultAuthBackend = combine oVaultAuthBackend 284 | , oAuthMethod = max (oAuthMethod opts1) (oAuthMethod opts2) 285 | , oSecretFile = combine oSecretFile 286 | , oCmd = combine oCmd 287 | , oArgs = combine oArgs 288 | , oConnectTls = combine oConnectTls 289 | , oValidateCerts = combine oValidateCerts 290 | , oInheritEnv = combine oInheritEnv 291 | , oInheritEnvBlacklist = combine oInheritEnvBlacklist 292 | , oRetryBaseDelay = combine oRetryBaseDelay 293 | , oRetryAttempts = combine oRetryAttempts 294 | , oLogLevel = combine oLogLevel 295 | , oUsePath = combine oUsePath 296 | , oMaxConcurrentRequests = combine oMaxConcurrentRequests 297 | , oDuplicateVariablesBehavior = combine oDuplicateVariablesBehavior 298 | } 299 | 300 | 301 | 302 | -- | Verifies that either an ```Options``` is complete, according to the test, 303 | -- or it gives a list of errors. Only checks for values that are not included 304 | -- in the default. 305 | isOptionsComplete :: Options Validated UnCompleted 306 | -> Either [OptionsError] (Options Validated Completed) 307 | isOptionsComplete opts = 308 | let errors = [UnspecifiedValue "CMD" | isNothing (oCmd opts)] 309 | ++ [UnspecifiedValue "--secrets-file" | isNothing (oSecretFile opts)] 310 | in if not (null errors) 311 | then Left errors 312 | else Right (castOptions opts) 313 | 314 | -- | The host part of an address 315 | type Host = String 316 | -- | The scheme part of address 317 | data ValidScheme = HTTP | HTTPS deriving (Eq, Show) 318 | 319 | -- | This function splits the address into three different parts. The first 320 | -- part is a Scheme, which must be either http or https. 321 | -- The Host part of the return type is the part between the scheme and the 322 | -- last colon in the address. The port is the part after the colon. 323 | -- 324 | -- Examples: 325 | -- http://localhost:80 -> (HTTP, "localhost", "80") 326 | -- https://localhost:80 -> (HTTPS, "localhost", "80") 327 | -- ftp://localhost:80 -> Left $ UnknownScheme ftp://localhost:80 328 | -- localhost:80 -> Left $ URIParseError localhost:80 329 | -- http://localhost:80/foo/bar -> Right (HTTP, "localhost", "80") 330 | splitAddress :: URI -> Either OptionsError (ValidScheme, Host, Int) 331 | splitAddress addr = do 332 | -- Demand uriAuthority not Nothing, as that means something didn't go 333 | -- exactly right when parsing URIs 334 | -- See https://github.com/haskell/network-uri/issues/19 335 | uriAuth <- maybe (Left $ URIParseError addr) Right $ uriAuthority addr 336 | when (uriPath addr `notElem` ["", "/"]) $ 337 | Left $ URIPathNotEmpty $ uriPath addr 338 | 339 | scheme <- case uriScheme addr of 340 | "https:" -> Right HTTPS 341 | "http:" -> Right HTTP 342 | _ -> Left $ UnknownScheme addr 343 | 344 | let 345 | host = uriRegName uriAuth 346 | portSection = uriPort uriAuth 347 | 348 | port <- 349 | -- Default port from scheme when no port is given 350 | if null portSection 351 | then case scheme of 352 | HTTPS -> Right 443 353 | HTTP -> Right 80 354 | else 355 | -- The parseURI itself should reject non-numeric ports of its own, but 356 | -- since they represent the port as a string one cannot be sure. 357 | maybe (Left $ NonNumericPort portSection) Right $ 358 | -- Strip the leading colon before reading the value 359 | readMaybe $ tail portSection 360 | 361 | pure (scheme, host, port) 362 | 363 | -- | LogLevel to run vaultenv under. Under @Error@, which is the default, we 364 | -- will print error messages in error cases. Examples: Vault gives 404s, our 365 | -- token is invalid, we don't have permissions for a certain path, Vault is 366 | -- unavailable. 367 | -- 368 | -- Under @Info@, we print some additional information related to the config. 369 | data LogLevel 370 | = Info 371 | | Error 372 | deriving (Eq, Ord, Show) 373 | 374 | instance Read LogLevel where 375 | readsPrec _ "error" = [(Error, "")] 376 | readsPrec _ "info" = [(Info, "")] 377 | readsPrec _ _ = [] 378 | 379 | -- | The behavior that we want to see for duplicate variables. The options are 380 | -- to error on duplicate variables, keep the existing variables or overwrite the variables, 381 | -- replacing them with the secrets. 382 | data DuplicateVariableBehavior 383 | = DuplicateError 384 | | DuplicateKeep 385 | | DuplicateOverwrite 386 | deriving (Eq, Ord, Show) 387 | 388 | instance Read DuplicateVariableBehavior where 389 | readsPrec _ "error" = [(DuplicateError, "")] 390 | readsPrec _ "keep" = [(DuplicateKeep, "")] 391 | readsPrec _ "overwrite" = [(DuplicateOverwrite, "")] 392 | readsPrec _ _ = [] 393 | 394 | -- | Updates the options with the final vault authentication backend that will be used. 395 | -- If none has been supplied then fallback to the default depending on the 396 | -- authentication method. 397 | getAuthBackend :: Options Validated UnCompleted -> Options Validated UnCompleted 398 | getAuthBackend opts = 399 | opts { oVaultAuthBackend = newBackend } 400 | where 401 | newBackend :: Maybe String 402 | newBackend = case oVaultAuthBackend opts of 403 | Just b -> Just b 404 | Nothing -> defaultFromAuthMethod 405 | 406 | defaultFromAuthMethod = 407 | case oAuthMethod opts of 408 | AuthKubernetes _ -> Just "kubernetes" 409 | AuthGitHub _ -> Just "github" 410 | _ -> Nothing 411 | 412 | -- | Parse program options from the command line and the process environment. 413 | parseOptions :: [EnvVar] -> [[EnvVar]] -> IO (Options Validated Completed) 414 | parseOptions localEnvVars envFileSettings = 415 | let eLocalEnvFlagsOptions = validateCopyAddr "local environment variables" $ parseEnvOptions localEnvVars 416 | eEnvFileSettingsOptions = map (validateCopyAddr "environment file" . parseEnvOptions) envFileSettings 417 | in do 418 | eParseResult <- validateCopyAddr "cli options" <$> OptParse.execParser parserCliOptions 419 | let results = eEnvFileSettingsOptions ++ [eLocalEnvFlagsOptions, eParseResult] 420 | if any isLeft results then 421 | die (unlines (map (("[ERROR] "++). show) $ lefts results)) 422 | else 423 | let 424 | combined = foldl (flip mergeOptions) defaultOptions (rights results) 425 | updated = getAuthBackend combined 426 | completed = isOptionsComplete updated 427 | in either 428 | (\ls -> die (unlines (map (("[ERROR] "++). show) ls))) 429 | return 430 | completed 431 | 432 | -- | Parses options from a list of environment variables. If an 433 | -- environment variable corresponding to the flag is set to @"true"@ or 434 | -- @"false"@, we use that as the default on the corresponding CLI option. 435 | -- Other options are either String, Int, [String] or LogLevel. 436 | parseEnvOptions :: [EnvVar] -> Options UnValidated UnCompleted 437 | parseEnvOptions envVars 438 | = Options 439 | { oVaultHost = lookupEnvString "VAULT_HOST" 440 | , oVaultPort = lookupEnvInt "VAULT_PORT" 441 | , oVaultAddr = lookupVaultAddr 442 | , oVaultAuthBackend = lookupEnvString "VAULT_AUTH_BACKEND" 443 | , oAuthMethod = case lookupEnvString "VAULT_TOKEN" of 444 | Just token -> AuthVaultToken (Text.pack token) 445 | Nothing -> case lookupEnvString "VAULTENV_KUBERNETES_ROLE" of 446 | Just role -> AuthKubernetes (Text.pack role) 447 | Nothing -> case lookupEnvString "VAULTENV_GITHUB_TOKEN" of 448 | Just token -> AuthGitHub (Text.pack token) 449 | Nothing -> AuthNone 450 | , oSecretFile = lookupEnvString "VAULTENV_SECRETS_FILE" 451 | , oCmd = lookupEnvString "CMD" 452 | , oArgs = lookupStringList "ARGS..." 453 | , oConnectTls = lookupEnvFlag "VAULTENV_CONNECT_TLS" 454 | , oValidateCerts = lookupEnvFlag "VAULTENV_VALIDATE_CERTS" 455 | , oInheritEnv = lookupEnvFlag "VAULTENV_INHERIT_ENV" 456 | , oInheritEnvBlacklist = lookupCommaSeparatedList "VAULTENV_INHERIT_ENV_BLACKLIST" 457 | , oRetryBaseDelay = MilliSeconds <$> lookupEnvInt "VAULTENV_RETRY_BASE_DELAY" 458 | , oRetryAttempts = lookupEnvInt "VAULTENV_RETRY_ATTEMPTS" 459 | , oLogLevel = lookupEnvLogLevel "VAULTENV_LOG_LEVEL" 460 | , oUsePath = lookupEnvFlag "VAULTENV_USE_PATH" 461 | , oMaxConcurrentRequests = lookupEnvInt "VAULTENV_MAX_CONCURRENT_REQUESTS" 462 | , oDuplicateVariablesBehavior = lookupDuplicateBehaviorFlag "VAULTENV_DUPLICATE_VARIABLE_BEHAVIOR" 463 | } 464 | where 465 | -- | Throws an error for an invalid key 466 | err :: String -> a 467 | err key = errorWithoutStackTrace $ "[ERROR]: Invalid value for environment variable " ++ key 468 | -- | Lookup a string in the ```envVars``` list 469 | lookupEnvString key = lookup key envVars 470 | -- | Lookup an integer using ```lookupEnvString```, 471 | -- parses is to a Just, Nothing means not an Int 472 | lookupEnvInt :: String -> Maybe Int 473 | lookupEnvInt key 474 | | isNothing sVal = Nothing 475 | | not (null sVal) && all isDigit (fromJust sVal) = read <$> sVal 476 | | otherwise = err key 477 | where sVal = lookupEnvString key 478 | -- | Lookup a list of strings using ```lookupEnvString``` 479 | lookupStringList :: String -> Maybe [String] 480 | lookupStringList key = words <$> lookupEnvString key 481 | -- | Lookup a comma-separated list of strings using ```lookupEnvString``` 482 | lookupCommaSeparatedList :: String -> Maybe [String] 483 | lookupCommaSeparatedList key = splitOn ',' <$> lookupEnvString key 484 | -- | Lookup an log level using ```lookupEnvString``` 485 | lookupEnvLogLevel :: String -> Maybe LogLevel 486 | lookupEnvLogLevel key = 487 | case lookup key envVars of 488 | Just "info" -> Just Info 489 | Just "error" -> Just Error 490 | Nothing -> Nothing 491 | _ -> err key 492 | 493 | lookupDuplicateBehaviorFlag :: String -> Maybe DuplicateVariableBehavior 494 | lookupDuplicateBehaviorFlag key = 495 | case lookup key envVars of 496 | Just "keep" -> Just DuplicateKeep 497 | Just "error" -> Just DuplicateError 498 | Just "overwrite" -> Just DuplicateOverwrite 499 | Nothing -> Nothing 500 | _ -> err key 501 | 502 | -- | Look up and parse VAULT_ADDR 503 | lookupVaultAddr :: Maybe URI 504 | lookupVaultAddr = do 505 | addrString <- lookupEnvString "VAULT_ADDR" 506 | pure $ fromMaybe 507 | (errorWithoutStackTrace "[Error]: Invalid value for environment variable VAULT_ADDR") $ 508 | parseURI addrString 509 | -- | Lookup a boolean flag using ```lookupEnvString``` 510 | lookupEnvFlag :: String -> Maybe Bool 511 | lookupEnvFlag key = 512 | case lookup key envVars of 513 | Just "true" -> Just True 514 | Just "false" -> Just False 515 | Nothing -> Nothing 516 | _ -> err key 517 | 518 | -- | This function adds metadata to the @Options@ parser so it can be used with 519 | -- execParser. 520 | parserCliOptions :: OptParse.ParserInfo (Options UnValidated UnCompleted) 521 | parserCliOptions = 522 | OptParse.info 523 | (OptParse.helper <*> versionOption <*> optionsParser) 524 | (OptParse.fullDesc <> OptParse.header header) 525 | where 526 | versionOption = OptParse.infoOption (showVersion version) (long "version" <> help "Show version") 527 | header = "vaultenv " ++ showVersion version ++ " - run programs with secrets from HashiCorp Vault" 528 | 529 | 530 | -- | Parser for our CLI options. Seems intimidating, but is straightforward 531 | -- once you know about applicative parsing patterns. We construct a parser for 532 | -- @Options@ by combining parsers for parts of the record. 533 | -- 534 | -- Toy example to illustrate the pattern: 535 | -- 536 | -- @ 537 | -- OptionRecord <$> parser1 <*> parser2 <*> parser3 538 | -- @ 539 | -- 540 | -- Here, the parser for @OptionRecord@ is the combination of parsers of its 541 | -- internal fields. 542 | -- 543 | -- The parsers get constructed by using different combinators from the 544 | -- @Options.Applicative@ module. Here, we use @strOption@, @option@, @argument@ 545 | -- and flag. These take a @Mod@ value, which can specify how to parse an 546 | -- option. These @Mod@ values have monoid instances and are composed as such. 547 | -- 548 | -- So in our example above, we could have the following definition for 549 | -- @parser1@: 550 | -- 551 | -- @ 552 | -- parser1 = strOption $ long "my-option" <> value "default" 553 | -- @ 554 | -- 555 | -- And have the thing compile if the first member of @OptionRecord@ would have 556 | -- type @String@. 557 | -- 558 | -- Another thing that should be noted is the way we use the Alternate instance 559 | -- of @Parser@s. The @connectTls@, @validateCerts@ and @inheritEnv@ parsers all 560 | -- have a cousin prefixed with @no@. Combining these with the alternate 561 | -- instance like so 562 | -- 563 | -- @ 564 | -- noConnectTls <|> connectTls 565 | -- @ 566 | -- 567 | -- means that they are both mutually exclusive. 568 | -- 569 | -- Why do we have the options for the affirmative case? (e.g. why does 570 | -- @--connect-tls@ exist if we default to that behavior?) Because we want to be 571 | -- able to override all config that happens via environment variables on the 572 | -- CLI. So @VAULTENV_CONNECT_TLS=false vaultenv --connect-tls@ should connect 573 | -- to Vault over a secure connection. Without this option, this use case is not 574 | -- possible. 575 | -- 576 | -- The way we do this in the parser is by taking an @EnvFlags@ record which 577 | -- contains the settings that were provided via environment variables (and the 578 | -- defaults, if there wasn't anything configured). We use this record to set 579 | -- the default values of the different behaviour switches. So, if an 580 | -- environment variable is used to configure the TLS option, that value will 581 | -- always be used, except if it is overridden on the CLI. 582 | optionsParser :: OptParse.Parser (Options UnValidated UnCompleted) 583 | optionsParser = Options 584 | <$> host 585 | <*> port 586 | <*> addr 587 | <*> authBackend 588 | <*> auth 589 | <*> secretsFile 590 | <*> cmd 591 | <*> cmdArgs 592 | <*> (noConnectTls <|> connectTls) 593 | <*> (noValidateCerts <|> validateCerts) 594 | <*> (noInheritEnv <|> inheritEnv) 595 | <*> inheritEnvBlacklist 596 | <*> baseDelayMs 597 | <*> retryAttempts 598 | <*> logLevel 599 | <*> usePath 600 | <*> maxConcurrentRequests 601 | <*> duplicateVariableBehavior 602 | where 603 | maybeStr = Just <$> str 604 | maybeStrOption = option maybeStr 605 | host 606 | = maybeStrOption 607 | $ long "host" 608 | <> metavar "HOST" 609 | <> value Nothing 610 | <> help ("Vault host, either an IP address or DNS name. Defaults to localhost. " ++ 611 | "Also configurable via VAULT_HOST.") 612 | port 613 | = option (Just <$> auto) 614 | $ long "port" 615 | <> metavar "PORT" 616 | <> value Nothing 617 | <> help "Vault port. Defaults to 8200. Also configurable via VAULT_PORT." 618 | addr = option (parseURI <$> str) 619 | $ long "addr" 620 | <> metavar "ADDR" 621 | <> value Nothing 622 | <> help ("Vault address, the scheme, either http:// or https://, the ip-address or DNS name, " ++ 623 | "followed by the port, separated with a ':'." ++ 624 | " Cannot be combined with either VAULT_PORT or VAULT_HOST") 625 | authBackend 626 | = maybeStrOption 627 | $ long "auth-backend" 628 | <> metavar "AUTH_BACKEND" 629 | <> value Nothing 630 | <> help ("Name of Vault authentication backend. " 631 | ++ "Defaults to 'kubernetes' or 'github' depending on the type of the " 632 | ++ "chosen authentication method. Also configurable via VAULT_AUTH_BACKEND.") 633 | token 634 | = option (AuthVaultToken <$> str) 635 | $ long "token" 636 | <> metavar "TOKEN" 637 | <> help "Token to authenticate to Vault with. Also configurable via VAULT_TOKEN." 638 | 639 | githubToken 640 | = option (AuthGitHub <$> str) 641 | $ long "github-token" 642 | <> metavar "TOKEN" 643 | <> help ("Authenticate using a GitHub personal access token." ++ 644 | "Also configurable via VAULTENV_GITHUB_TOKEN.") 645 | 646 | kubernetesRole 647 | = option (AuthKubernetes <$> str) 648 | $ long "kubernetes-role" 649 | <> metavar "ROLE" 650 | <> help ("Authenticate using Kubernetes service account in /var/run/secrets/kubernetes.io, " ++ 651 | "with the given role. Also configurable via VAULTENV_KUBERNETES_ROLE.") 652 | 653 | auth = token <|> kubernetesRole <|> githubToken <|> pure AuthNone 654 | 655 | secretsFile = let 656 | original 657 | = maybeStrOption 658 | $ long "secrets-file" 659 | <> metavar "FILENAME" 660 | <> value Nothing 661 | <> help ("Config file specifying which secrets to request. Also configurable " ++ 662 | "via VAULTENV_SECRETS_FILE." ) 663 | alias 664 | = maybeStrOption 665 | $ long "secret-file" 666 | <> metavar "FILENAME" 667 | <> value Nothing 668 | <> help "alias for `--secrets-file`" 669 | in original <|> alias 670 | 671 | cmd 672 | = argument maybeStr 673 | $ metavar "CMD" 674 | <> help "command to run after fetching secrets" 675 | <> value Nothing 676 | cmdArgs 677 | = (\lst -> if null lst then Nothing else Just lst) <$> 678 | many ( argument str 679 | ( metavar "ARGS..." 680 | <> help "Arguments to pass to CMD, defaults to nothing") 681 | ) 682 | noConnectTls 683 | = flag Nothing (Just False) 684 | $ long "no-connect-tls" 685 | <> help ("Don't use TLS when connecting to Vault. Default: use TLS. Also " ++ 686 | "configurable via VAULTENV_CONNECT_TLS.") 687 | connectTls 688 | = flag Nothing (Just True) 689 | $ long "connect-tls" 690 | <> help ("Always connect to Vault via TLS. Default: use TLS. Can be used " ++ 691 | "to override VAULTENV_CONNECT_TLS.") 692 | noValidateCerts 693 | = flag Nothing (Just True) 694 | $ long "no-validate-certs" 695 | <> help ("Don't validate TLS certificates when connecting to Vault. Default: " ++ 696 | "validate certs. Also configurable via VAULTENV_VALIDATE_CERTS.") 697 | validateCerts 698 | = flag Nothing (Just True) 699 | $ long "validate-certs" 700 | <> help ("Always validate TLS certificates when connecting to Vault. Default: " ++ 701 | "validate certs. Can be used to override VAULTENV_CONNECT_TLS.") 702 | noInheritEnv 703 | = flag Nothing (Just False) 704 | $ long "no-inherit-env" 705 | <> help ("Don't merge the parent environment with the secrets file. Default: " ++ 706 | "merge environments. Also configurable via VAULTENV_INHERIT_ENV.") 707 | inheritEnv 708 | = flag Nothing (Just True) 709 | $ long "inherit-env" 710 | <> help ("Always merge the parent environment with the secrets file. Default: " ++ 711 | "merge environments. Can be used to override VAULTENV_INHERIT_ENV.") 712 | 713 | maybeStrList = Just . splitOn ',' <$> str 714 | maybeStrListOption = option maybeStrList 715 | 716 | inheritEnvBlacklist 717 | = maybeStrListOption 718 | $ long "inherit-env-blacklist" 719 | <> metavar "COMMA_SEPARATED_NAMES" 720 | <> value Nothing 721 | <> help ("Comma-separated list of environment variable names to remove from " ++ 722 | "the environment before executing CMD. Also configurable via " ++ 723 | "VAULTENV_INHERIT_ENV_BLACKLIST. Has no effect if no-inherit-env is set!") 724 | 725 | baseDelayMs 726 | = fmap MilliSeconds <$> option (Just <$> auto) 727 | ( long "retry-base-delay-milliseconds" 728 | <> metavar "MILLISECONDS" 729 | <> value Nothing 730 | <> help ("Base delay for vault connection retrying. Defaults to 40ms. " ++ 731 | "Also configurable via VAULTENV_RETRY_BASE_DELAY_MS.") 732 | ) 733 | retryAttempts 734 | = option (Just <$> auto) 735 | $ long "retry-attempts" 736 | <> metavar "NUM" 737 | <> value Nothing 738 | <> help ("Maximum number of vault connection retries. Defaults to 9. " ++ 739 | "Also configurable through VAULTENV_RETRY_ATTEMPTS.") 740 | logLevel 741 | = option (Just <$> auto) 742 | $ long "log-level" 743 | <> value Nothing 744 | <> metavar "error | info" 745 | 746 | <> help ("Log-level to run vaultenv under. Options: 'error' or 'info'. " ++ 747 | "Defaults to 'error'. Also configurable via VAULTENV_LOG_LEVEL") 748 | usePath 749 | = flag Nothing (Just True) 750 | $ long "use-path" 751 | <> help ("Use PATH for finding the executable that vaultenv should call. Default: " ++ 752 | "don't search PATH. Also configurable via VAULTENV_USE_PATH.") 753 | maxConcurrentRequests 754 | = option (Just <$> auto) 755 | $ long "max-concurrent-requests" 756 | <> metavar "NUM" 757 | <> value Nothing 758 | <> help ("Maximum number of concurrent requests to vault. Defaults to 8. " ++ 759 | "Pass 0 to disable the limit. " ++ 760 | "Also configurable through VAULTENV_MAX_CONCURRENT_REQUESTS.") 761 | 762 | duplicateVariableBehavior 763 | = option (Just <$> auto) 764 | $ long "duplicate-variable-behavior" 765 | <> value Nothing 766 | <> metavar "error | keep | overwrite" 767 | <> help ("Changes the behavior of duplicate variables. 'error` produces an error" ++ 768 | " for duplicate variables. 'keep' keeps the value already in the environment" ++ 769 | " , ignoring the secret. 'overwrite' overwrite the environment variable in favor" ++ 770 | " of the secret. Default to 'error'." ++ 771 | " Also configurable through VAULTENV_DUPLICATE_VARIABLE_BEHAVIOR.") 772 | 773 | -- | Split a list of elements on the given separator, returning sublists without this 774 | -- separator item. 775 | -- 776 | -- Example: 777 | -- 778 | -- >>> splitOn ',' "somestring,anotherstring" 779 | -- ["somestring", "anotherstring"] 780 | splitOn :: (Eq a) => a -> [a] -> [[a]] 781 | splitOn sep x 782 | = case span (/= sep) x of 783 | (y, []) -> [y] 784 | (y, ys) -> y : splitOn sep (tail ys) 785 | 786 | -- | Search for environment files in default locations and load them in order. 787 | -- 788 | -- This function tries to read the following files in order to obtain 789 | -- environment configuration. This is implicit behavior and allows the user to 790 | -- configure vaultenv without setting up environment variables or passing CLI 791 | -- flags. This is nicer for interactive usage. 792 | readConfigFromEnvFiles :: IO [[(String, String)]] 793 | readConfigFromEnvFiles = do 794 | xdgDir <- (Just <$> Dir.getXdgDirectory Dir.XdgConfig "vaultenv") 795 | `catchIOError` const (pure Nothing) 796 | cwd <- Dir.getCurrentDirectory 797 | let 798 | machineConfigFile = "/etc/vaultenv.conf" 799 | 800 | userConfigFile :: Maybe FilePath 801 | userConfigFile = fmap (++ "/vaultenv.conf") xdgDir 802 | cwdConfigFile = cwd ++ "/.env" 803 | 804 | -- @doesFileExist@ doesn't throw exceptions, it catches @IOError@s and 805 | -- returns @False@ if those are encountered. 806 | machineConfigExists <- Dir.doesFileExist machineConfigFile 807 | machineConfig <- if machineConfigExists 808 | then DotEnv.parseFile machineConfigFile 809 | else pure [] 810 | 811 | userConfigExists <- case userConfigFile of 812 | Nothing -> pure False 813 | Just fp -> Dir.doesFileExist fp 814 | userConfig <- if userConfigExists 815 | then DotEnv.parseFile (fromJust userConfigFile) -- safe because of loadUserConfig 816 | else pure [] 817 | 818 | cwdConfigExists <- Dir.doesFileExist cwdConfigFile 819 | cwdConfig <- if cwdConfigExists 820 | then DotEnv.parseFile cwdConfigFile 821 | else pure [] 822 | 823 | -- Deduplicate, user config takes precedence over machine config 824 | let config = [machineConfig, userConfig, cwdConfig] 825 | pure config 826 | -------------------------------------------------------------------------------- /src/KeyMap.hs: -------------------------------------------------------------------------------- 1 | -- | KeyMap, extending Aeson.KeyMap to support replacing HashMap. 2 | module KeyMap 3 | ( KeyMap 4 | , lookupDefault 5 | , lookup 6 | , mapMaybe 7 | , fromList 8 | , toList 9 | , singleton 10 | 11 | , Key 12 | , fromText 13 | , toText 14 | , fromString 15 | , toString 16 | ) where 17 | 18 | import Data.Aeson.Key (Key, fromText, toText, fromString, toString) 19 | import Data.Aeson.KeyMap (KeyMap, mapMaybe, fromList, toList, singleton) 20 | import Data.Maybe (fromMaybe) 21 | 22 | import qualified Data.Aeson.KeyMap as KM 23 | 24 | -- | lookupDefault for KeyMap based on lookupDefault from HashMap. 25 | lookupDefault :: v -> Key -> KeyMap v -> v 26 | lookupDefault d k km = fromMaybe d $ KM.lookup k km 27 | -------------------------------------------------------------------------------- /src/Response.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | -- | Types to hold and json-decode responses from Vault. 4 | module Response 5 | ( ClientToken (..) 6 | ) where 7 | 8 | import Data.Aeson (FromJSON, (.:)) 9 | import Data.Text (Text) 10 | 11 | import qualified Data.Aeson as Aeson 12 | 13 | -- | The "client_token" field from an /auth/kubernetes/login or 14 | -- /auth/github/login response. 15 | newtype ClientToken = ClientToken Text deriving (Eq, Show) 16 | 17 | instance FromJSON ClientToken where 18 | parseJSON = Aeson.withObject "AuthResponse" $ \obj -> do 19 | auth <- obj .: "auth" 20 | clientToken <- auth .: "client_token" 21 | pure $ ClientToken clientToken 22 | -------------------------------------------------------------------------------- /src/SecretsFile.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | {-# LANGUAGE ScopedTypeVariables #-} 3 | {-# LANGUAGE FlexibleContexts #-} 4 | 5 | {-| 6 | Module : SecretsFile 7 | Description : Parser for the vaultenv secret file format 8 | 9 | Contains a Megaparsec parser for the config format used by vaultenv to specify 10 | secrets. We support two versions of the format. This parser accepts both 11 | versions of the format in use. Version one of the format is a list of secrets, 12 | which vaultenv will fetch from a generic backend mounted at @secret/@. 13 | 14 | Version 2 allows users to specify mountpoints. Both versions of the format 15 | allow users to specify which keys are fetched from Vault, as well as specify 16 | the name of the environment variable under which secrets will be made availabe. 17 | 18 | The entrypoints here are the @readSecretsFile@ and @readSecretList@ functions. 19 | 20 | If you are user, please see the README for more information. 21 | -} 22 | module SecretsFile where 23 | 24 | import Control.Applicative.Combinators (some, many, option, optional) 25 | import Control.Exception (try, displayException) 26 | import Data.Char (toUpper, isSpace, isControl) 27 | import Data.Functor (void) 28 | import Data.List (intercalate) 29 | import Data.Void (Void) 30 | import qualified Text.Megaparsec as MP 31 | import qualified Text.Megaparsec.Char as MPC 32 | import qualified Text.Megaparsec.Char.Lexer as MPL 33 | 34 | data Secret = Secret 35 | { sMount :: String 36 | , sPath :: String 37 | , sKey :: String 38 | , sVarName :: String 39 | } deriving (Eq, Show) 40 | 41 | data SFVersion 42 | = V1 43 | | V2 44 | deriving (Show) 45 | 46 | type Parser = MP.Parsec Void String 47 | 48 | -- | Error modes of this module. 49 | -- 50 | -- We either get IO errors because we cannot open the secrets file, or we 51 | -- cannot parse it. 52 | data SFError = IOErr IOError | ParseErr (MP.ParseErrorBundle String Void) 53 | 54 | instance Show SFError where 55 | show sfErr = case sfErr of 56 | IOErr ioErr -> displayException ioErr 57 | ParseErr pe -> MP.errorBundlePretty pe 58 | 59 | -- | Read a list of secrets from a file 60 | readSecretsFile :: FilePath -> IO (Either SFError [Secret]) 61 | readSecretsFile fp = do 62 | contentsOrErr <- safeReadFile fp 63 | case contentsOrErr of 64 | Right c -> do 65 | let parseResult = parseSecretsFile fp c 66 | case parseResult of 67 | Right res -> pure $ Right res 68 | Left err -> pure $ Left (ParseErr err) 69 | Left err -> pure $ Left (IOErr err) 70 | 71 | -- | Read a file, catching all IOError exceptions. 72 | safeReadFile :: FilePath -> IO (Either IOError String) 73 | safeReadFile fp = (try . readFile) fp 74 | 75 | -- | Parse a String as a SecretsFile. 76 | parseSecretsFile :: FilePath -> String -> Either (MP.ParseErrorBundle String Void) [Secret] 77 | parseSecretsFile = MP.parse secretsFileP 78 | 79 | -- | SpaceConsumer parser, which is responsible for stripping all whitespace. 80 | -- 81 | -- Sometimes, we require explicit newlines, therefore, we don't handle those 82 | -- here. @isSpace@ works on any unicode whitespace character. Megaparsec comes 83 | -- with some helpers that would make this better, but here we need to roll our 84 | -- own whitespace parser, because we want to preserve newlines. 85 | whitespace :: Parser () 86 | whitespace = MPL.space whitespaceChars lineComment blockComment 87 | where 88 | whitespaceChars = void $ MP.takeWhile1P (Just "whitespace") (\c -> isSpace c && c /= '\n') 89 | lineComment = MP.empty 90 | blockComment = MP.empty 91 | 92 | -- | Parses one or multiple newlines separated by whitespace. 93 | newlines :: Parser () 94 | newlines = void $ some $ lexeme $ MPC.char '\n' 95 | 96 | -- | Helper which consumes all whitespace after a parser 97 | lexeme :: Parser a -> Parser a 98 | lexeme = MPL.lexeme whitespace 99 | 100 | -- | Helper which looks for a string and consumes trailing whitespace. 101 | symbol :: String -> Parser String 102 | symbol = MPL.symbol whitespace 103 | 104 | -- | Top level parser of the secrets file 105 | -- 106 | -- Parses the magic version number and dispatches to the Mount block based 107 | -- parser or the list based parser based on that. 108 | secretsFileP :: Parser [Secret] 109 | secretsFileP = do 110 | _ <- optional newlines 111 | _ <- whitespace 112 | version <- versionP 113 | case version of 114 | V1 -> many (secretP version "secret") 115 | V2 -> concat <$> many secretBlockP 116 | 117 | -- | Parse the file version 118 | -- 119 | -- We need @MP.try@ because we need to backtrack after reading VERSION. (As 120 | -- some secrets could very well start with that path. 121 | versionP :: Parser SFVersion 122 | versionP = option V1 $ MP.try $ do 123 | _ <- symbol "VERSION" 124 | _ <- symbol "2" 125 | _ <- newlines 126 | pure V2 127 | 128 | -- | Parse a secret block 129 | -- 130 | -- Exclusive to V2 of the format. A secret block consists of a line describing 131 | -- the mount location followed by secret specifications. 132 | secretBlockP :: Parser [Secret] 133 | secretBlockP = do 134 | _ <- symbol "MOUNT" 135 | mountPath <- lexeme pathComponentP 136 | _ <- newlines 137 | many (MP.try (lexeme (secretP V2 mountPath))) 138 | 139 | -- | Parses legal Vault path components. 140 | -- 141 | -- A Vault path allows a surprising amount of characters. Spaces, quotes and 142 | -- whatnot are all allowed. We don't want to complicate the parser and 143 | -- the format by specifying escaping for all kinds of things, so we impose the 144 | -- following restrictions: 145 | -- 146 | -- - We don't support mounts, paths and keys with whitespace in them. 147 | -- - We don't support control characters (vault doesn't either) 148 | -- - All other characters except @=@ and @#@ are allowed. Supporting paths 149 | -- with these characters in them would lead to ambiguities when parsing 150 | -- paths such as: 151 | -- 152 | -- FOO=foo=bar/baz#quix 153 | -- 154 | -- and 155 | -- 156 | -- foo#bar/baz#quix 157 | -- 158 | -- If this is undesired, have a compelling usecase, and a good proposal for 159 | -- supporting this, please open a ticket. 160 | pathComponentP :: Parser String 161 | pathComponentP = MP.takeWhile1P (Just "path component") isAllowed 162 | where isAllowed c = not (isSpace c) && c /= '#' && c /= '=' && not (isControl c) 163 | 164 | -- | Parse a secret specification line 165 | -- 166 | -- The version of the fileformat we're parsing determines the way we report 167 | -- variable information. For V2, the mount point is part of the variable name, 168 | -- to allow for disambiguation. For V1, this is not needed. 169 | secretP :: SFVersion -> String -> Parser Secret 170 | secretP version mount = do 171 | secret <- lexeme $ do 172 | varName <- optional $ MP.try secretVarP 173 | path <- pathComponentP 174 | _ <- symbol "#" 175 | key <- pathComponentP 176 | 177 | pure Secret { sMount = mount 178 | , sPath = path 179 | , sKey = key 180 | , sVarName = maybe (getVarName version mount path key) id varName 181 | } 182 | _ <- newlines 183 | pure secret 184 | 185 | -- | Parses a secret variable. 186 | -- 187 | -- We're restrictrive in the characters we allow in environment variables. We 188 | -- don't allow special characters or whitespace. Environment variables have to 189 | -- start with a letter or underscore which can be followed by letters 190 | -- underscores and digits. This is similar to what Zsh and Bash allow in their 191 | -- `export` statements. Even though the Unix process environment is technically 192 | -- just a string and you can put all kinds of things in there, most programs 193 | -- and standard libraries don't seem to support this. 194 | -- 195 | -- Please open a ticket if you require looser restrictions. 196 | secretVarP :: Parser String 197 | secretVarP = do 198 | -- Environment variables have to start with a letter or underscore and can be 199 | -- followed by letters, underscores and digits. 200 | varStart <- MP.oneOf asciiLettersUnderscore 201 | varRest <- MP.many $ MP.oneOf (asciiLettersUnderscore ++ digits) 202 | _ <- symbol "=" 203 | pure (varStart:varRest) 204 | 205 | -- | Helper list for ASCII chars plus the underscore 206 | asciiLettersUnderscore :: [MP.Token String] 207 | asciiLettersUnderscore = ['a'..'z'] ++ ['A'..'Z'] ++ ['_'] 208 | 209 | -- | Helper list for ASCII digits 210 | digits :: [MP.Token String] 211 | digits = ['0'..'9'] 212 | 213 | -- | Convert a secret name into the name of the environment variable that it 214 | -- will be available under. 215 | getVarName :: SFVersion -> String -> String -> String -> String 216 | getVarName version mount path key = fmap format $ intercalate "_" components 217 | where underscore '/' = '_' 218 | underscore '-' = '_' 219 | underscore c = c 220 | format = toUpper . underscore 221 | components = case version of 222 | V1 -> [path, key] 223 | V2 -> [mount, path, key] 224 | -------------------------------------------------------------------------------- /stack.yaml: -------------------------------------------------------------------------------- 1 | # Also take care to update the compiler in default.nix to the 2 | # compiler used in this Stackage snapshot. 3 | resolver: ghc-9.6.4 4 | 5 | packages: 6 | - "." 7 | 8 | # This makes stack pick up our nix environment for building by default. 9 | nix: 10 | enable: true 11 | shell-file: nix/stack-shell.nix 12 | path: ["nixpkgs=./nix/nixpkgs-pinned.nix"] 13 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | # generated by the tests 2 | integration/__pycache__ 3 | .cache 4 | container.tar.gz 5 | -------------------------------------------------------------------------------- /test/ConfigSpec.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ScopedTypeVariables #-} 2 | module ConfigSpec where 3 | 4 | import Data.Either(isRight, isLeft) 5 | import Data.Maybe(fromJust) 6 | import Network.URI (URI(..), parseURI, isUnescapedInURIComponent) 7 | import Test.Hspec 8 | import Test.QuickCheck 9 | 10 | import Config 11 | 12 | getHost 13 | :: Either OptionsError (ValidScheme, String, Int) 14 | -> Either OptionsError String 15 | getHost = fmap $ \(_, host, _) -> host 16 | 17 | getScheme 18 | :: Either OptionsError (ValidScheme, String, Int) 19 | -> Either OptionsError ValidScheme 20 | getScheme = fmap $ \(scheme, _, _) -> scheme 21 | 22 | getPort 23 | :: Either OptionsError (ValidScheme, String, Int) 24 | -> Either OptionsError Int 25 | getPort = fmap $ \(_, _, port) -> port 26 | 27 | -- Generates strings that are valid as host parts of a URI 28 | genHost :: Gen String 29 | genHost = arbitrary `suchThat` all isUnescapedInURIComponent 30 | 31 | -- Valid hosts and ports 32 | genHostPort :: Gen (String, Int) 33 | genHostPort = (,) <$> genHost <*> (arbitrary `suchThat` (>=0)) 34 | 35 | spec :: SpecWith () 36 | spec = 37 | describe "Split addr" $ do 38 | it "should accept http schemes " $ 39 | getScheme (splitAddress $ fromJust $ parseURI "http://localhost:80") 40 | `shouldBe` Right HTTP 41 | it "should accept https schemes " $ 42 | getScheme (splitAddress $ fromJust $ parseURI "https://localhost:80") 43 | `shouldBe` Right HTTPS 44 | it "should reject any other scheme" $ 45 | property $ \scheme -> 46 | let uri = parseURI $ scheme ++ "://localhost:80" 47 | in scheme `notElem` ["http", "https"] 48 | ==> maybe True (isLeft . splitAddress) uri 49 | it "should split any valid addr properly" $ 50 | forAll genHostPort (\(host, port) -> 51 | let 52 | httpAddr :: Maybe URI 53 | httpAddr = parseURI $ "http://" ++ host ++ ":" ++ show port 54 | 55 | httpsAddr :: Maybe URI 56 | httpsAddr = parseURI $ "https://" ++ host ++ ":" ++ show port 57 | in 58 | (splitAddress <$> httpAddr `shouldBe` Just (Right (HTTP, host, port))) 59 | .&&. (splitAddress <$> httpsAddr `shouldBe` Just (Right (HTTPS, host, port))) 60 | ) 61 | it "should parse addr without explicit port" $ 62 | forAll genHost (\host -> 63 | let 64 | httpAddr :: Maybe URI 65 | httpAddr = parseURI $ "http://" ++ host 66 | 67 | httpsAddr :: Maybe URI 68 | httpsAddr = parseURI $ "https://" ++ host 69 | in (splitAddress <$> httpAddr `shouldBe` Just (Right (HTTP, host, 80))) 70 | .&&. (splitAddress <$> httpsAddr `shouldBe` Just (Right (HTTPS, host, 443))) 71 | ) 72 | it "should accept the default configuration" $ 73 | isRight (validateCopyAddr "" $ castOptions defaultOptions) 74 | it "should reject mismatched port" $ 75 | let 76 | options = defaultOptions{ 77 | oVaultPort = Just 1234 78 | } 79 | in isLeft (validateCopyAddr "" $ castOptions options) 80 | it "should reject mismatched host" $ 81 | let 82 | options = defaultOptions{ 83 | oVaultHost = Just "invalid_host" 84 | } 85 | in isLeft (validateCopyAddr "" $ castOptions options) 86 | it "should reject mismatched TLS" $ 87 | let 88 | options = defaultOptions{ 89 | oConnectTls = Just False 90 | } 91 | in isLeft (validateCopyAddr "" $ castOptions options) 92 | it "should reject invalid schemes" $ 93 | let 94 | options = defaultOptions{ 95 | oVaultAddr = parseURI "ftp://localhost:8200" 96 | } 97 | in isLeft (validateCopyAddr "" $ castOptions options) 98 | it "should accept URLs with trailing slash " $ 99 | let 100 | options = defaultOptions{ 101 | oVaultAddr = parseURI "https://localhost:8200/" 102 | } 103 | in isRight (validateCopyAddr "" $ castOptions options) 104 | it "should reject URLs with non-empty paths" $ 105 | let 106 | options = defaultOptions{ 107 | oVaultAddr = parseURI "https://localhost:8200/foo" 108 | } 109 | in isLeft (validateCopyAddr "" $ castOptions options) 110 | -------------------------------------------------------------------------------- /test/KeyMapSpec.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ScopedTypeVariables #-} 2 | 3 | module KeyMapSpec where 4 | 5 | import Data.Text (Text) 6 | import Test.Hspec 7 | import Test.QuickCheck 8 | import Test.QuickCheck.Instances () 9 | 10 | import qualified Data.HashMap.Internal.Strict as HM 11 | 12 | import qualified KeyMap as KM 13 | 14 | mapFirst :: (a->b) -> [(a,c)] -> [(b,c)] 15 | mapFirst _ [] = [] 16 | mapFirst f ((x,y) : rest) = (f x, y) : mapFirst f rest 17 | 18 | spec :: SpecWith () 19 | spec = 20 | describe "KeyMap" $ do 21 | it "lookupDefault matches the HashMap implementation" $ 22 | property $ \((fallback :: Text), key, mapping) -> 23 | let keyMapValue = KM.lookupDefault fallback (KM.fromText key) (KM.fromList $ mapFirst KM.fromText mapping) 24 | hashMapValue = HM.lookupDefault fallback key (HM.fromList mapping) 25 | in keyMapValue `shouldBe` hashMapValue 26 | -------------------------------------------------------------------------------- /test/ResponseSpec.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE LambdaCase #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | 4 | module ResponseSpec where 5 | 6 | import Test.Hspec (SpecWith, describe, it, shouldBe) 7 | 8 | import qualified Data.Aeson as Aeson 9 | import qualified Data.ByteString.Lazy as LazyByteString 10 | 11 | import Response (ClientToken (..)) 12 | 13 | -- Example response from /v1/auth/kubernetes/login 14 | -- Secrets are from a test setup, these are not real secrets. 15 | loginResponse :: LazyByteString.ByteString 16 | loginResponse = 17 | -- Replace single quotes (U+0027) with double quotes (U+0022), so we can write 18 | -- the json string here as a string literal without too many escapes. 19 | LazyByteString.map (\case 20 | 0x27 -> 0x22 21 | x -> x 22 | ) "{'request_id':'43cfeccb-ebd6-aa9f-bcd0-e1c184b625ae','lease_id':'','renewable':false,'lease_duration':0,'data':null,'wrap_info':null,'warnings':null,'auth':{'client_token':'s.J1hknwSvoU5iDB7QfAJ2G15V','accessor':'u3D5ESJ0OzenuqwUdSX87VTZ','policies':['default','testapp-kv-ro'],'token_policies':['default','testapp-kv-ro'],'metadata':{'role':'vaultenv-test-role','service_account_name':'test-vault-auth','service_account_namespace':'default','service_account_secret_name':'','service_account_uid':'0036f224-8b4e-4bd2-b298-560b7b39c011'},'lease_duration':86400,'renewable':true,'entity_id':'349d417f-d5d9-2ae5-a042-b801848334a7','token_type':'service','orphan':true}}" 23 | 24 | spec :: SpecWith () 25 | spec = do 26 | describe "Response.ClientToken" $ do 27 | it "can be parsed from json" $ do 28 | Aeson.eitherDecode loginResponse `shouldBe` Right (ClientToken "s.J1hknwSvoU5iDB7QfAJ2G15V") 29 | -------------------------------------------------------------------------------- /test/SecretFileSpec.hs: -------------------------------------------------------------------------------- 1 | module SecretFileSpec where 2 | 3 | import Test.Hspec 4 | 5 | import Data.Either (isRight, isLeft) 6 | 7 | import qualified SecretsFile 8 | import qualified System.Directory as Dir 9 | 10 | spec :: SpecWith () 11 | spec = do 12 | describe "SecretFile.readSecretList" $ do 13 | it "parses all golden tests succesfully" $ do 14 | goldenTestContents <- Dir.listDirectory "test/golden" 15 | let goldenTestFiles = map ("test/golden/" <>) goldenTestContents 16 | parseResults <- mapM SecretsFile.readSecretsFile goldenTestFiles 17 | parseResults `shouldSatisfy` (all isRight) 18 | 19 | it "rejects all invalid examples succesfully" $ do 20 | invalidTestContents <- Dir.listDirectory "test/invalid" 21 | let invalidTestFiles = map ("test/invalid/" <>) invalidTestContents 22 | parseResults <- mapM SecretsFile.readSecretsFile invalidTestFiles 23 | parseResults `shouldSatisfy` (all isLeft) 24 | -------------------------------------------------------------------------------- /test/Spec.hs: -------------------------------------------------------------------------------- 1 | {-# OPTIONS_GHC -F -pgmF hspec-discover #-} 2 | -------------------------------------------------------------------------------- /test/default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import ../nix/nixpkgs-pinned.nix 2 | # Allow vault to be used only as a part of this testing environment shell. 3 | # We are not allowing the use of vault as a part of our final package 4 | # because vault-1.16.1 is lisenced under BSL-1.1 5 | { config.allowUnfreePredicate = pkg: 6 | (pkgs.lib.getName pkg) == "vault" && 7 | (pkgs.lib.getVersion pkg) == "1.17.2"; 8 | } 9 | }: 10 | with pkgs; buildEnv { 11 | name = "vaultenv-testenv"; 12 | paths = [ 13 | cachix 14 | glibcLocales # So you can export LOCALE_ARCHIVE=$(nix path-info)/lib/locale/locale-archive. 15 | perl # For "prove" 16 | python3 17 | stack 18 | vault 19 | minikube # Setting up a kubernetes cluster for the integration tests 20 | kubectl 21 | ]; 22 | } 23 | -------------------------------------------------------------------------------- /test/golden/empty-block.secrets: -------------------------------------------------------------------------------- 1 | VERSION 2 2 | MOUNT empty 3 | MOUNT nonempty 4 | FOO_BAR=foo#bar 5 | MOUNT empty2 6 | -------------------------------------------------------------------------------- /test/golden/empty-v1.secrets: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/channable/vaultenv/9dba383b909721b984fc76a1276ed097874c24bd/test/golden/empty-v1.secrets -------------------------------------------------------------------------------- /test/golden/empty-v2.secrets: -------------------------------------------------------------------------------- 1 | VERSION 2 2 | -------------------------------------------------------------------------------- /test/golden/special.secrets: -------------------------------------------------------------------------------- 1 | foob@ar#baz 2 | foՔob@ar#baz 3 | -------------------------------------------------------------------------------- /test/golden/v1-version.secrets: -------------------------------------------------------------------------------- 1 | VERSIONfoobar#foo 2 | -------------------------------------------------------------------------------- /test/golden/v1.secrets: -------------------------------------------------------------------------------- 1 | foo#bar 2 | foo/bar#baz 3 | FOO=bar#baz 4 | BAR=foo/baz#quix 5 | single_underscore=foo/single#underscore 6 | double__underscore=foo/double#underscore 7 | _leading_underscore=foo/double#underscore 8 | -------------------------------------------------------------------------------- /test/golden/v2.secrets: -------------------------------------------------------------------------------- 1 | VERSION 2 2 | 3 | MOUNT secret 4 | foo#bar 5 | BAR=foo/baz#bar 6 | 7 | MOUNT otherthing 8 | foo#bar 9 | BAR=foo/baz#bar 10 | -------------------------------------------------------------------------------- /test/golden/variables.secrets: -------------------------------------------------------------------------------- 1 | BAR_BAZ=foo/bar#baz 2 | -------------------------------------------------------------------------------- /test/golden/whitespace.secrets: -------------------------------------------------------------------------------- 1 | 2 | VERSION 2 3 | 4 | MOUNT secret 5 | stuff/and#things 6 | 7 | 8 | -------------------------------------------------------------------------------- /test/integration/bad_token.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "1..1" 4 | 5 | export VAULTENV_RETRY_ATTEMPTS=1 6 | stack exec --no-nix-pure -- vaultenv \ 7 | --no-connect-tls \ 8 | --token thiswillfail \ 9 | --host ${VAULT_HOST} \ 10 | --port ${VAULT_PORT} \ 11 | --secrets-file ${VAULT_SEEDS} \ 12 | /bin/echo "not ok 1 - this should never print" 13 | 14 | echo "ok 1 - vault didn't fetch any secrets" 15 | -------------------------------------------------------------------------------- /test/integration/duplicate_env_var.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "1..4" 4 | 5 | export TESTING_KEY=testing666 6 | stack exec --no-nix-pure -- vaultenv \ 7 | --no-connect-tls \ 8 | --host ${VAULT_HOST} \ 9 | --port ${VAULT_PORT} \ 10 | --secrets-file ${VAULT_SEEDS} -- \ 11 | /bin/echo "not ok 1 - vaultenv didn't error with duplicate env var" 12 | 13 | echo "ok 1 - vaultenv didn't complete" 14 | 15 | stack exec --no-nix-pure -- vaultenv \ 16 | --no-connect-tls \ 17 | --host ${VAULT_HOST} \ 18 | --port ${VAULT_PORT} \ 19 | --duplicate-variable-behavior error \ 20 | --secrets-file ${VAULT_SEEDS} -- \ 21 | /bin/echo "not ok 2 - vaultenv didn't error with duplicate env var" 22 | 23 | echo "ok 2 - vaultenv didn't complete" 24 | 25 | 26 | stack exec --no-nix-pure -- vaultenv \ 27 | --no-connect-tls \ 28 | --host ${VAULT_HOST} \ 29 | --port ${VAULT_PORT} \ 30 | --duplicate-variable-behavior keep \ 31 | --secrets-file ${VAULT_SEEDS} -- \ 32 | /usr/bin/env \ 33 | | grep "TESTING_KEY=testing666" 34 | 35 | if [ $? -eq 0 ]; then 36 | echo "ok 3 - vaultenv passed through test var" 37 | else 38 | echo "not ok 3 - vaultenv didn´t pass through test var" 39 | fi 40 | 41 | stack exec --no-nix-pure -- vaultenv \ 42 | --no-connect-tls \ 43 | --host ${VAULT_HOST} \ 44 | --port ${VAULT_PORT} \ 45 | --duplicate-variable-behavior overwrite \ 46 | --secrets-file ${VAULT_SEEDS} -- \ 47 | /usr/bin/env \ 48 | | grep "TESTING_KEY=testing42" 49 | 50 | if [ $? -eq 0 ]; then 51 | echo "ok 4 - vaultenv overwrote test var" 52 | else 53 | echo "not ok 4 - vaultenv didn´t overwrite test var" 54 | fi 55 | 56 | unset TESTING_KEY 57 | -------------------------------------------------------------------------------- /test/integration/environment_inheritance.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "1..2" 4 | 5 | export TEST_VAR="willbeset" 6 | stack exec --no-nix-pure -- vaultenv \ 7 | --no-connect-tls \ 8 | --host ${VAULT_HOST} \ 9 | --port ${VAULT_PORT} \ 10 | --secrets-file ${VAULT_SEEDS} \ 11 | --inherit-env \ 12 | /usr/bin/env \ 13 | | grep "TEST_VAR" 14 | 15 | if [ $? -eq 0 ]; then 16 | echo "ok 1 - vaultenv passed through test var" 17 | else 18 | echo "not ok 1 - vaultenv didn´t pass through test var" 19 | fi 20 | 21 | stack exec --no-nix-pure -- vaultenv \ 22 | --no-connect-tls \ 23 | --host ${VAULT_HOST} \ 24 | --port ${VAULT_PORT} \ 25 | --secrets-file ${VAULT_SEEDS} \ 26 | --no-inherit-env \ 27 | /usr/bin/env \ 28 | | grep -v "TEST_VAR" 29 | 30 | if [ $? -eq 0 ]; then 31 | echo "ok 2 - vaultenv didn't pass through test var" 32 | else 33 | echo "not ok 2 - vaultenv passed through test var" 34 | fi 35 | -------------------------------------------------------------------------------- /test/integration/happy_path.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eufo pipefail 4 | 5 | echo "1..7" 6 | 7 | if [ $VAULT_TOKEN = "integration" ]; then 8 | echo "ok 1 - vault token set" 9 | else 10 | echo "not ok 1 - vault token not set" 11 | fi 12 | 13 | test_env_contents() { 14 | # Arguments: 15 | # 16 | # $1 - Test number 17 | # $2 - Env var to check 18 | # $3 - Contents of (first line of) env var 19 | # $4 - Secrets file path 20 | 21 | OUTPUT=$(stack exec --no-nix-pure -- vaultenv \ 22 | --no-connect-tls \ 23 | --host ${VAULT_HOST} \ 24 | --port ${VAULT_PORT} \ 25 | --secrets-file $4 -- \ 26 | /usr/bin/env) 27 | 28 | GREPPED=`grep "^$2" <<< $OUTPUT` 29 | grep $3 <<< $GREPPED 30 | 31 | echo "ok $1 - happy path" 32 | } 33 | 34 | test_env_contents 2 "TESTING_KEY" "testing42" ${VAULT_SEEDS} 35 | test_env_contents 3 "TEST_TEST" "testing42" ${VAULT_SEEDS} 36 | test_env_contents 4 "TEST__TEST" "testing42" ${VAULT_SEEDS} 37 | test_env_contents 5 "_TEST__TEST" "testing42" ${VAULT_SEEDS} 38 | 39 | test_env_contents 6 "SECRET_TESTING_KEY" "testing42" ${VAULT_SEEDS_V2} 40 | 41 | # Also test the happy path with `--vault-addr` 42 | export VAULT_ADDR="http://${VAULT_HOST}:${VAULT_PORT}" 43 | stack exec --no-nix-pure -- vaultenv \ 44 | --secrets-file ${VAULT_SEEDS} \ 45 | -- \ 46 | echo "ok 7 - Happy path from VAULT_ADDR" 47 | -------------------------------------------------------------------------------- /test/integration/inherit-env-blacklist-both.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | echo "1..4" 6 | 7 | testing_keys_present() { 8 | stack exec --no-nix-pure -- vaultenv \ 9 | --no-connect-tls \ 10 | --host ${VAULT_HOST} \ 11 | --port ${VAULT_PORT} \ 12 | --secrets-file ${VAULT_SEEDS} \ 13 | "$@" \ 14 | /usr/bin/env | grep --count -e "TESTING_KEY=testing42" -e "VAULT_TOKEN=integration" 15 | } 16 | 17 | # Test that the variables are present when no blacklist is used. 18 | if [[ $(testing_keys_present) -eq 2 ]]; then 19 | echo "ok 1 - keys present when not blacklisted" 20 | else 21 | echo "not ok 1 - keys absent while not blacklisted" 22 | fi 23 | 24 | # Test that blacklisting secrets via the `--inherit-env-blacklist` argument works 25 | if [[ $(testing_keys_present --inherit-env-blacklist VAULT_TOKEN,TESTING_KEY) -eq 0 ]]; then 26 | echo "ok 2 - keys absent when blacklisted via argument" 27 | else 28 | echo "not ok 2 - keys present while blacklisted via argument" 29 | fi 30 | 31 | # Test that blacklisting via the environment variable works. 32 | export VAULTENV_INHERIT_ENV_BLACKLIST=VAULT_TOKEN,TESTING_KEY 33 | if [[ $(testing_keys_present) -eq 0 ]]; then 34 | echo "ok 3 - keys absent when blacklisted via environment" 35 | else 36 | echo "not ok 3 - keys present while blacklisted via environment" 37 | fi 38 | unset VAULTENV_INHERIT_ENV_BLACKLIST 39 | 40 | # Test that blacklisting via the .env config file works. 41 | echo "VAULTENV_INHERIT_ENV_BLACKLIST=VAULT_TOKEN,TESTING_KEY" > .env 42 | if [[ $(testing_keys_present) -eq 0 ]]; then 43 | echo "ok 4 - keys absent when blacklisted via config file" 44 | else 45 | echo "not ok 4 - keys present while blacklisted via config file" 46 | fi 47 | rm .env 48 | -------------------------------------------------------------------------------- /test/integration/inherit-env-blacklist-existing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | echo "1..6" 6 | 7 | TEMP_RUN_DIRECTORY=$(mktemp -d) 8 | vault_token_present() { 9 | stack exec --no-nix-pure --cwd $TEMP_RUN_DIRECTORY -- vaultenv \ 10 | --no-connect-tls \ 11 | --host ${VAULT_HOST} \ 12 | --port ${VAULT_PORT} \ 13 | --secrets-file ${VAULT_SEEDS} \ 14 | "$@" \ 15 | /usr/bin/env | grep --count "VAULT_TOKEN=integration" 16 | } 17 | 18 | ## Test blacklisting a variable from the outer environment (VAULT_TOKEN). 19 | # Test that the variable is present when no blacklist is used. 20 | if [[ $(vault_token_present) -eq 1 ]]; then 21 | echo "ok 1 - VAULT_TOKEN present when not blacklisted" 22 | else 23 | echo "not ok 1 - VAULT_TOKEN absent while not blacklisted" 24 | fi 25 | 26 | # Test that blacklisting secrets via the `--inherit-env-blacklist` argument works 27 | if [[ $(vault_token_present --inherit-env-blacklist VAULT_TOKEN) -eq 0 ]]; then 28 | echo "ok 2 - VAULT_TOKEN absent when blacklisted via argument" 29 | else 30 | echo "not ok 2 - VAULT_TOKEN present while blacklisted via argument" 31 | fi 32 | 33 | # # Test that blacklisting via the environment variable works. 34 | export VAULTENV_INHERIT_ENV_BLACKLIST=VAULT_TOKEN 35 | if [[ $(vault_token_present) -eq 0 ]]; then 36 | echo "ok 3 - VAULT_TOKEN absent when blacklisted via environment" 37 | else 38 | echo "not ok 3 - VAULT_TOKEN present while blacklisted via environment" 39 | fi 40 | unset VAULTENV_INHERIT_ENV_BLACKLIST 41 | 42 | # Test that blacklisting via the .env config file works. 43 | echo "VAULTENV_INHERIT_ENV_BLACKLIST=VAULT_TOKEN" > "${TEMP_RUN_DIRECTORY}/.env" 44 | if [[ $(vault_token_present) -eq 0 ]]; then 45 | echo "ok 4 - VAULT_TOKEN absent when blacklisted via config file" 46 | else 47 | echo "not ok 4 - VAULT_TOKEN present while blacklisted via config file" 48 | fi 49 | 50 | # Test that the blacklist from config can be disabled with the CLI 51 | vault_token_present_with_disabled_blacklist() { 52 | stack exec --no-nix-pure --cwd $TEMP_RUN_DIRECTORY -- vaultenv \ 53 | --no-connect-tls \ 54 | --host ${VAULT_HOST} \ 55 | --port ${VAULT_PORT} \ 56 | --secrets-file ${VAULT_SEEDS} \ 57 | --inherit-env-blacklist '' \ 58 | "$@" \ 59 | /usr/bin/env | grep --count "VAULT_TOKEN=integration" 60 | } 61 | 62 | if [[ $(vault_token_present_with_disabled_blacklist) -eq 1 ]]; then 63 | echo "ok 5 - VAULT_TOKEN present when blacklisted via config but blacklist disabled with CLI" 64 | else 65 | echo "not ok 5 - VAULT_TOKEN absent when blacklisted via config but blacklist disabled with CLI" 66 | fi 67 | 68 | # Test that the blacklist from config can be disabled with the environment 69 | export VAULTENV_INHERIT_ENV_BLACKLIST= 70 | if [[ $(vault_token_present) -eq 1 ]]; then 71 | echo "ok 6 - VAULT_TOKEN present when blacklisted via config but blacklist disabled with environment" 72 | else 73 | echo "not ok 6 - VAULT_TOKEN absent when blacklisted via config but blacklist disabled with environment" 74 | fi 75 | unset VAULTENV_INHERIT_ENV_BLACKLIST 76 | 77 | rm "${TEMP_RUN_DIRECTORY}/.env" 78 | -------------------------------------------------------------------------------- /test/integration/inherit-env-blacklist-secret.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | echo "1..4" 6 | 7 | TEMP_RUN_DIRECTORY=$(mktemp -d) 8 | testing_key_present() { 9 | stack exec --no-nix-pure --cwd $TEMP_RUN_DIRECTORY -- vaultenv \ 10 | --no-connect-tls \ 11 | --host ${VAULT_HOST} \ 12 | --port ${VAULT_PORT} \ 13 | --secrets-file ${VAULT_SEEDS} \ 14 | "$@" \ 15 | /usr/bin/env | grep --count "TESTING_KEY=testing42" 16 | } 17 | 18 | ## Test blacklisting a variable that would normally be set by Vaultenv itself (TESTING_KEY). 19 | # Test that the variable is present when no blacklist is used. 20 | if [[ $(testing_key_present) -eq 1 ]]; then 21 | echo "ok 1 - TESTING_KEY present when not blacklisted" 22 | else 23 | echo "not ok 1 - TESTING_KEY absent while not blacklisted" 24 | fi 25 | 26 | # Test that blacklisting secrets via the `--inherit-env-blacklist` argument works 27 | if [[ $(testing_key_present --inherit-env-blacklist TESTING_KEY) -eq 0 ]]; then 28 | echo "ok 2 - TESTING_KEY absent when blacklisted via argument" 29 | else 30 | echo "not ok 2 - TESTING_KEY present while blacklisted via argument" 31 | fi 32 | 33 | # # Test that blacklisting via the environment variable works. 34 | export VAULTENV_INHERIT_ENV_BLACKLIST=TESTING_KEY 35 | if [[ $(testing_key_present) -eq 0 ]]; then 36 | echo "ok 3 - TESTING_KEY absent when blacklisted via environment" 37 | else 38 | echo "not ok 3 - TESTING_KEY present while blacklisted via environment" 39 | fi 40 | unset VAULTENV_INHERIT_ENV_BLACKLIST 41 | 42 | # Test that blacklisting via the .env config file works. 43 | echo "VAULTENV_INHERIT_ENV_BLACKLIST=TESTING_KEY" > "${TEMP_RUN_DIRECTORY}/.env" 44 | if [[ $(testing_key_present) -eq 0 ]]; then 45 | echo "ok 4 - TESTING_KEY absent when blacklisted via config file" 46 | else 47 | echo "not ok 4 - TESTING_KEY present while blacklisted via config file" 48 | fi 49 | rm "${TEMP_RUN_DIRECTORY}/.env" 50 | -------------------------------------------------------------------------------- /test/integration/invalid_addr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "1..5" 4 | 5 | #test if the addr is accepted when the same as the host, port and scheme (no tls) 6 | stack exec --no-nix-pure -- vaultenv \ 7 | --no-connect-tls \ 8 | --host ${VAULT_HOST} \ 9 | --port ${VAULT_PORT} \ 10 | --addr http://${VAULT_HOST}:${VAULT_PORT} \ 11 | --secrets-file ${VAULT_SEEDS} -- \ 12 | /bin/echo "ok 1 - vaultenv correctly recognized the addr" 13 | 14 | if [ $? -ne 0 ]; then 15 | echo "not ok 1 - vaultenv didn't complete" 16 | fi 17 | 18 | #test if the addr is rejected when the same as the host, port, but different form the scheme (use tls) 19 | stack exec --no-nix-pure -- vaultenv \ 20 | --connect-tls \ 21 | --host ${VAULT_HOST} \ 22 | --port ${VAULT_PORT} \ 23 | --secrets-file ${VAULT_SEEDS} \ 24 | --addr http://${VAULT_HOST}:${VAULT_PORT} \ 25 | /bin/echo "not ok 2 - this should never print" 26 | 27 | echo "ok 2 - vault rejected the scheme" 28 | 29 | #test if rejected with a different scheme (https vs no-tls) 30 | stack exec --no-nix-pure -- vaultenv \ 31 | --no-connect-tls \ 32 | --host ${VAULT_HOST} \ 33 | --port ${VAULT_PORT} \ 34 | --secrets-file ${VAULT_SEEDS} \ 35 | --addr https://${VAULT_HOST}:${VAULT_PORT} \ 36 | /bin/echo "not ok 3 - this should never print" 37 | 38 | echo "ok 3 - vault rejected the scheme" 39 | 40 | #test if rejected with an invalid host 41 | stack exec --no-nix-pure -- vaultenv \ 42 | --no-connect-tls \ 43 | --host invalidhost \ 44 | --port ${VAULT_PORT} \ 45 | --secrets-file ${VAULT_SEEDS} \ 46 | --addr http://${VAULT_HOST}:${VAULT_PORT} \ 47 | /bin/echo "not ok 4 - this should never print" 48 | 49 | echo "ok 4 - vault rejected the host" 50 | 51 | #test if rejected with an invalid port 52 | stack exec --no-nix-pure -- vaultenv \ 53 | --no-connect-tls \ 54 | --host ${VAULT_HOST} \ 55 | --port 4321 \ 56 | --secrets-file ${VAULT_SEEDS} \ 57 | --addr http://${VAULT_HOST}:${VAULT_PORT} \ 58 | /bin/echo "not ok 5 - this should never print" 59 | 60 | echo "ok 5 - vault rejected the port" 61 | -------------------------------------------------------------------------------- /test/integration/kubernetes_auth.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Verify Vault Kubernetes authentication. 5 | 6 | WARNING 7 | This script expects a (local) Kubernetes cluster to be running (tested with 8 | minikube) and it runs arbitrary `kubectl` commands. Do not run this with kubectl 9 | pointed at a production cluster! 10 | 11 | Before you can run this test, you need to build a container with Vaultenv and 12 | put it in Minikube's registry, as follows: 13 | 14 | nix build --file kubernetes_auth_container.nix --out-link container.tar.gz 15 | minikube image load container.tar.gz 16 | """ 17 | 18 | import os 19 | import signal 20 | import base64 21 | import subprocess 22 | import textwrap 23 | import time 24 | import json 25 | 26 | from typing import Any, Set 27 | from pathlib import Path 28 | from urllib.request import urlopen 29 | 30 | import tap 31 | 32 | 33 | def kubectl(*args: str, **kwargs: Any) -> subprocess.CompletedProcess: 34 | return subprocess.run(['kubectl', *args], **kwargs) 35 | 36 | 37 | def vault(*args: str, **kwargs: Any) -> subprocess.CompletedProcess: 38 | return subprocess.run(['vault', *args], **kwargs) 39 | 40 | 41 | def tap_starts_with_success(status: str, haystack: str) -> None: 42 | if haystack.startswith('Success!'): 43 | tap.ok(status) 44 | else: 45 | tap.not_ok(status) 46 | 47 | 48 | def main() -> None: 49 | """ 50 | Start a Vault server, and enable Kubernetes authentication on it. Then start 51 | a pod with the test image that runs Vaultenv to retrieve some secrets and 52 | then runs "env". Then check the logs for that pod to see if we really 53 | retrieved the secrets. 54 | """ 55 | tap.plan(7) 56 | 57 | tap.diagnose("Setting up Kubernetes service account ...") 58 | kubectl( 59 | '--namespace', 'default', 'create', 'serviceaccount', 'test-vault-auth', 60 | check=False, 61 | ) 62 | kubectl( 63 | 'apply', '--filename', '-', 64 | input=textwrap.dedent( 65 | """\ 66 | --- 67 | apiVersion: rbac.authorization.k8s.io/v1 68 | kind: ClusterRoleBinding 69 | metadata: 70 | name: role-tokenreview-binding 71 | namespace: default 72 | roleRef: 73 | apiGroup: rbac.authorization.k8s.io 74 | kind: ClusterRole 75 | name: system:auth-delegator 76 | subjects: 77 | - kind: ServiceAccount 78 | name: test-vault-auth 79 | namespace: default 80 | """), 81 | encoding='utf-8', 82 | check=True, 83 | ) 84 | 85 | tap.diagnose("Starting Vault server ...") 86 | vault_server = run_vault_server() 87 | 88 | # Ensure that all of the `vault` calls below connect over http to the dev 89 | # server. 90 | os.environ["VAULT_ADDR"] = "http://127.0.0.1:8200" 91 | 92 | tap.diagnose("Adding policy and test value to Vault ...") 93 | output = vault( 94 | "policy", "write", "testapp-kv-ro", "-", 95 | input=textwrap.dedent( 96 | """\ 97 | path "secret/data/testapp/*" { 98 | capabilities = ["read", "list"] 99 | } 100 | path "secret/metadata/testapp/*" { 101 | capabilities = ["list"] 102 | } 103 | """ 104 | ), 105 | encoding="utf-8", 106 | stdout=subprocess.PIPE, 107 | check=True, 108 | ).stdout 109 | tap_starts_with_success('Created testapp-kv-ro policy', output) 110 | 111 | vault( 112 | "kv", 113 | "put", 114 | "secret/testapp/config", 115 | "username=vaultenvtestuser", 116 | "password=hunter2", 117 | check=True, 118 | ) 119 | 120 | # Get the name of the service account and export it where Vault can find it. 121 | vault_service_account_name = kubectl( 122 | "get", "sa", "test-vault-auth", "--output", "jsonpath={.secrets[*]['name']}", 123 | capture_output=True, 124 | check=True, 125 | encoding='utf-8', 126 | ).stdout 127 | os.environ["VAULT_SA_NAME"] = vault_service_account_name 128 | if vault_service_account_name.startswith('test-vault-auth'): 129 | tap.ok(f"Vault service account created: {vault_service_account_name}") 130 | else: 131 | tap.not_ok("No valid Vault service account.") 132 | 133 | service_account_jwt_b64 = kubectl( 134 | "get", "secret", vault_service_account_name, "--output", "go-template={{ .data.token }}", 135 | capture_output=True, 136 | check=True, 137 | encoding='utf-8', 138 | ).stdout 139 | service_account_jwt = base64.b64decode(service_account_jwt_b64).decode('utf-8') 140 | 141 | kubernetes_ca_cert_b64 = kubectl( 142 | "config", "view", "--raw", "--minify", "--flatten", "--output", "jsonpath={.clusters[].cluster.certificate-authority-data}", 143 | capture_output=True, 144 | check=True, 145 | encoding='utf-8', 146 | ).stdout 147 | kubernetes_ca_cert = base64.b64decode(kubernetes_ca_cert_b64).decode('utf-8') 148 | 149 | kubernetes_host = kubectl( 150 | "config", "view", "--raw", "--minify", "--flatten", "--output", 151 | "jsonpath={.clusters[].cluster.server}", 152 | capture_output=True, 153 | check=True, 154 | encoding='utf-8', 155 | ).stdout 156 | 157 | # We need to tell Vault what the issuer of the JWT tokens is. But to figure 158 | # out what the issuer is, we first need to enable issuer discovery. Then we 159 | # can query the issuers through a "kubectl proxy" reverse proxy. 160 | kubectl( 161 | "create", "clusterrolebinding", "oidc-reviewer", 162 | "--clusterrole=system:service-account-issuer-discovery", 163 | "--group=system:unauthenticated", 164 | ) 165 | kubectl_proxy = subprocess.Popen( 166 | ["kubectl", "proxy", "--port", "8001"], 167 | stdin=subprocess.DEVNULL, 168 | stdout=subprocess.DEVNULL, 169 | ) 170 | sleep_seconds = 0.1 171 | time.sleep(sleep_seconds) 172 | openid_config_bytes = urlopen("http://127.0.0.1:8001/.well-known/openid-configuration") 173 | openid_config_json = json.load(openid_config_bytes) 174 | issuer = openid_config_json['issuer'] 175 | kubectl_proxy.terminate() 176 | 177 | output = vault("auth", "enable", "kubernetes", check=True, stdout=subprocess.PIPE, encoding='utf-8').stdout 178 | tap_starts_with_success('Enabled Kubernetes auth in Vault', output) 179 | 180 | output = vault( 181 | "write", 182 | "auth/kubernetes/config", 183 | f"token_reviewer_jwt={service_account_jwt}", 184 | f"kubernetes_host={kubernetes_host}", 185 | f"kubernetes_ca_cert={kubernetes_ca_cert}", 186 | f"issuer={issuer}", 187 | check=True, 188 | stdout=subprocess.PIPE, 189 | encoding='utf-8', 190 | ).stdout 191 | tap_starts_with_success('Set Kubernetes auth config', output) 192 | 193 | output = vault( 194 | "write", 195 | "auth/kubernetes/role/vaultenv-test-role", 196 | "bound_service_account_names=test-vault-auth", 197 | "bound_service_account_namespaces=default", 198 | "policies=testapp-kv-ro", 199 | "ttl=24h", 200 | check=True, 201 | stdout=subprocess.PIPE, 202 | encoding='utf-8', 203 | ).stdout 204 | tap_starts_with_success('Bind service account role', output) 205 | 206 | # Delete the pod, in case it was left over from a previous test run. 207 | kubectl("delete", "pod", "vaultenv-test-pod", check=False) 208 | 209 | kubectl( 210 | 'apply', '--filename', '-', 211 | input=textwrap.dedent( 212 | """ 213 | apiVersion: v1 214 | kind: Pod 215 | 216 | metadata: 217 | name: vaultenv-test-pod 218 | 219 | spec: 220 | containers: 221 | - name: vaultenv-test 222 | image: vaultenv/vaultenv:test 223 | # Image should already be loaded into the registry before we 224 | # start the test. It should be locally built, so we can't pull 225 | # it from anywhere anyway. 226 | imagePullPolicy: Never 227 | args: [ 228 | "/bin/vaultenv", 229 | "--log-level", "info", 230 | # Inside the container, Minikube adds an entry to /etc/hosts 231 | # that resolves to the IP of the host's network interface. 232 | "--addr", "http://host.minikube.internal:8200", 233 | "--kubernetes-role", "vaultenv-test-role", 234 | "--secrets-file", "/lib/test.secrets", 235 | "--", 236 | "/bin/env" 237 | ] 238 | 239 | # Allow host access, because the Vault server runs on the host network. 240 | hostNetwork: true 241 | dnsPolicy: Default 242 | serviceAccount: test-vault-auth 243 | """), 244 | encoding='utf-8', 245 | check=True, 246 | ) 247 | 248 | # Wait up to 10 seconds for the pod to become ready. 249 | print("# Waiting for pod to become ready ...") 250 | for _ in range(20): 251 | ready = kubectl( 252 | "get", "pod", "vaultenv-test-pod", "--output", "jsonpath={.status.containerStatuses[].ready}", 253 | stdout=subprocess.PIPE, 254 | check=True, 255 | ).stdout 256 | if ready == b'true': 257 | break 258 | else: 259 | sleep_seconds = 0.5 260 | time.sleep(sleep_seconds) 261 | 262 | # Then wait up to another 10 seconds for all logs to come in. 263 | print("# Waiting for pod logs ...") 264 | logs = [] 265 | for _ in range(20): 266 | logs = kubectl( 267 | "logs", "vaultenv-test-pod", 268 | stdout=subprocess.PIPE, 269 | check=True, 270 | encoding="utf-8", 271 | ).stdout.splitlines() 272 | 273 | # Vaultenv in info mode prints 16 lines of config at the start. The next 274 | # line will be an error or a result, so only after that line, we can 275 | # continue. 276 | if len(logs) > 16: 277 | break 278 | else: 279 | sleep_seconds = 0.5 280 | time.sleep(sleep_seconds) 281 | 282 | expected_lines = [ 283 | "DATA_TESTAPP_CONFIG_USERNAME=vaultenvtestuser", 284 | "DATA_TESTAPP_CONFIG_PASSWORD=hunter2", 285 | ] 286 | for line in expected_lines: 287 | if line in logs: 288 | tap.ok(f'{line} was retrieved') 289 | else: 290 | tap.not_ok(f'{line} was not retrieved') 291 | 292 | # Clean up the pod now that the test is done. 293 | kubectl("delete", "pod", "vaultenv-test-pod", check=False) 294 | 295 | # We need to kill and restart the server or the kernel will accept the TCP 296 | # connections while the Vault server is paused. 297 | vault_server.terminate() 298 | wait_seconds = 5 299 | vault_server.wait(timeout=wait_seconds) 300 | 301 | 302 | def run_vault_server() -> subprocess.Popen: 303 | """ 304 | Run the Vault dev server with Kubernetes auth enabled. 305 | 306 | Return a handle to the Vault server process. 307 | """ 308 | vault_server = subprocess.Popen( 309 | [ 310 | "vault", 311 | "server", 312 | "-dev", 313 | "-dev-root-token-id=integration", 314 | # Listen on 0.0.0.0 so the birdged Minikube network can also reach 315 | # the server. 316 | "-dev-listen-address", "0.0.0.0:8200", 317 | ], 318 | stdin=subprocess.DEVNULL, 319 | stdout=subprocess.DEVNULL, 320 | stderr=subprocess.DEVNULL, 321 | ) 322 | sleep_seconds = 1 323 | time.sleep(sleep_seconds) 324 | 325 | return vault_server 326 | 327 | 328 | if __name__ == "__main__": 329 | main() 330 | -------------------------------------------------------------------------------- /test/integration/kubernetes_auth_container.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? (import ../../nix/nixpkgs-pinned.nix) {} }: 2 | 3 | let 4 | vaultenv = (import ../../nix/release.nix).vaultenv; 5 | 6 | secretsFile = pkgs.writeTextFile { 7 | name = "test.secrets"; 8 | destination = "/lib/test.secrets"; 9 | text = 10 | '' 11 | data/testapp/config#username 12 | data/testapp/config#password 13 | ''; 14 | }; 15 | 16 | # When Vaultenv runs in the container in the pod, it needs to access Vault 17 | # that is running outside. When the pod is running in Minikube, Vaultenv can 18 | # reach the host network at host.minikube.internal. Its ip address is written 19 | # in /etc/hosts, which is mounted by Minikube inside the container. But we 20 | # still need to convince glibc to actually read /etc/hosts, and for that, we 21 | # need to provide an nsswitch config that has a "hosts: files" line. 22 | nsswitch = pkgs.writeTextFile { 23 | name = "nsswitch.conf"; 24 | destination = "/etc/nsswitch.conf"; 25 | text = "hosts: files dns"; 26 | }; 27 | 28 | container = pkgs.dockerTools.buildLayeredImage { 29 | name = "vaultenv/vaultenv"; 30 | tag = "test"; 31 | contents = [ 32 | vaultenv 33 | secretsFile 34 | nsswitch 35 | pkgs.coreutils 36 | pkgs.bash 37 | pkgs.iputils 38 | ]; 39 | }; 40 | 41 | in 42 | container 43 | -------------------------------------------------------------------------------- /test/integration/no_token_configured.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "1..1" 4 | 5 | unset VAULT_TOKEN 6 | stack exec --no-nix-pure -- vaultenv \ 7 | --no-connect-tls \ 8 | --host ${VAULT_HOST} \ 9 | --port ${VAULT_PORT} \ 10 | --secrets-file ${VAULT_SEEDS} \ 11 | /bin/echo "not ok 1 - completed, but no token is configured" 12 | 13 | echo "ok 1 - vaultenv errors when token isn't configured" 14 | -------------------------------------------------------------------------------- /test/integration/retry_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Tests whether the retry mechanism in Vaultenv works correctly by starting Vaultenv 4 | before the Vault server is ready. Runs after the rest of the tests in 5 | integration_test.sh, and requires that there is no Vault server running. 6 | 7 | Requires these environment variables (should all be set by integration_test.sh): 8 | - VAULT_ADDR, VAULT_HOST, VAULT_PORT, VAULT_TOKEN: Credentials to connect to development Vault 9 | - VAULT_SEEDS, VAULT_SEEDS_V2: Paths to a V1 and V2 secrets file 10 | """ 11 | 12 | import os 13 | import signal 14 | import subprocess 15 | from pathlib import Path 16 | from time import sleep 17 | from typing import Set 18 | 19 | import tap 20 | 21 | 22 | def main() -> None: 23 | """ 24 | Test whether the retry behaviour in Vaultenv works as expected. 25 | 26 | We do this by: 27 | - Starting two Vaultenv processes (one for each version of the secrets file 28 | format) and letting them run for 5 seconds (during which they should retry) 29 | - Suspending the Vaultenv processes with the SIGSTOP signal 30 | - Setting up a Vault server process with the appropriate secrets 31 | - Resuming the Vaultenv processes with the SIGCONT signal 32 | - Checking whether the Vaultenv processes exited with code 0 and got the right 33 | secrets (by having them run /usr/bin/env) 34 | - Killing the Vault server 35 | 36 | We do this twice, once for version 1 of Vault's KV secret store API and once for 37 | version 2 of the API. 38 | 39 | The SIGSTOP/SIGCONT signals are necessary to prevent the Vaultenv processes from 40 | contacting the Vault server while it is up, but does not yet contain the right 41 | secrets (which causes a non-retryable error by design). 42 | """ 43 | v1_secrets_file = Path(os.environ["VAULT_SEEDS"]) 44 | v2_secrets_file = Path(os.environ["VAULT_SEEDS_V2"]) 45 | 46 | tap.plan(4) 47 | 48 | tap.diagnose("Starting Vaultenv processes") 49 | v1_handle = run_vaultenv(v1_secrets_file) 50 | v2_handle = run_vaultenv(v2_secrets_file) 51 | 52 | sleep(5) 53 | tap.diagnose("Pausing Vaultenv processes") 54 | v1_handle.send_signal(signal.SIGSTOP) 55 | v2_handle.send_signal(signal.SIGSTOP) 56 | 57 | tap.diagnose("Starting Vault server") 58 | vault_server = run_vault_server() 59 | sleep(5) 60 | 61 | tap.diagnose("Resuming Vaultenv processes") 62 | v1_handle.send_signal(signal.SIGCONT) 63 | v2_handle.send_signal(signal.SIGCONT) 64 | 65 | check_vaultenv_result(v1_handle, secrets_version=1, api_version=1) 66 | check_vaultenv_result(v2_handle, secrets_version=2, api_version=1) 67 | 68 | # We need to kill and restart the server or the kernel will accept the TCP 69 | # connections while the Vault server is paused. 70 | vault_server.terminate() 71 | vault_server.wait(timeout=5) 72 | 73 | tap.diagnose("Starting Vaultenv processes") 74 | v1_handle = run_vaultenv(v1_secrets_file) 75 | v2_handle = run_vaultenv(v2_secrets_file) 76 | sleep(5) 77 | 78 | tap.diagnose("Pausing Vaultenv processes") 79 | v1_handle.send_signal(signal.SIGSTOP) 80 | v2_handle.send_signal(signal.SIGSTOP) 81 | 82 | tap.diagnose("Starting Vault server") 83 | vault_server = run_vault_server() 84 | sleep(5) 85 | 86 | tap.diagnose("Resuming Vaultenv processes") 87 | v1_handle.send_signal(signal.SIGCONT) 88 | v2_handle.send_signal(signal.SIGCONT) 89 | 90 | check_vaultenv_result(v1_handle, secrets_version=1, api_version=2) 91 | check_vaultenv_result(v2_handle, secrets_version=2, api_version=2) 92 | 93 | tap.diagnose("Killing Vault server") 94 | vault_server.terminate() 95 | vault_server.wait(timeout=10) 96 | 97 | def run_vaultenv(secrets_file: Path) -> subprocess.Popen: 98 | """ 99 | Run Vaultenv with the secrets file from the given path. 100 | 101 | Return a subprocess.Popen-handle to the running Vaultenv process. 102 | """ 103 | return subprocess.Popen( 104 | [ 105 | "stack", 106 | "run", 107 | "--no-nix-pure", 108 | "--", 109 | "vaultenv", 110 | "--no-connect-tls", 111 | "--host", 112 | os.environ["VAULT_HOST"], 113 | "--port", 114 | os.environ["VAULT_PORT"], 115 | "--secrets-file", 116 | str(secrets_file), 117 | "/usr/bin/env", 118 | ], 119 | stdin=subprocess.DEVNULL, 120 | stdout=subprocess.PIPE, 121 | stderr=None, # Inherit calling process's stderr 122 | text=True, 123 | ) 124 | 125 | 126 | def check_vaultenv_result( 127 | process_handle: subprocess.Popen, secrets_version: int, api_version: int 128 | ) -> None: 129 | """ 130 | Check the return code and output of the Vaultenv process under the given 131 | process_handle. 132 | 133 | Emit an appropriate TAP message for the result of the test. 134 | """ 135 | 136 | description = f"v{secrets_version} secrets/v{api_version} API" 137 | 138 | return_code = process_handle.wait() 139 | if return_code != 0: 140 | tap.not_ok(f"{description} returned with code {return_code}") 141 | return 142 | 143 | expected_env: Set[str] 144 | if secrets_version == 1: 145 | expected_env = { 146 | "TESTING_KEY=testing42", 147 | "TESTING_OTHERKEY=testing8", 148 | "TESTING2_FOO=val1", 149 | "TESTING2_BAR=val2", 150 | "TEST_TEST=testing42", 151 | "TEST__TEST=testing42", 152 | "_TEST__TEST=testing42", 153 | } 154 | elif secrets_version == 2: 155 | expected_env = { 156 | "SECRET_TESTING_KEY=testing42", 157 | "SECRET_TESTING_OTHERKEY=testing8", 158 | "SECRET_TESTING2_FOO=val1", 159 | "SECRET_TESTING2_BAR=val2", 160 | } 161 | 162 | actual_env = set(line.strip() for line in process_handle.stdout.readlines()) 163 | 164 | if not expected_env <= actual_env: 165 | missing = ", ".join(expected_env - actual_env) 166 | tap.not_ok(f"{description} missing vars {missing}") 167 | return 168 | 169 | tap.ok(description) 170 | 171 | 172 | def run_vault_server() -> subprocess.Popen: 173 | """ 174 | Run the Vault dev server and seed it with the same set of secrets as in 175 | test_integration.sh. 176 | 177 | Return a handle to the Vault server process. 178 | """ 179 | env = { 180 | "VAULT_TOKEN": "integration", 181 | "VAULT_HOST": "127.0.0.1", 182 | "VAULT_PORT": "8200", 183 | "VAULT_ADDR": "http://127.0.0.1:8200", 184 | # Pass through the PATH, so we can locate the vault binary. 185 | "PATH": os.getenv("PATH"), 186 | } 187 | 188 | vault_server = subprocess.Popen( 189 | ["vault", "server", "-dev", "-dev-root-token-id=integration"], 190 | stdin=subprocess.DEVNULL, 191 | stdout=subprocess.DEVNULL, 192 | stderr=subprocess.DEVNULL, 193 | ) 194 | sleep(1) # As in integration_test.sh 195 | 196 | subprocess.run(["vault", "secrets", "disable", "secret"], check=True, env=env) 197 | subprocess.run( 198 | ["vault", "secrets", "enable", "-path=secret", "-version=1", "kv"], 199 | check=True, 200 | env=env, 201 | ) 202 | subprocess.run( 203 | ["vault", "kv", "put", "secret/testing", "key=testing42", "otherkey=testing8"], 204 | check=True, 205 | stdout=subprocess.DEVNULL, 206 | stderr=subprocess.DEVNULL, 207 | env=env, 208 | ) 209 | subprocess.run( 210 | ["vault", "kv", "put", "secret/testing2", "foo=val1", "bar=val2"], 211 | check=True, 212 | stdout=subprocess.DEVNULL, 213 | stderr=subprocess.DEVNULL, 214 | ) 215 | return vault_server 216 | 217 | 218 | if __name__ == "__main__": 219 | main() 220 | -------------------------------------------------------------------------------- /test/integration/tap.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for producing output according to the Test Anything Protocol. 3 | The program prove(1) can process this output. 4 | """ 5 | 6 | 7 | _test_number = 0 8 | 9 | 10 | def plan(n: int) -> None: 11 | """Print the number of tests that are to be run.""" 12 | print(f"1..{n}") 13 | 14 | 15 | def ok(message: str) -> None: 16 | """Report that the current test succeeded.""" 17 | global _test_number 18 | _test_number += 1 19 | print(f"ok {_test_number} - {message}") 20 | 21 | 22 | def not_ok(message: str) -> None: 23 | """Report that the current test failed.""" 24 | global _test_number 25 | _test_number += 1 26 | print(f"not ok {_test_number} - {message}") 27 | 28 | 29 | def diagnose(message: str) -> None: 30 | """Print debug output.""" 31 | print(f"# {message}") 32 | -------------------------------------------------------------------------------- /test/integration_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export VAULT_TOKEN="integration" 4 | export VAULT_HOST="127.0.0.1" 5 | export VAULT_PORT="8200" 6 | export VAULT_ADDR="http://${VAULT_HOST}:${VAULT_PORT}" 7 | 8 | set -e 9 | 10 | # Check the vault command exists: 11 | if ! which vault > /dev/null; then 12 | echo "vault: command not found" 13 | exit 1 14 | fi 15 | 16 | if [[ $(vault -version) != "Vault v1"* ]]; then 17 | echo "Wrong vault version installed. Integration tests require Vault >= 1.0" 18 | echo "Get it here: https://www.vaultproject.io/downloads.html" 19 | exit 1 20 | fi 21 | 22 | # Start the Vault server we're going to use for our integration tests 23 | vault server -dev -dev-root-token-id=${VAULT_TOKEN} &> /dev/null & 24 | 25 | # Brief timeout hack to avoid race condition where we write secrets but 26 | # vault hasn't booted yet. 27 | sleep 1 28 | 29 | # Seed Vault and some secret files 30 | export VAULT_SEEDS="$(mktemp)" 31 | cat > ${VAULT_SEEDS} << EOF 32 | testing#key 33 | testing#otherkey 34 | testing2#foo 35 | testing2#bar 36 | TEST_TEST=testing#key 37 | TEST__TEST=testing#key 38 | _TEST__TEST=testing#key 39 | EOF 40 | 41 | export VAULT_SEEDS_V2="$(mktemp)" 42 | cat > ${VAULT_SEEDS_V2} < /dev/null 61 | vault kv put secret/testing2 foo=val1 bar=val2 &> /dev/null 62 | 63 | # Run all tests if we didn't get any arguments. 64 | if [ $# -eq 0 ]; then 65 | echo "Running all tests" 66 | echo "NOTE: You might see errors in the log. That's part of what we test." 67 | echo "Look at the test summary report to see if there's anything wrong." 68 | prove $(find integration -type f -iname '*.sh' ! -name '_*') 69 | else 70 | echo "Running only $1" 71 | $1 72 | fi 73 | 74 | # Upgrade the default API. 75 | echo "Running tests for Key/Value mount V2" 76 | vault kv enable-versioning secret 77 | 78 | # Run all tests if we didn't get any arguments. 79 | if [ $# -eq 0 ]; then 80 | echo "Running all tests" 81 | echo "NOTE: You might see errors in the log. That's part of what we test." 82 | echo "Look at the test summary report to see if there's anything wrong." 83 | prove $(find integration -type f -iname '*.sh' ! -name '_*') 84 | else 85 | echo "Running only $1" 86 | $1 87 | fi 88 | 89 | # Cleanup the vault dev server 90 | kill %% 91 | 92 | # TODO: fix and enable the kubernetes tests. 93 | # Run the kubernetes auth tests 94 | # This tests run their own Vault 95 | # nix build --file integration/kubernetes_auth_container.nix --out-link container.tar.gz 96 | # minikube start 97 | # minikube image load container.tar.gz 98 | # prove --comments integration/kubernetes_auth.py 99 | # minikube stop 100 | 101 | # The retry tests work locally, but not in CI. 102 | # This is due to the SIGSTOP signal to the vaultenv processes not coming through, 103 | # which is probably due to implementation specifics of Semaphore CI. 104 | if [[ ${CI:-"false"} == "false" ]]; 105 | then 106 | PYTHONUNBUFFERED=1 prove --comments integration/retry_tests.py 107 | fi 108 | -------------------------------------------------------------------------------- /test/invalid/ambiguous.secrets: -------------------------------------------------------------------------------- 1 | foo#bar/baz#quix 2 | FOO=foo=bar/baz#quix 3 | -------------------------------------------------------------------------------- /test/invalid/v1.secrets: -------------------------------------------------------------------------------- 1 | VERSION 1 2 | 3 | foo#bar 4 | -------------------------------------------------------------------------------- /test/invalid/whitespace-wrong-mount.secrets: -------------------------------------------------------------------------------- 1 | VERSION 2 2 | 3 | MOUNT 4 | testing 5 | foo#bar 6 | -------------------------------------------------------------------------------- /test/invalid/whitespace-wrong-version.secrets: -------------------------------------------------------------------------------- 1 | VERSION 2 | 2 3 | 4 | MOUNT secret 5 | secret#whatevs 6 | -------------------------------------------------------------------------------- /test/invalid/wrong_envvar_names.secrets: -------------------------------------------------------------------------------- 1 | 5_shouldnt_lead_with_numbers=testing#secret 2 | -------------------------------------------------------------------------------- /vaultenv.nix: -------------------------------------------------------------------------------- 1 | { haskellPackages 2 | , mkDerivation 3 | , lib 4 | , pkgs 5 | , glibcLocales 6 | , nix-gitignore 7 | }: 8 | let 9 | dependencies = import ./nix/haskell-dependencies.nix haskellPackages; 10 | in 11 | mkDerivation { 12 | pname = "vaultenv"; 13 | version = "0.19.0"; 14 | 15 | src = 16 | let 17 | # We do not want to include all files, because that leads to a lot of things that nix 18 | # has to copy to the temporary build directory that we don't want to have in there 19 | # (e.g. the `.stack-work` directory, the `.git` directory, etc.) 20 | prefixWhitelist = builtins.map builtins.toString [ 21 | ./package.yaml 22 | # Blanket-include for subdirectories 23 | ./app 24 | ./src 25 | ./test 26 | ]; 27 | # Compute source based on whitelist 28 | whitelistedSrc = lib.cleanSourceWith { 29 | src = lib.cleanSource ./.; 30 | filter = path: _type: lib.any (prefix: lib.hasPrefix prefix path) prefixWhitelist; 31 | }; 32 | rootGitignoredSrc = lib.cleanSourceWith { 33 | src = whitelistedSrc; 34 | filter = 35 | let 36 | gitignore = builtins.readFile ./.gitignore; 37 | extraFilter = path: type: 38 | # Where we're going we don't need no documentation 39 | ! (lib.hasSuffix ".md" path); 40 | in 41 | nix-gitignore.gitignoreFilterPure extraFilter gitignore ./.; 42 | }; 43 | testGitignoredSrc = lib.cleanSourceWith { 44 | src = rootGitignoredSrc; 45 | filter = 46 | let 47 | gitignore = builtins.readFile ./test/.gitignore; 48 | in 49 | nix-gitignore.gitignoreFilter gitignore ./test; 50 | }; 51 | in 52 | testGitignoredSrc; 53 | 54 | configureFlags = [ 55 | # Do not print a wall of text before compiling 56 | "--verbose=0" 57 | # Treat warnings as errors 58 | "--ghc-option=-Werror" 59 | ]; 60 | 61 | isLibrary = false; 62 | isExecutable = true; 63 | 64 | buildTools = [ haskellPackages.hpack glibcLocales ]; 65 | preConfigure = '' 66 | # Generate the cabal file from package.yaml 67 | hpack . 68 | ''; 69 | 70 | postInstall = '' 71 | # By default, a Haskell derivation contains extra stuff that causes the output to retain 72 | # a dependency on GHC and other build tools. Those are not necessary for running the 73 | # executable, so we get rid of that. 74 | rm -rf $out/lib 75 | ''; 76 | 77 | # Disable features that we don't need 78 | enableLibraryProfiling = false; 79 | doHoogle = false; 80 | doHaddock = false; 81 | 82 | # Always run the unit tests 83 | doCheck = false; 84 | testToolDepends = []; 85 | 86 | libraryHaskellDepends = dependencies; 87 | librarySystemDepends = [ glibcLocales ]; 88 | 89 | license = lib.licenses.bsd3; 90 | } 91 | --------------------------------------------------------------------------------