├── tests ├── .gitignore ├── test-values │ ├── 001-basic.yaml │ ├── 002-backups.yaml │ └── 003-restore.yaml ├── README.md ├── helm │ ├── 010-lint.sh │ ├── 050-cleanup.sh │ ├── 030-vault-init-unseal.sh │ └── 020-deploy.sh ├── clean_namespace.sh └── test.sh ├── env.gocd.yaml ├── vault-security-model.png ├── helm_charts └── vault │ ├── templates │ ├── vault.postStart.configMap.yaml │ ├── vault.ha.service.yaml │ ├── vault.unsealKeys.secret.yaml │ ├── vault.tls.secret.yaml │ ├── NOTES.txt │ ├── consul.preInstall.configMap.yaml │ ├── vault.service.yaml │ ├── consul.service.yaml │ ├── vault.preInstall.configMap.yaml │ ├── consul.backup.configMap.yaml │ ├── vault.pdb.yaml │ ├── consul.pdb.yaml │ ├── consul.restore.configMap.yaml │ ├── vault.ingress.yaml │ ├── consul.configMap.yaml │ ├── vault.consulClient.configMap.yaml │ ├── vault.configMap.yaml │ ├── vault.preInstall.customCA.job.yaml │ ├── vault.preInstall.vaultSecrets.job.yaml │ ├── consul.backup.deployment.yaml │ ├── consul.restore.job.yaml │ ├── _helpers.tpl │ ├── consul.preInstall.job.yaml │ ├── consul.statefulset.yaml │ └── vault.deployment.yaml │ ├── preInstallVault │ ├── create-vault-secrets.sh │ ├── generate-vault-cert.sh │ └── README.md │ ├── Chart.yaml │ ├── postStart │ └── unseal-vault.sh │ ├── preInstallConsul │ ├── create-consul-secrets.sh │ ├── generate-consul-tls-certs.sh │ ├── openssl.cnf │ └── README.md │ └── values.yaml ├── init ├── vault-unseal.sh └── vault-init.sh ├── LICENSE ├── k8s-executor.sh ├── deploy.gocd.yaml ├── docs └── 0.1.2-MIGRATION.md ├── CHANGELOG.md └── README.md /tests/.gitignore: -------------------------------------------------------------------------------- 1 | .chart 2 | -------------------------------------------------------------------------------- /tests/test-values/001-basic.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | TODO: 2 | * Auto-unseal 3 | * Backups 4 | -------------------------------------------------------------------------------- /tests/test-values/002-backups.yaml: -------------------------------------------------------------------------------- 1 | Consul: 2 | Backup: 3 | Enabled: true 4 | -------------------------------------------------------------------------------- /tests/test-values/003-restore.yaml: -------------------------------------------------------------------------------- 1 | Consul: 2 | Restore: 3 | RestoreFile: "foo" 4 | -------------------------------------------------------------------------------- /env.gocd.yaml: -------------------------------------------------------------------------------- 1 | environments: 2 | public: 3 | pipelines: 4 | - vault-deploy-test 5 | -------------------------------------------------------------------------------- /tests/helm/010-lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Linting Helm Chart" 4 | helm lint ./helm_charts/vault 5 | -------------------------------------------------------------------------------- /vault-security-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PremiereGlobal/vault-helm-chart/HEAD/vault-security-model.png -------------------------------------------------------------------------------- /tests/clean_namespace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CLUSTER=$(kubectl config current-context) 4 | NAMESPACE=${1} 5 | 6 | kubectl get ns ${NAMESPACE} > /dev/null 2>&1 7 | 8 | if [ $? -eq 0 ]; then 9 | echo "Deleting the following namespace on cluster ${CLUSTER}" 10 | echo ${NAMESPACE} 11 | read -r -p "Continue? [y/N] " response 12 | if [[ "$response" =~ ^([yY][eE][sS]|[yY])+$ ]] 13 | then 14 | kubectl delete ns ${NAMESPACE} 15 | fi 16 | fi 17 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.postStart.configMap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.Vault.AutoUnseal }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "vault.fullname" . }}-post-start 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-post-start" 11 | data: 12 | {{ (.Files.Glob "postStart/*").AsConfig | indent 2 }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.ha.service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ template "vault.fullname" . }}-ha 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 10 | spec: 11 | ports: 12 | - name: ha-forwarding 13 | port: {{.Values.Vault.HaForwardingPort}} 14 | selector: 15 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 16 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.unsealKeys.secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.Vault.AutoUnseal }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "vault.fullname" . }}-vault-keys 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 11 | annotations: 12 | "helm.sh/hook": pre-install 13 | "helm.sh/hook-weight": "100" 14 | type: Opaque 15 | data: 16 | key1: {{ "foo" | b64enc }} 17 | {{- end }} 18 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.tls.secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.Vault.Tls.CertString .Values.Vault.Tls.KeyString }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ template "vault.fullname" . }}.tls 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 11 | type: kubernetes.io/tls 12 | data: 13 | tls.crt: {{ .Values.Vault.Tls.CertString | b64enc }} 14 | tls.key: {{ .Values.Vault.Tls.KeyString | b64enc }} 15 | {{- end }} 16 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | Vault has been installed/upgraded. 2 | 3 | It may take several minutes for all of the components to become available. 4 | 5 | If this is a new install, initialize Vault by running the following: 6 | kubectl exec -it $(kubectl --namespace {{ .Release.Namespace }} get pods -o jsonpath="{.items[0].metadata.name}" -l "component={{ .Release.Name }}-{{ .Values.Vault.ComponentName }}") -c {{ template "vault.fullname" . }} --namespace {{ .Release.Namespace }} sh 7 | vault operator init {{ template "tls_skip_verify" . }} 8 | 9 | To unseal Vault, execute into each running pod and execute: 10 | vault operator unseal 11 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.preInstall.configMap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.PreInstall.ComponentName}}" 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{.Values.Consul.PreInstall.ComponentName}}" 10 | annotations: 11 | # This is what defines this resource as a hook. Without this line, the 12 | # job is considered part of the release. 13 | "helm.sh/hook": pre-install 14 | "helm.sh/hook-weight": "50" 15 | data: 16 | {{ (.Files.Glob "preInstallConsul/*").AsConfig | indent 2 }} 17 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ template "vault.fullname" . }} 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 10 | spec: 11 | {{- if not .Values.Vault.Ingress.Enabled }} 12 | type: NodePort 13 | {{- end }} 14 | ports: 15 | - name: https 16 | port: {{.Values.Vault.HttpPort}} 17 | {{- if not .Values.Vault.Ingress.Enabled }} 18 | nodePort: {{.Values.Vault.NodePort}} 19 | {{- end }} 20 | selector: 21 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 22 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{ .Values.Consul.ComponentName }}" 10 | annotations: 11 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 12 | spec: 13 | ports: 14 | # This is only here to expose a service so that consul agents 15 | # can use K8S DNS for discovery 16 | - name: foo 17 | port: 80 18 | clusterIP: None 19 | selector: 20 | component: "{{ .Release.Name }}-{{ .Values.Consul.ComponentName }}" 21 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.preInstall.configMap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.Vault.Tls.LetsEncrypt.Enabled }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.PreInstall.ComponentName }}" 11 | annotations: 12 | # This is what defines this resource as a hook. Without this line, the 13 | # job is considered part of the release. 14 | "helm.sh/hook": pre-install 15 | "helm.sh/hook-weight": "50" 16 | data: 17 | {{ (.Files.Glob "preInstallVault/*").AsConfig | indent 2 }} 18 | {{- end }} 19 | -------------------------------------------------------------------------------- /tests/helm/050-cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Generate a unique name for this build 4 | RELEASE_PREFIX="vault" 5 | UNIQUE_NAME=$RELEASE_PREFIX-$GO_PIPELINE_LABEL 6 | 7 | echo "Deleting namespace: $UNIQUE_NAME..." 8 | kubectl delete namespace $UNIQUE_NAME 9 | 10 | echo "Deleting helm installation: $UNIQUE_NAME..." 11 | helm delete $UNIQUE_NAME --purge 12 | 13 | echo "Waiting for cleanup to finish in Kubernetes..." 14 | COUNT=1 15 | while true 16 | do 17 | kubectl get namespace $UNIQUE_NAME &>/dev/null 18 | if [ $? -eq 1 ]; then 19 | echo "All clean!..." 20 | break 21 | elif [ $COUNT -gt 300 ]; then 22 | echo "Timeout exit (try $COUNT)..." 23 | exit 1 24 | else 25 | echo "Waiting for cleanup to finish (try $COUNT)..." 26 | fi 27 | 28 | COUNT=$((COUNT+1)) 29 | sleep 1 30 | done 31 | -------------------------------------------------------------------------------- /tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BRANCH=${1-"master"} 4 | NAMESPACE=${2-"vault-test"} 5 | TEST=${3-"001-basic"} 6 | LOCAL_HELM_CHART=${4} 7 | 8 | if [ -z "${LOCAL_HELM_CHART}" ]; then 9 | rm -rf .chart 10 | mkdir -p .chart 11 | git clone git@github.com:ReadyTalk/vault-helm-chart.git .chart 12 | cd .chart; git checkout ${BRANCH}; cd .. 13 | LOCAL_HELM_CHART=".chart/helm_charts/vault" 14 | fi 15 | 16 | helm delete vault-test --purge 17 | 18 | # ./clean_namespace.sh ${NAMESPACE} 19 | kubectl create ns ${NAMESPACE} 20 | helm upgrade --install \ 21 | --namespace ${NAMESPACE} \ 22 | --values test-values/${TEST}.yaml \ 23 | vault-test \ 24 | ${LOCAL_HELM_CHART} 25 | 26 | # TODO: Wait for vault to come up... 27 | 28 | # ../init/vault-init.sh vault-test vault-test true 29 | # ../init/vault-unseal.sh vault-test vault-test 30 | -------------------------------------------------------------------------------- /init/vault-unseal.sh: -------------------------------------------------------------------------------- 1 | if [ $# -lt 2 ] 2 | then 3 | echo "Invalid arguments provided" 4 | echo "Valid usage: "`basename "$0"`" " 5 | exit 1 6 | fi 7 | 8 | RELEASE=$1 9 | NAMESPACE=$2 10 | COMPONENT="${RELEASE}-vault" 11 | REQUIRED_KEY_COUNT=3 12 | 13 | SECRET_NAME="$RELEASE-vault-keys" 14 | 15 | echo "Getting unseal keys from Kubernetes secret" 16 | UNSEAL_KEYS=$(kubectl get secret -n $NAMESPACE ${SECRET_NAME} -o yaml | grep -e "key[0-9]\:" | awk '{print $2}') 17 | 18 | for i in `seq 1 $REQUIRED_KEY_COUNT`; 19 | do 20 | KEY=$(echo "$UNSEAL_KEYS" | sed "${i}q;d" | base64 --decode) 21 | kubectl get po -l component=$COMPONENT,release=$RELEASE -n $NAMESPACE \ 22 | | awk '{if(NR>1)print $1}' \ 23 | | xargs -I % kubectl exec -n $NAMESPACE -c $RELEASE % -- sh -c "vault operator unseal --tls-skip-verify $KEY"; 24 | done 25 | -------------------------------------------------------------------------------- /helm_charts/vault/preInstallVault/create-vault-secrets.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | HERITAGE=$(grep "heritage=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 4 | RELEASE=$(grep "release=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 5 | CHART=$(grep "chart=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 6 | COMPONENT=$(grep "component=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 7 | 8 | kubectl delete secrets -l release=$RELEASE,component=$COMPONENT 9 | 10 | # Create k8s Secret for Vault SSL Cert (The cert facing users) 11 | cat /certs/$(ls /certs | grep ".*.crt.chain$") >> /certs/$(ls /certs | grep ".*.crt$") 12 | kubectl create secret tls \ 13 | $FULL_NAME.tls \ 14 | --cert=/certs/$(ls /certs | grep ".*.crt$") \ 15 | --key=/certs/$(ls /certs | grep ".*.key$") 16 | kubectl label secret \ 17 | $FULL_NAME.tls \ 18 | heritage=$HERITAGE \ 19 | release=$RELEASE \ 20 | chart=$CHART \ 21 | component=$COMPONENT 22 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.backup.configMap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.Consul.Backup.Enabled }} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.Backup.ComponentName}}" 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Consul.Backup.ComponentName }}" 11 | data: 12 | config.json : | 13 | { 14 | "server": false, 15 | "datacenter": "{{.Values.Consul.Datacenter}}", 16 | "retry_join": ["{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}"], 17 | "bind_addr": "0.0.0.0", 18 | "ca_file": "/consul/ca/ca.crt.pem", 19 | "cert_file": "/consul/tls/tls.crt", 20 | "key_file": "/consul/tls/tls.key", 21 | "client_addr": "127.0.0.1", 22 | "data_dir": "/consul/data" 23 | } 24 | {{- end}} 25 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.pdb.yaml: -------------------------------------------------------------------------------- 1 | # PodDisruptionBudget to prevent degrading the Vault cluster through 2 | # voluntary cluster changes. 3 | {{- if (and (ne (.Values.Vault.maxUnavailable | toString) "-") .Values.Vault.maxUnavailable) }} 4 | apiVersion: policy/v1beta1 5 | kind: PodDisruptionBudget 6 | metadata: 7 | name: {{ template "vault.fullname" . }}-{{.Values.Vault.ComponentName}} 8 | labels: 9 | heritage: {{ .Release.Service | quote }} 10 | release: {{ .Release.Name | quote }} 11 | chart: {{ template "vault.chart" . }} 12 | component: "{{ .Release.Name }}-{{.Values.Vault.ComponentName}}" 13 | spec: 14 | maxUnavailable: {{ template "vault.pdb.maxUnavailable" . }} 15 | selector: 16 | matchLabels: 17 | heritage: {{ .Release.Service | quote }} 18 | release: {{ .Release.Name | quote }} 19 | chart: {{ template "vault.chart" . }} 20 | component: "{{ .Release.Name }}-{{.Values.Vault.ComponentName}}" 21 | {{- end }} 22 | 23 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.pdb.yaml: -------------------------------------------------------------------------------- 1 | # PodDisruptionBudget to prevent degrading the Consul cluster through 2 | # voluntary cluster changes. 3 | {{- if (and (ne (.Values.Consul.maxUnavailable | toString) "-") .Values.Consul.maxUnavailable) }} 4 | apiVersion: policy/v1beta1 5 | kind: PodDisruptionBudget 6 | metadata: 7 | name: {{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}} 8 | labels: 9 | heritage: {{ .Release.Service | quote }} 10 | release: {{ .Release.Name | quote }} 11 | chart: {{ template "vault.chart" . }} 12 | component: "{{ .Release.Name }}-{{.Values.Consul.ComponentName}}" 13 | spec: 14 | maxUnavailable: {{ template "consul.pdb.maxUnavailable" . }} 15 | selector: 16 | matchLabels: 17 | heritage: {{ .Release.Service | quote }} 18 | release: {{ .Release.Name | quote }} 19 | chart: {{ template "vault.chart" . }} 20 | component: "{{ .Release.Name }}-{{.Values.Consul.ComponentName}}" 21 | {{- end }} 22 | 23 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.restore.configMap.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Release.IsInstall }} 2 | {{- if .Values.Consul.Restore.RestoreFile }} 3 | apiVersion: v1 4 | kind: ConfigMap 5 | metadata: 6 | name: {{ template "vault.fullname" . }}-{{.Values.Consul.Restore.ComponentName}} 7 | labels: 8 | heritage: {{ .Release.Service | quote }} 9 | release: {{ .Release.Name | quote }} 10 | chart: {{ template "vault.chart" . }} 11 | component: "{{ .Release.Name }}-{{ .Values.Consul.Restore.ComponentName }}" 12 | data: 13 | config.json : | 14 | { 15 | "server": false, 16 | "datacenter": "{{.Values.Consul.Datacenter}}", 17 | "retry_join": ["{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}"], 18 | "bind_addr": "0.0.0.0", 19 | "ca_file": "/consul/ca/ca.crt.pem", 20 | "cert_file": "/consul/tls/tls.crt", 21 | "key_file": "/consul/tls/tls.key", 22 | "client_addr": "127.0.0.1", 23 | "data_dir": "/consul/data" 24 | } 25 | {{- end}} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /helm_charts/vault/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | description: A Hashicorp Vault chart for Kubernetes 3 | name: vault 4 | version: 0.2.0 5 | keywords: [hashicorp, vault, secret, consul] 6 | icon: https://raw.githubusercontent.com/docker-library/docs/726714ced14b1e14b6dd99fc82f20f14f1d3cfb1/vault/logo.png 7 | sources: 8 | - https://github.com/hashicorp/vault 9 | - https://github.com/hashicorp/consul 10 | - https://github.com/djenriquez/vault-ui 11 | - https://github.com/bartlettc/docker-kubectl 12 | - https://github.com/bartlettc/omgwtfssl-kubernetes 13 | - https://github.com/bartlettc/letsencrypt-acm:kubernetes 14 | - https://github.com/thorix/consul-backup 15 | - https://github.com/helm/charts/tree/master/incubator/vault 16 | - https://github.com/hashicorp/consul-helm 17 | maintainers: 18 | - name: Ryan Wilcox 19 | email: ryan.wilcox@readytalk.com 20 | - name: Chris Bartlett 21 | email: christopher.bartlett@readytalk.com 22 | contributors: 23 | - name: Miroslav E. Hadzhiev 24 | email: miroslav.hadzhiev@gmail.com 25 | -------------------------------------------------------------------------------- /helm_charts/vault/preInstallVault/generate-vault-cert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Join string by characters 4 | function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } 5 | 6 | IFS=',' read -ra NAMES <<< "$VAULT_TLS_ALT_SERVER_NAMES" 7 | ALT_NAME_STRING=$(join_by '","' "${NAMES[@]}") 8 | if [ -n "$ALT_NAME_STRING" ]; then 9 | ALT_NAME_STRING=',"'$ALT_NAME_STRING'"' 10 | fi 11 | 12 | if [ "$ACME_ENVIRONMENT" == "production" ]; then 13 | ACME_DIRECTORY_URL="https://acme-v01.api.letsencrypt.org/directory" 14 | else 15 | ACME_DIRECTORY_URL="https://acme-staging.api.letsencrypt.org/directory" 16 | fi 17 | 18 | export LETSENCRYPT_AWS_CONFIG='{"acme_account_key": "'$ACME_ACCOUNT_KEY'","acme_directory_url": "'$ACME_DIRECTORY_URL'","domains": [{ "hosts": ["'$VAULT_TLS_SERVER_NAME'"'$ALT_NAME_STRING'],"key_type": "rsa" }]}' 19 | EC2_AVAIL_ZONE=`curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone` 20 | export AWS_DEFAULT_REGION="`echo \"$EC2_AVAIL_ZONE\" | sed -e 's:\([0-9][0-9]*\)[a-z]*\$:\\1:'`" 21 | mkdir -p /certs 22 | .venv/bin/python letsencrypt-aws.py update-certificates 23 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.Vault.Ingress.Enabled }} 2 | apiVersion: extensions/v1beta1 3 | kind: Ingress 4 | metadata: 5 | name: {{ template "vault.fullname" . }} 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 11 | annotations: 12 | kubernetes.io/ingress.class: "nginx" 13 | spec: 14 | rules: 15 | - host: {{.Values.Vault.Tls.ServerName}} 16 | http: 17 | paths: 18 | - path: / 19 | backend: 20 | serviceName: {{ template "vault.fullname" . }} 21 | servicePort: {{.Values.Vault.HttpPort}} 22 | {{- $root := . -}} 23 | {{- range .Values.Vault.Tls.AlternateServerNames | split "," }} 24 | - host: {{ . | quote }} 25 | http: 26 | paths: 27 | - path: / 28 | backend: 29 | serviceName: "{{ template "vault.fullname" $root }}" 30 | servicePort: {{$root.Values.Vault.HttpPort}} 31 | {{- end }} 32 | {{- end }} 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 ReadyTalk 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/helm/030-vault-init-unseal.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # For testing, and "un-initing" 4 | # kubectl exec vault-local-vault-consul-0 -n vault-local -- sh -c "consul kv delete -ca-file=/etc/consul/ca/ca.crt.pem -client-cert=/etc/consul/tls/tls.crt -client-key=/etc/consul/tls/tls.key -http-addr=https://127.0.0.1:8500 -tls-server-name=vault-local-vault-consul -recurse vault" 5 | 6 | RELEASE_PREFIX="vault" 7 | UNIQUE_NAME=$RELEASE_PREFIX-$GO_PIPELINE_LABEL 8 | 9 | # Disable if doing a true test 10 | SKIP_TLS_FLAG="-tls-skip-verify" 11 | 12 | FIRST_VAULT_POD=$(kubectl get po -l component=vault,release=$UNIQUE_NAME -n $UNIQUE_NAME | awk '{if(NR==2)print $1}') 13 | INIT_MESSAGE=$(kubectl exec -n $UNIQUE_NAME -c $UNIQUE_NAME-vault-vault $FIRST_VAULT_POD -- sh -c "vault init $SKIP_TLS_FLAG" 2>&1) 14 | echo "$INIT_MESSAGE" 15 | KEYS=$(echo "$INIT_MESSAGE" | grep "Unseal Key" | awk '{print $4}') 16 | for i in ${KEYS[@]}; 17 | do 18 | kubectl get po -l component=vault -n $UNIQUE_NAME \ 19 | | awk '{if(NR>1)print $1}' \ 20 | | xargs -I % kubectl exec -n $UNIQUE_NAME -c $UNIQUE_NAME-vault-vault % -- sh -c "vault unseal $SKIP_TLS_FLAG $i"; 21 | done 22 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.configMap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{ .Values.Consul.ComponentName }}" 10 | data: 11 | config.json : | 12 | { 13 | "server": true, 14 | "leave_on_terminate": true, 15 | "bind_addr": "0.0.0.0", 16 | "ca_file": "/consul/ca/ca.crt.pem", 17 | "cert_file": "/consul/tls/tls.crt", 18 | "client_addr": "0.0.0.0", 19 | "data_dir": "/consul/data", 20 | "datacenter": {{.Values.Consul.Datacenter | quote }}, 21 | "key_file": "/consul/tls/tls.key", 22 | "ports": { 23 | "http": -1, 24 | "https": {{.Values.Consul.HttpPort}} 25 | }, 26 | "verify_incoming_rpc": true, 27 | "verify_incoming_https": true, 28 | "verify_outgoing": true, 29 | "verify_server_hostname": true, 30 | "disable_anonymous_signature": true, 31 | "disable_update_check": true 32 | } 33 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.consulClient.configMap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: "{{ template "vault.fullname" . }}-{{.Values.Vault.ConsulClient.ComponentName}}" 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 11 | data: 12 | config.json : | 13 | { 14 | "bind_addr": "0.0.0.0", 15 | "ca_file": "/consul/ca/ca.crt.pem", 16 | "cert_file": "/consul/tls/tls.crt", 17 | "data_dir": "/consul/data", 18 | "datacenter": {{.Values.Consul.Datacenter | quote }}, 19 | "key_file": "/consul/tls/tls.key", 20 | "addresses": { 21 | "http": "unix:///consul-unix-socket/consul-client.sock" 22 | }, 23 | "unix_sockets": { 24 | "mode": "600" 25 | }, 26 | "ports": { 27 | "http": {{.Values.Consul.HttpPort}}, 28 | "https": -1 29 | }, 30 | "client_addr": "0.0.0.0", 31 | "verify_incoming": true, 32 | "verify_outgoing": true, 33 | "verify_server_hostname": true 34 | } 35 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.configMap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ template "vault.fullname" . }} 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 11 | data: 12 | config.json : | 13 | storage "consul" { 14 | address = "unix:///consul-unix-socket/consul-client.sock" 15 | scheme = "http" 16 | disable_registration = "{{default "false" .Values.Vault.DisableConsulRegistration}}" 17 | } 18 | listener "tcp" { 19 | address = "0.0.0.0:{{.Values.Vault.HttpPort}}" 20 | tls_key_file = "/vault/tls/tls.key" 21 | tls_cert_file = "/vault/tls/tls.crt" 22 | } 23 | telemetry { 24 | prometheus_retention_time = "24h" 25 | disable_hostname = true 26 | } 27 | disable_mlock = false 28 | default_lease_ttl = "{{.Values.Vault.DefaultLeaseTtl}}" 29 | max_lease_ttl = "{{.Values.Vault.MaxLeaseTtl}}" 30 | ui = {{.Values.Vault.Ui.Enabled}} 31 | {{- if .Values.Vault.HostAliases }} 32 | nsswitch.conf: | 33 | hosts: files mdns4_minimal [NOTFOUND=return] dns 34 | {{- end }} 35 | -------------------------------------------------------------------------------- /helm_charts/vault/postStart/unseal-vault.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Three scenarios 4 | # * Vault is not running yet 5 | # ** Just wait... 6 | # * Vault is running but not initialized 7 | # ** Exit, it needs to be initialized and unsealed manually 8 | # * Vault is running and initialized but sealed 9 | # ** Take action and unseal the vault, then exit 10 | # * Vault is running, initialized and unsealed 11 | # ** all is good, exit 12 | 13 | COUNT=1 14 | LIMIT=30 15 | while [ 1 ] 16 | do 17 | echo "Checking if Vault is up and running (try $COUNT)..." &> /proc/1/fd/1 18 | VAULT_STATUS=$(vault status $1 2>&1) 19 | EXIT_STATUS=$? 20 | if echo "$VAULT_STATUS" | grep "server is not yet initialized"; then 21 | echo "Vault not initialized. Must be manually unsealed. Exiting..." &> /proc/1/fd/1 22 | exit 0 23 | elif [ $EXIT_STATUS -eq 2 ]; then 24 | echo "Vault Sealed. Unsealing..." &> /proc/1/fd/1 25 | for i in `seq 1 3` 26 | do 27 | vault operator unseal $1 $(cat /etc/vault/keys/key$i) &> /proc/1/fd/1 28 | done 29 | exit 0 30 | elif [ $COUNT -ge $LIMIT ]; then 31 | # Dont know what happened... Exiting 32 | echo "$VAULT_STAUS" &> /proc/1/fd/1 33 | exit 1 34 | else 35 | # For debugging 36 | echo "$VAULT_STATUS" &> /proc/1/fd/1 37 | ps aux &> /proc/1/fd/1 38 | fi 39 | COUNT=$((COUNT+1)) 40 | sleep 1 41 | done 42 | -------------------------------------------------------------------------------- /k8s-executor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # KOPS_NAME and KOPS_STATE_STORE need to be set 4 | 5 | # Env variables from GOCD or defaulted for local builds 6 | GO_TRIGGER_USER="${GO_TRIGGER_USER:-"local"}" 7 | GO_PIPELINE_LABEL="${GO_PIPELINE_LABEL:-local}" 8 | GO_PIPELINE_NAME="${GO_PIPELINE_NAME:-"local"}" 9 | GO_STAGE_NAME="${GO_STAGE_NAME:-"local"}" 10 | GO_JOB_NAME="${GO_JOB_NAME:-"local"}" 11 | BRANCH_NAME="${GO_SCM_GITBRANCHES_CURRENT_BRANCH:-"local"}" 12 | SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 13 | 14 | # If running locally, use user's kubeconfig 15 | KUBE_CONFIG_MOUNT="" 16 | KUBE_CONFIG_EXPORT_COMMAND="true" 17 | if [ "$GO_TRIGGER_USER" == "local" ]; then 18 | echo "Loading kube config from local env..." 19 | KUBE_CONFIG_MOUNT="-v ${HOME}/.kube:/root/.kube" 20 | else 21 | echo "Loading kube config from AWS..." 22 | KUBE_CONFIG_EXPORT_COMMAND="kops export kubecfg --name $KOPS_NAME" 23 | fi 24 | 25 | echo "Deploying Application w/ Helm..." 26 | docker pull bartlettc/docker-kubectl 27 | docker run \ 28 | --rm \ 29 | -e "KOPS_STATE_STORE=$KOPS_STATE_STORE" \ 30 | -e "KOPS_NAME=$KOPS_NAME" \ 31 | -e "GO_PIPELINE_LABEL=$GO_PIPELINE_LABEL" \ 32 | -e "GO_TRIGGER_USER=$GO_TRIGGER_USER" \ 33 | -e "GO_PIPELINE_NAME=$GO_PIPELINE_NAME" \ 34 | -e "GO_STAGE_NAME=$GO_STAGE_NAME" \ 35 | -e "GO_JOB_NAME=$GO_JOB_NAME" \ 36 | -e "BRANCH_NAME=$BRANCH_NAME" \ 37 | $KUBE_CONFIG_MOUNT \ 38 | -v $SOURCE_DIR:/workdir \ 39 | -w /workdir \ 40 | bartlettc/docker-kubectl \ 41 | bash -c "$KUBE_CONFIG_EXPORT_COMMAND && $@" 42 | -------------------------------------------------------------------------------- /init/vault-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 2 ] 4 | then 5 | echo "Invalid arguments provided" 6 | echo "Valid usage: "`basename "$0"`" )" 7 | exit 1 8 | fi 9 | 10 | RELEASE=$1 11 | NAMESPACE=$2 12 | CHART_NAME="vault" 13 | COMPONENT="${RELEASE}-vault" 14 | ADD_SECRET=${3-"false"} 15 | 16 | SECRET_NAME="$RELEASE-vault-keys" 17 | 18 | LABELS=$(kubectl get secret -l release=$RELEASE -n $NAMESPACE --show-labels | sed -n 2p | awk '{print $5}' | sed 's/\,/ /g') 19 | FIRST_VAULT_POD=$(kubectl get po -l component=$COMPONENT,release=$RELEASE -n $NAMESPACE | awk '{if(NR==2)print $1}') 20 | INIT_MESSAGE=$(kubectl exec -n $NAMESPACE -c $RELEASE $FIRST_VAULT_POD -- sh -c "vault operator init --tls-skip-verify" 2>&1) 21 | 22 | echo "$INIT_MESSAGE" 23 | if [[ ${INIT_MESSAGE} != *"Error initializing Vault"* && "${ADD_SECRET}" == "true" ]]; then 24 | echo 25 | echo 26 | echo "Deleting existing Vault key secret" 27 | kubectl delete secret -n $NAMESPACE $SECRET_NAME --ignore-not-found=true 28 | echo "Creating Vault key secret: $SECRET_NAME" 29 | KEYS=$(echo "$INIT_MESSAGE" | grep "Unseal Key" | awk '{print $4}') 30 | CREATE_SECRET_COMMAND="kubectl create secret generic $SECRET_NAME -n $NAMESPACE " 31 | COUNT=1 32 | for i in ${KEYS[@]}; 33 | do 34 | CREATE_SECRET_COMMAND="$CREATE_SECRET_COMMAND --from-literal=key$COUNT=$i" 35 | COUNT=$((COUNT+1)) 36 | done 37 | $(echo $CREATE_SECRET_COMMAND) 38 | kubectl label secret -n $NAMESPACE \ 39 | $SECRET_NAME \ 40 | $LABELS 41 | kubectl label secret -n $NAMESPACE \ 42 | --overwrite \ 43 | $SECRET_NAME \ 44 | component="${RELEASE}-vault-init" 45 | fi 46 | -------------------------------------------------------------------------------- /helm_charts/vault/preInstallConsul/create-consul-secrets.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | HERITAGE=$(grep "heritage=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 4 | RELEASE=$(grep "release=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 5 | CHART=$(grep "chart=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 6 | COMPONENT=$(grep "component=" /etc/podinfo/labels | sed 's/^.*="\(.*\)"$/\1/') 7 | 8 | kubectl delete secrets -l release=$RELEASE,component=$COMPONENT 9 | 10 | # Create K8s Secret for Consul Gossip Key in a json format so it can be mounted 11 | # It seems that k8s doesn't like to use JSON with --from-literal= 12 | GOSSIPKEY=$(cat /dev/urandom | head -c 24 | base64; echo) 13 | echo "{\"encrypt\": \"$GOSSIPKEY\"}" > encrypt.json 14 | kubectl create secret generic \ 15 | $FULL_NAME-consul-gossip-json \ 16 | --from-file=encrypt.json 17 | rm -rf encrypt.json 18 | kubectl label secret \ 19 | $FULL_NAME-consul-gossip-json \ 20 | heritage=$HERITAGE \ 21 | release=$RELEASE \ 22 | chart=$CHART \ 23 | component=$COMPONENT 24 | 25 | # Create k8s Secret for Consul CA 26 | kubectl create secret generic \ 27 | $FULL_NAME-consul.ca \ 28 | --from-file=/certs/cert/ca.crt.pem 29 | kubectl label secret \ 30 | $FULL_NAME-consul.ca \ 31 | heritage=$HERITAGE \ 32 | release=$RELEASE \ 33 | chart=$CHART \ 34 | component=$COMPONENT 35 | 36 | # Create k8s Secret for Consul TLS Cert 37 | kubectl create secret tls \ 38 | $FULL_NAME-consul.tls \ 39 | --cert=/certs/cert/consul.crt.pem \ 40 | --key=/certs/private/consul.key.pem 41 | kubectl label secret \ 42 | $FULL_NAME-consul.tls \ 43 | heritage=$HERITAGE \ 44 | release=$RELEASE \ 45 | chart=$CHART \ 46 | component=$COMPONENT 47 | -------------------------------------------------------------------------------- /deploy.gocd.yaml: -------------------------------------------------------------------------------- 1 | pipelines: 2 | vault-deploy-test: 3 | group: vault 4 | label_template: "${gitBranches[:8]}" 5 | materials: 6 | gitBranches: 7 | scm: 527fb0d8-c691-4693-a601-20a26ddd7d41 8 | stages: 9 | - helm-lint: 10 | clean_workspace: true 11 | jobs: 12 | helm-lint: 13 | resources: 14 | - docker 15 | tasks: 16 | - exec: 17 | command: "./k8s-executor.sh" 18 | arguments: 19 | - "./tests/helm/010-lint.sh" 20 | - helm-install: 21 | clean_workspace: true 22 | jobs: 23 | helm-install: 24 | resources: 25 | - docker 26 | tasks: 27 | - exec: 28 | command: "./k8s-executor.sh" 29 | arguments: 30 | - "./tests/helm/020-deploy.sh" 31 | - vault-init-unseal: 32 | clean_workspace: true 33 | jobs: 34 | vault-init-unseal: 35 | resources: 36 | - docker 37 | tasks: 38 | - exec: 39 | command: "./k8s-executor.sh" 40 | arguments: 41 | - "./tests/helm/030-vault-init-unseal.sh" 42 | - cleanup: 43 | clean_workspace: true 44 | jobs: 45 | cleanup: 46 | resources: 47 | - docker 48 | tasks: 49 | - exec: 50 | run_if: any 51 | command: "./k8s-executor.sh" 52 | arguments: 53 | - "./tests/helm/050-cleanup.sh" 54 | -------------------------------------------------------------------------------- /helm_charts/vault/preInstallVault/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Secrets used by Vault 2 | 3 | Five secrets must be created to spin up a new vault instance 4 | 1. `gossip-key` which Consul uses to encrypt gossip communications 5 | 1. `vault-consul.ca` which contains a PEM-encoded CA cert to use to verify the Vault server SSL certificate 6 | 1. `vault-consul.tls` which contains a PEM-encoded client certificate for TLS authentication to the Vault server along with an unencrypted PEM-encoded private key matching the client certificate 7 | 1. `vault.ca` which contains a PEM-encoded CA cert to use to verify the Vault server SSL certificate 8 | 1. `vault.tls` which contains a PEM-encoded client certificate for TLS authentication to the Vault server along with an unencrypted PEM-encoded private key matching the client certificate 9 | 10 | ### Create/Update Vault Secrets 11 | 12 | 1. Generate new CA and Consul certs 13 | 1. TBD - Ryan Wilcox 14 | 1. Ensure the following files are in the `certs/` directory and are up to date 15 | 1. `consul.ca` - Consul CA Cert 16 | 1. `consul.crt` - Consul TLS Cert 17 | 1. `consul.key` - Consul TLS Private Key 18 | 1. `vault.ca` - Vault CA Cert 19 | 1. `vault.crt` - Vault TLS Cert 20 | 1. `vault.key` - Vault TLS Private Key 21 | 1. Run the following to replace all consul/vault K8S secrets 22 | ``` 23 | ./apply.sh 24 | ``` 25 | Add a `true` parameter at the end if you also want to create/replace the Consul gossip key with a new auto-generated one. **-*WARNING*-** replacing the gossip key should not be done on a running cluster as it may cause inconsistent keys between Consul nodes. This should only be done when creating or completely destroying/re-creating a Vault cluster. 26 | ``` 27 | ./apply.sh true 28 | ``` 29 | -------------------------------------------------------------------------------- /docs/0.1.2-MIGRATION.md: -------------------------------------------------------------------------------- 1 | # Migration to chart v0.1.2 2 | The breaking change with chart 0.1.2 requires a full backup and restore of Consul as the chart will destroy all existing components and recreate them with new names. Below are the steps to backup and restore Consul after the upgrade. 3 | 4 | 1. Execute into one of your Consul pods (using pod name `vault-prod-vault-consul-0` in this example). 5 | ``` 6 | kubectl exec -it vault-prod-vault-consul-0 sh 7 | ``` 8 | 2. Create a Consul snapshot (replacing `dc1` with your datacenter name, if different) 9 | ``` 10 | consul snapshot save -client-cert /consul/tls/tls.crt -client-key /consul/tls/tls.key -ca-file /consul/ca/ca.crt.pem -http-addr https://localhost:8500 -tls-server-name server.dc1.consul migration.snapshot 11 | ``` 12 | 3. Exit out of the pod and from your local machine, copy the snapshot out (change the name of the pod, if different) 13 | ``` 14 | kubectl cp vault-prod-vault-consul-0:migration.snapshot ./ 15 | ``` 16 | 4. Delete your Helm release and do a fresh installation with the new chart (this will obviously cause downtime). The Vault pods will not become ready since you're starting with an empty Consul cluster but the restore will take care of that and you'll just need to unseal it again. 17 | 5. Copy the snapshot file into a Consul pod again (note the Consul pod names will have changed) 18 | ``` 19 | kubectl cp migration.snapshot vault-prod-consul-0:/migration.snapshot 20 | ``` 21 | 6. Execute into that pod 22 | ``` 23 | kubectl exec -it vault-prod-consul-0 sh 24 | ``` 25 | 7. Restore the snapshot 26 | ``` 27 | consul snapshot restore -client-cert /consul/tls/tls.crt -client-key /consul/tls/tls.key -ca-file /consul/ca/ca.crt.pem -http-addr https://localhost:8500 -tls-server-name server.dc1.consul /migration.snapshot 28 | ``` 29 | 8. At this point the data will be restored and you can unseal your vault and be up and running. 30 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.preInstall.customCA.job.yaml: -------------------------------------------------------------------------------- 1 | {{- if and (not .Values.Vault.Tls.LetsEncrypt.Enabled) (not .Values.Vault.Tls.CertString) }} 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.PreInstall.ComponentName }}" 11 | annotations: 12 | "helm.sh/hook": pre-install 13 | "helm.sh/hook-weight": "100" 14 | spec: 15 | activeDeadlineSeconds: {{.Values.Vault.PreInstall.JobDeadline}} 16 | template: 17 | metadata: 18 | name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 19 | labels: 20 | heritage: {{ .Release.Service | quote }} 21 | release: {{ .Release.Name | quote }} 22 | chart: {{ template "vault.chart" . }} 23 | component: "{{ .Release.Name }}-{{.Values.Vault.PreInstall.ComponentName}}" 24 | spec: 25 | restartPolicy: Never 26 | containers: 27 | - name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 28 | image: "{{.Values.Misc.omgwtfssl.Image}}:{{.Values.Misc.omgwtfssl.ImageTag}}" 29 | imagePullPolicy: "Always" 30 | env: 31 | - name: CA_EXPIRE 32 | value: "60" 33 | - name: SSL_EXPIRE 34 | value: "60" 35 | - name: SSL_IP 36 | value: "127.0.0.1" 37 | - name: SSL_SUBJECT 38 | value: "{{.Values.Vault.Tls.ServerName}}" 39 | - name: SSL_DNS 40 | value: "{{.Values.Vault.Tls.AlternateServerNames}},{{- .Release.Name -}}.{{- .Release.Namespace -}}.svc.cluster.local" 41 | - name: SILENT 42 | value: "true" 43 | - name: K8S_SECRET_NAME 44 | value: "{{ template "vault.fullname" . }}.tls" 45 | - name: K8S_SECRET_SEPARATE_CA 46 | value: "true" 47 | - name: K8S_SECRET_LABELS 48 | value: "heritage={{ .Release.Service }} release={{ .Release.Name }} chart={{ template "vault.chart" . }} component={{ .Release.Name }}-{{.Values.Vault.PreInstall.ComponentName}}" 49 | {{- end }} 50 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v0.2.0 2 | * Added Pod Disruptions Budgets (PDBs) to Consul and Vault 3 | * Added optional rolling update partitioning for Consul 4 | * Added AntiAffinities for Consul and Vault 5 | * Bumped Vault to 1.1.0 and Consul to 1.4.4 6 | * Added Liveness probes to Consul and Vault 7 | * Added more configurable Readiness probe for Vault 8 | 9 | # v0.1.2 10 | 11 | ### Action Required (Breaking Change) 12 | 13 | * ACTION REQUIRED: This release contains a major change that standardizes and shortens the naming/labeling conventions of resources created with this Helm chart ([see #40](https://github.com/ReadyTalk/vault-helm-chart/issues/40)). As a result, in-place upgrades from chart v0.1.0 or v0.1.1 by simply doing a `helm upgrade` will result is many breaking issues. It may be possible to upgrade with a complex set of steps but the recommendation is that you backup your Consul cluster and do a fresh `helm install` with the new chart. Then you can restore your data with a `consul snapshot restore`. See the [0.1.2 migration guide](docs/0.1.2-MIGRATION.md) for a full set of steps. 14 | 15 | # v0.1.1 16 | 17 | ## Changelog since v0.1.0 18 | 19 | ### Major Updates 20 | * Removed third party UI in favor of the build-in OSS UI 21 | * Fixed many issues that occured in more recent Kubernetes releases 22 | 23 | ### Notable Issues Fixed 24 | * [#14](https://github.com/ReadyTalk/vault-helm-chart/issues/14) Separated restore job so that it wouldn't run on each restart of the backup pod 25 | * [#17](https://github.com/ReadyTalk/vault-helm-chart/issues/17) Added HostAlias value options 26 | * [#18](https://github.com/ReadyTalk/vault-helm-chart/issues/18) Added NodePort value options 27 | * [#20](https://github.com/ReadyTalk/vault-helm-chart/issues/20) Upgraded default Vault version to 0.9.5 28 | * [#22](https://github.com/ReadyTalk/vault-helm-chart/issues/22) Added option to use your own custom TLS certificate 29 | * [#38](https://github.com/ReadyTalk/vault-helm-chart/issues/38) Remove Consul UI options (in an effort to migrate to the official Consul helm chart) 30 | * [#36](https://github.com/ReadyTalk/vault-helm-chart/issues/36) Removed third party UI in favor of the build-in OSS UI 31 | * [#39](https://github.com/ReadyTalk/vault-helm-chart/issues/39) Removed option to enable the Consul UI 32 | * [#43](https://github.com/ReadyTalk/vault-helm-chart/issues/43) Fixed ConfigMap read-only issue for Consul 33 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.preInstall.vaultSecrets.job.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.Vault.Tls.LetsEncrypt.Enabled }} 2 | apiVersion: batch/v1 3 | kind: Job 4 | metadata: 5 | name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Vault.PreInstall.ComponentName }}" 11 | annotations: 12 | "helm.sh/hook": pre-install 13 | "helm.sh/hook-weight": "100" 14 | spec: 15 | activeDeadlineSeconds: {{.Values.Vault.PreInstall.JobDeadline}} 16 | template: 17 | metadata: 18 | name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 19 | labels: 20 | heritage: {{ .Release.Service | quote }} 21 | release: {{ .Release.Name | quote }} 22 | chart: {{ template "vault.chart" . }} 23 | component: "{{ .Release.Name }}-{{ .Values.Vault.PreInstall.ComponentName }}" 24 | spec: 25 | restartPolicy: Never 26 | containers: 27 | - name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 28 | image: "{{.Values.Misc.letsencrypt.Image}}:{{.Values.Misc.letsencrypt.ImageTag}}" 29 | imagePullPolicy: "Always" 30 | env: 31 | - name: FULL_NAME 32 | value: {{ template "vault.fullname" . }} 33 | - name: "VAULT_TLS_SERVER_NAME" 34 | value: "{{.Values.Vault.Tls.ServerName}}" 35 | - name: "VAULT_TLS_ALT_SERVER_NAMES" 36 | value: "{{.Values.Vault.Tls.AlternateServerNames}}" 37 | - name: ACME_ENVIRONMENT 38 | value: {{.Values.Vault.Tls.LetsEncrypt.Environment}} 39 | - name: ACME_ACCOUNT_KEY 40 | value: {{.Values.Vault.Tls.LetsEncrypt.AcmeAccountKey}} 41 | command: 42 | - "bash" 43 | - "-ec" 44 | - | 45 | chmod -R +x /pre-install 46 | /pre-install/generate-vault-cert.sh 47 | /pre-install/create-vault-secrets.sh 48 | volumeMounts: 49 | - name: pre-install 50 | mountPath: /pre-install 51 | - name: podinfo 52 | mountPath: /etc/podinfo 53 | readOnly: true 54 | volumes: 55 | - name: pre-install 56 | configMap: 57 | name: {{ template "vault.fullname" . }}-{{.Values.Vault.PreInstall.ComponentName}} 58 | - name: podinfo 59 | downwardAPI: 60 | items: 61 | - path: "labels" 62 | fieldRef: 63 | fieldPath: metadata.labels 64 | {{- end }} 65 | -------------------------------------------------------------------------------- /tests/helm/020-deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Generate a unique name for this build 4 | RELEASE_PREFIX="vault" 5 | UNIQUE_NAME=$RELEASE_PREFIX-$GO_PIPELINE_LABEL 6 | 7 | echo "Creating namespace: $UNIQUE_NAME..." 8 | kubectl create namespace $UNIQUE_NAME 9 | kubectl label namespace $UNIQUE_NAME \ 10 | trigger=$GO_TRIGGER_USER \ 11 | pipeline=$GO_PIPELINE_NAME \ 12 | stage-name=$GO_STAGE_NAME \ 13 | job-name=$GO_JOB_NAME \ 14 | branch-name=$BRANCH_NAME 15 | 16 | echo "Installing vault: $UNIQUE_NAME..." 17 | helm install \ 18 | --name $UNIQUE_NAME \ 19 | --namespace $UNIQUE_NAME \ 20 | ./helm_charts/vault 21 | 22 | # Wait for Consul to come up ready 23 | CONSUL_NUM_REPLICAS=$(kubectl get statefulset -l component=consul -n $UNIQUE_NAME -o json | jq .items[].spec.replicas) 24 | echo "Waiting for $CONSUL_NUM_DESIRED_PODS Consul pods to be ready..." 25 | COUNT=1 26 | while true 27 | do 28 | CONSUL_CONTAINERS_RUNNING=$(kubectl get po -l component=consul -n $UNIQUE_NAME -o json | jq '.items[].status.containerStatuses[].state | has("running")' | grep -c "true") 29 | CONSUL_CONTAINERS_READY=$(kubectl get po -l component=consul -n $UNIQUE_NAME -o json | jq .items[].status.containerStatuses[].ready | grep -c "true") 30 | if [[ $CONSUL_CONTAINERS_RUNNING -eq $CONSUL_NUM_REPLICAS && $CONSUL_CONTAINERS_READY -eq $CONSUL_NUM_REPLICAS ]]; then 31 | echo "Consul is ready!" 32 | break 33 | elif [[ $COUNT -gt 60 ]]; then 34 | echo "Consul did not come up in time!" 35 | exit 1 36 | else 37 | echo "Waiting ($CONSUL_CONTAINERS_RUNNING/$CONSUL_NUM_REPLICAS running; $CONSUL_CONTAINERS_READY/$CONSUL_NUM_REPLICAS ready)..." 38 | fi 39 | 40 | COUNT=$((COUNT+1)) 41 | sleep 1 42 | done 43 | 44 | # Wait for Vault pod to come up ready (3 pods running, 3 containers ready, 3 containers not ready) 45 | VAULT_NUM_REPLICAS=$(kubectl get deployment -l component=vault -n $UNIQUE_NAME -o json | jq .items[].spec.replicas) 46 | VAULT_NUM_CONTAINERS=$(($VAULT_NUM_REPLICAS * 2)) 47 | echo "Waiting for $VAULT_NUM_DESIRED_PODS Vault pods to be ready..." 48 | COUNT=1 49 | while true 50 | do 51 | VAULT_CONTAINERS_RUNNING=$(kubectl get po -l component=vault -n $UNIQUE_NAME -o json | jq '.items[].status.containerStatuses[].state | has("running")' | grep -c "true") 52 | VAULT_CONTAINERS_READY=$(kubectl get po -l component=vault -n $UNIQUE_NAME -o json | jq .items[].status.containerStatuses[].ready | grep -c "true") 53 | if [[ $VAULT_CONTAINERS_RUNNING -eq $VAULT_NUM_CONTAINERS && $VAULT_CONTAINERS_READY -eq $VAULT_NUM_REPLICAS ]]; then 54 | echo "Vault is ready!" 55 | break 56 | elif [[ $COUNT -gt 60 ]]; then 57 | echo "Vault did not come up in time!" 58 | exit 1 59 | else 60 | echo "Waiting ($VAULT_CONTAINERS_RUNNING/$VAULT_NUM_CONTAINERS running; $VAULT_CONTAINERS_READY/$VAULT_NUM_CONTAINERS ready)..." 61 | fi 62 | 63 | COUNT=$((COUNT+1)) 64 | sleep 1 65 | done 66 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.backup.deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.Consul.Backup.Enabled }} 2 | apiVersion: extensions/v1beta1 3 | kind: Deployment 4 | metadata: 5 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.Backup.ComponentName}}" 6 | labels: 7 | heritage: {{ .Release.Service | quote }} 8 | release: {{ .Release.Name | quote }} 9 | chart: {{ template "vault.chart" . }} 10 | component: "{{ .Release.Name }}-{{ .Values.Consul.Backup.ComponentName }}" 11 | spec: 12 | replicas: {{ .Values.Consul.Backup.Replicas }} 13 | selector: 14 | matchLabels: 15 | release: {{ .Release.Name | quote }} 16 | component: "{{ .Release.Name }}-{{ .Values.Consul.Backup.ComponentName }}" 17 | template: 18 | metadata: 19 | labels: 20 | heritage: {{ .Release.Service | quote }} 21 | release: {{ .Release.Name | quote }} 22 | chart: {{ template "vault.chart" . }} 23 | component: "{{ .Release.Name }}-{{ .Values.Consul.Backup.ComponentName }}" 24 | spec: 25 | containers: 26 | - name: "{{ template "vault.fullname" . }}-{{.Values.Consul.Backup.ComponentName}}" 27 | image: "{{.Values.Consul.Backup.Image}}:{{.Values.Consul.Backup.ImageTag}}" 28 | imagePullPolicy: {{.Values.Consul.Backup.ImagePullPolicy}} 29 | env: 30 | - name: "CONSUL_ARGS" 31 | value: -config-dir=/consul/config -config-dir=/consul/secrets 32 | - name: "S3_URL" 33 | value: {{.Values.Consul.Backup.S3URL}}/ 34 | - name: "SLEEP_DURATION" 35 | value: {{.Values.Consul.Backup.SleepDuration | quote}} 36 | resources: 37 | limits: 38 | cpu: {{ .Values.Consul.Backup.Cpu }} 39 | memory: {{ .Values.Consul.Backup.Memory }} 40 | requests: 41 | cpu: {{ .Values.Consul.Backup.Cpu }} 42 | memory: {{ .Values.Consul.Backup.Memory }} 43 | volumeMounts: 44 | - name: consul-config 45 | mountPath: /consul/config 46 | readOnly: false 47 | - name: tls-consul 48 | mountPath: /consul/tls 49 | readOnly: true 50 | - name: ca-consul 51 | mountPath: /consul/ca 52 | readOnly: true 53 | - name: gossip-key 54 | mountPath: /consul/secrets 55 | readOnly: true 56 | volumes: 57 | - name: consul-config 58 | configMap: 59 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.Backup.ComponentName}}" 60 | items: 61 | - key: config.json 62 | path: config.json 63 | - name: ca-consul 64 | secret: 65 | secretName: {{ template "vault.fullname" . }}-consul.ca 66 | - name: tls-consul 67 | secret: 68 | secretName: {{ template "vault.fullname" . }}-consul.tls 69 | - name: gossip-key 70 | secret: 71 | secretName: {{ template "vault.fullname" . }}-consul-gossip-json 72 | {{- end }} 73 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.restore.job.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Release.IsInstall }} 2 | {{- if .Values.Consul.Restore.RestoreFile }} 3 | apiVersion: batch/v1 4 | kind: Job 5 | metadata: 6 | name: {{ template "vault.fullname" . }}-{{.Values.Consul.Restore.ComponentName}} 7 | labels: 8 | heritage: {{ .Release.Service | quote }} 9 | release: {{ .Release.Name | quote }} 10 | chart: {{ template "vault.chart" . }} 11 | component: "{{ .Release.Name }}-{{ .Values.Consul.Restore.ComponentName }}" 12 | spec: 13 | activeDeadlineSeconds: 100 14 | template: 15 | metadata: 16 | name: {{ template "vault.fullname" . }}-{{.Values.Consul.Restore.ComponentName}} 17 | labels: 18 | heritage: {{ .Release.Service | quote }} 19 | release: {{ .Release.Name | quote }} 20 | chart: {{ template "vault.chart" . }} 21 | component: "{{ .Release.Name }}-{{ .Values.Consul.Restore.ComponentName }}" 22 | spec: 23 | restartPolicy: Never 24 | containers: 25 | - name: {{ template "vault.fullname" . }}-{{.Values.Consul.Restore.ComponentName}} 26 | image: "{{.Values.Consul.Backup.Image}}:{{.Values.Consul.Backup.ImageTag}}" 27 | imagePullPolicy: {{.Values.Consul.Backup.ImagePullPolicy}} 28 | env: 29 | - name: "CONSUL_ARGS" 30 | value: -config-dir=/consul/config -config-dir=/consul/secrets 31 | - name: "S3_URL" 32 | value: {{.Values.Consul.Restore.S3URL}} 33 | - name: "AWS_ACCESS_KEY_ID" 34 | value: "{{.Values.Consul.Restore.AwsAccessKeyId}}" 35 | - name: "AWS_SECRET_ACCESS_KEY" 36 | value: "{{.Values.Consul.Restore.AwsSecretAccessKey}}" 37 | command: 38 | - "/bin/sh" 39 | - "-ec" 40 | - | 41 | /init.sh {{.Values.Consul.Restore.RestoreFile}} 42 | volumeMounts: 43 | - name: consul-config 44 | mountPath: /consul/config 45 | readOnly: false 46 | - name: tls-consul 47 | mountPath: /consul/tls 48 | readOnly: true 49 | - name: ca-consul 50 | mountPath: /consul/ca 51 | readOnly: true 52 | - name: gossip-key 53 | mountPath: /consul/secrets 54 | readOnly: true 55 | volumes: 56 | - name: consul-config 57 | configMap: 58 | name: {{ template "vault.fullname" . }}-{{.Values.Consul.Restore.ComponentName}} 59 | items: 60 | - key: config.json 61 | path: config.json 62 | - name: tls 63 | secret: 64 | secretName: {{ template "vault.fullname" . }}.tls 65 | - name: ca-consul 66 | secret: 67 | secretName: {{ template "vault.fullname" . }}-consul.ca 68 | - name: tls-consul 69 | secret: 70 | secretName: {{ template "vault.fullname" . }}-consul.tls 71 | - name: gossip-key 72 | secret: 73 | secretName: {{ template "vault.fullname" . }}-consul-gossip-json 74 | {{- end }} 75 | {{- end }} 76 | -------------------------------------------------------------------------------- /helm_charts/vault/preInstallConsul/generate-consul-tls-certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # The goal of this script is to create the needed certs for Vault to Consul TLS communication 3 | # This script can be run every time there is a deployment of the helm chart 4 | # Notes: 5 | # It seems that openssl.cnf will not take ENV variables for the common name with like ${ENV::TLS_COMMON_NAME} 6 | # https://stackoverflow.com/questions/8075274/is-it-possible-making-openssl-skipping-the-country-common-name-prompts 7 | 8 | export ALTNAMECONSUL=server.$DATACENTER.consul 9 | 10 | # Having a cert signed with localhost is only used to bfuscate traffic from like sysdig tools running as privileged 11 | export ALTNAMEVAULT=localhost 12 | 13 | # These values will be used in the signed certs 14 | countryName=$TLS_COUNTRY_NAME 15 | localityName=$TLS_LOCALITY_NAME 16 | emailAddress=$TLS_EMAIL_ADDRESS 17 | organizationName=$TLS_ORGANIZATION_NAME 18 | stateOrProvinceName=$TLS_STATE_OR_PROVINCE_NAME 19 | organizationalUnitName=$TLS_ORGANIZATION_UNIT_NAME 20 | # Common name will be added per cert with variables below and extensions options from the openssl.cnf 21 | 22 | # Lets delete all created certs and start over when this script is ran 23 | mkdir -p /certs 24 | rm -rf /certs/* 25 | touch /certs/certindex 26 | touch /certs/index.txt 27 | touch /certs/index.txt.attr 28 | echo 000a > /certs/serial 29 | mkdir -p /certs/private 30 | mkdir -p /certs/cert 31 | mkdir -p /certs/csr 32 | 33 | # CA certs 34 | echo "Creating CA..." 35 | commonName='vault-ca' 36 | cmd="openssl req -config ./openssl.cnf \ 37 | -newkey rsa -days 3650 -x509 -nodes -extensions v3_ca \ 38 | -subj /C=${countryName}/ST=${stateOrProvinceName}/L=${localityName}/O=${organizationName}/OU=${organizationalUnitName}/CN=${commonName} \ 39 | -out /certs/cert/ca.crt.pem \ 40 | -keyout /certs/private/ca.key.pem" 41 | $cmd 42 | if [[ $? -ne 0 ]] ; then 43 | echo -e "\n\nERROR: CA certificate request not valid"; exit 1 44 | fi 45 | 46 | # Consul certs (client/server) 47 | echo "Creating Consul cert..." 48 | commonName='consul' 49 | cmd="openssl req -config ./openssl.cnf \ 50 | -newkey rsa -nodes \ 51 | -subj /C=${countryName}/ST=${stateOrProvinceName}/L=${localityName}/O=${organizationName}/OU=${organizationalUnitName}/CN=${commonName} \ 52 | -out /certs/csr/${commonName}.csr.pem \ 53 | -keyout /certs/private/${commonName=}.key.pem" 54 | $cmd &>/dev/null 55 | if [[ $? -ne 0 ]] ; then 56 | echo -e "\n\nERROR: Consul certificate request not valid"; exit 1 57 | fi 58 | 59 | cmd="openssl ca -batch -config ./openssl.cnf \ 60 | -notext -extensions server_cert -notext \ 61 | -in /certs/csr/${commonName}.csr.pem \ 62 | -subj /C=${countryName}/ST=${stateOrProvinceName}/L=${localityName}/O=${organizationName}/OU=${organizationalUnitName}/CN=${commonName} \ 63 | -out /certs/cert/${commonName=}.crt.pem" 64 | $cmd &>/dev/null 65 | if [[ $? -ne 0 ]] ; then 66 | echo -e "\n\nERROR: Consul certificate request not valid"; exit 1 67 | fi 68 | -------------------------------------------------------------------------------- /helm_charts/vault/preInstallConsul/openssl.cnf: -------------------------------------------------------------------------------- 1 | [ ca ] 2 | # `man ca` 3 | default_ca = CA_default 4 | 5 | [ CA_default ] 6 | # Directory and file locations. 7 | dir = /certs 8 | certs = $dir/cert 9 | crl_dir = $dir/crl 10 | new_certs_dir = $dir/cert 11 | database = $dir/index.txt 12 | serial = $dir/serial 13 | RANDFILE = $dir/private/.rand 14 | 15 | # The root key and root certificate. 16 | private_key = $dir/private/ca.key.pem 17 | certificate = $dir/cert/ca.crt.pem 18 | 19 | # SHA-1 is deprecated, so use SHA-2 instead. 20 | default_md = sha256 21 | 22 | name_opt = ca_default 23 | cert_opt = ca_default 24 | default_days = 375 # this is for the server/client certs 25 | preserve = no 26 | policy = policy_strict 27 | 28 | [ policy_strict ] 29 | # The root CA should only sign intermediate certificates that match. 30 | # See the POLICY FORMAT section of `man ca`. 31 | countryName = match 32 | stateOrProvinceName = match 33 | organizationName = match 34 | organizationalUnitName = optional 35 | commonName = supplied 36 | emailAddress = optional 37 | 38 | [ req ] 39 | # Options for the `req` tool (`man req`). 40 | default_bits = 2048 41 | distinguished_name = req_distinguished_name 42 | string_mask = utf8only 43 | 44 | # SHA-1 is deprecated, so use SHA-2 instead. 45 | default_md = sha256 46 | 47 | # Extension to add when the -x509 option is used. 48 | x509_extensions = v3_ca 49 | 50 | [ req_distinguished_name ] 51 | # See . 52 | countryName = Country Name (2 letter code) 53 | stateOrProvinceName = State or Province Name 54 | localityName = Locality Name 55 | 0.organizationName = Organization Name 56 | organizationalUnitName = Organizational Unit Name 57 | commonName = Common Name 58 | emailAddress = Email Address 59 | 60 | [ v3_ca ] 61 | # Extensions for a typical CA (`man x509v3_config`). 62 | subjectKeyIdentifier = hash 63 | authorityKeyIdentifier = keyid:always,issuer 64 | basicConstraints = critical, CA:true 65 | keyUsage = critical, digitalSignature, cRLSign, keyCertSign 66 | 67 | [ server_cert ] 68 | # Extensions for server certificates (`man x509v3_config`). 69 | basicConstraints = CA:FALSE 70 | nsCertType = server 71 | nsComment = "OpenSSL Generated Server Certificate" 72 | subjectKeyIdentifier = hash 73 | authorityKeyIdentifier = keyid,issuer:always 74 | keyUsage = critical, digitalSignature, keyEncipherment 75 | extendedKeyUsage = serverAuth, clientAuth 76 | subjectAltName = @alt_names 77 | 78 | [ alt_names ] 79 | DNS.1 = ${ENV::FULL_NAME}-consul 80 | DNS.2 = ${ENV::ALTNAMECONSUL} 81 | 82 | [ crl_ext ] 83 | # Extension for CRLs (`man x509v3_config`). 84 | authorityKeyIdentifier =keyid:always 85 | -------------------------------------------------------------------------------- /helm_charts/vault/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for Vault. 2 | # This is a YAML-formatted file. 3 | 4 | # Consul storage backend configuration 5 | Consul: 6 | ComponentName: "consul" 7 | Cpu: "150m" 8 | Datacenter: "dc1" 9 | Image: "consul" 10 | ImageTag: "1.4.4" 11 | ImagePullPolicy: "IfNotPresent" 12 | Memory: "320Mi" 13 | Replicas: 5 14 | maxUnavailable: 2 15 | updatePartition: 0 # Use to control a careful rolling update. 16 | HttpPort: 8500 17 | SerflanPort: 8301 18 | SerflanUdpPort: 8301 19 | SerfwanPort: 8302 20 | SerfwanUdpPort: 8302 21 | ServerPort: 8300 22 | ConsulDnsPort: 8600 23 | PreInstall: 24 | ComponentName: "consul-preinstall" 25 | JobDeadline: 60 26 | Tls: 27 | CountryName: "US" 28 | LocalityName: "placeholder" 29 | EmailAddress: "placeholder@placeholder.com" 30 | OrganizationName: "placeholder" 31 | StateOrProvinceName: "CO" 32 | OrganizationUnitName: "placeholder" 33 | Backup: 34 | Enabled: false 35 | ComponentName: "backup" 36 | Replicas: 1 37 | ImagePullPolicy: "Always" 38 | Image: "thorix/consul-backup" 39 | ImageTag: "latest" 40 | Cpu: "512m" 41 | Memory: "200Mi" 42 | S3URL: "" # {{ .Release.Name }} will get added to the end 43 | SleepDuration: 7200 # 2 hours 44 | 45 | Restore: 46 | ComponentName: "restore" 47 | S3URL: "" 48 | RestoreFile: "" # file name to restore from s3 49 | AwsAccessKeyId: "" 50 | AwsSecretAccessKey: "" 51 | 52 | # Vault server configuration 53 | Vault: 54 | ComponentName: "vault" 55 | AutoUnseal: false 56 | HttpPort: 8200 57 | HaForwardingPort: 8201 58 | Ingress: 59 | Enabled: true # If enabled, will use service type ClusterIP 60 | NodePort: 30825 # Ignored if Ingress.Enabled = true 61 | Image: "vault" 62 | ImageTag: "1.1.0" 63 | ImagePullPolicy: "IfNotPresent" 64 | LogLevel: "info" 65 | Replicas: 3 66 | maxUnavailable: 1 67 | Cpu: "512m" 68 | Memory: "384Mi" 69 | # HostAliases: 70 | # - ip: "127.0.0.1" 71 | # hostnames: 72 | # - "foo.local" 73 | DisableConsulRegistration: "false" 74 | DefaultLeaseTtl: "768h" 75 | MaxLeaseTtl: "768h" 76 | Readiness: 77 | readyIfStandby: true 78 | readyIfSealed: false 79 | readyIfUninitialized: true 80 | ConsulClient: 81 | ComponentName: "consul-client" 82 | PreInstall: 83 | ComponentName: "vault-preinstall" 84 | JobDeadline: 60 85 | Tls: 86 | ServerName: "vault.consul" 87 | AlternateServerNames: "vault-alt.consul" # Comma separated 88 | CertString: "" 89 | KeyString: "" 90 | LetsEncrypt: 91 | Enabled: false 92 | Environment: "stage" # production/stage 93 | AcmeAccountKey: "s3:///stage.pem" 94 | Ui: 95 | Enabled: true 96 | Misc: 97 | kubectl: 98 | Image: "bartlettc/docker-kubectl" 99 | ImageTag: "latest" 100 | omgwtfssl: 101 | Image: "bartlettc/omgwtfssl-kubernetes" 102 | ImageTag: "latest" 103 | letsencrypt: 104 | Image: "bartlettc/letsencrypt-acm" 105 | ImageTag: "kubernetes" 106 | -------------------------------------------------------------------------------- /helm_charts/vault/preInstallConsul/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Secrets used by Consul 2 | 3 | 4 | 5 | - [Overview](#overview) 6 | - [Testing TLS](#testing-tls) 7 | - [Install openssl](#install-openssl) 8 | - [Test the TLS connection without a client cert](#test-the-tls-connection-without-a-client-cert) 9 | - [Test the client cert](#test-the-client-cert) 10 | 11 | 12 | 13 | 14 | ## Overview 15 | Consul uses [two forms of encryption](https://www.consul.io/docs/agent/encryption.html) which require secrets; Gossip Encryption and RPC Encryption. These secrets are automatically created via a Kubernetes Job during the the Helm preInstall process. 16 | 17 | Three secrets are created for Consul: 18 | * `consul-gossip-key` contains the encryption key for the gossip protocol 19 | * `consul.ca` contains the self-signed CA cert 20 | * `consul.tls` contains a certificate signed by the CA that is used for client-server RPC communication between Consul agents. 21 | 22 | The Kubernetes secret names are prefixed by the Helm release name and chart name. 23 | 24 | ### Gossip Encryption 25 | `consul-gossip-key` is created by simply generating a random 24-byte string. 26 | 27 | ### RPC Encryption 28 | 29 | 30 | ## Consul RPC TLS Certificate 31 | 32 | 33 | ## Testing TLS 34 | 35 | To make sure the ports that can be accessed with only clients that have a trusted signed cert the following are notes to verify this case. 36 | 37 | ### Install openssl 38 | ```bash 39 | apk update 40 | apk add openssl 41 | ``` 42 | 43 | ### Test the TLS connection without a client cert 44 | 45 | ```bash 46 | openssl s_client -connect 127.0.0.1:8500 -CAfile /etc/consul/ca/ca.crt.pem 47 | ``` 48 | 49 | You will see the following: 50 | ```bash 51 | --- 52 | SSL handshake has read 1763 bytes and written 138 bytes 53 | --- 54 | New, TLSv1/SSLv3, Cipher is ECDHE-RSA-AES256-GCM-SHA384 55 | Server public key is 2048 bit 56 | Secure Renegotiation IS supported 57 | Compression: NONE 58 | Expansion: NONE 59 | No ALPN negotiated 60 | SSL-Session: 61 | Protocol : TLSv1.2 62 | Cipher : ECDHE-RSA-AES256-GCM-SHA384 63 | Session-ID: 64 | Session-ID-ctx: 65 | Master-Key: 15B3B6D02423D33B329824EF3B5C3493FFD697EA1CC8727968D501EAA59CD5B6DD63119AF3A529F489716617DA9D4DD7 66 | Key-Arg : None 67 | PSK identity: None 68 | PSK identity hint: None 69 | SRP username: None 70 | Start Time: 1501618926 71 | Timeout : 300 (sec) 72 | Verify return code: 0 (ok) 73 | --- 74 | ``` 75 | 76 | ### Test the client cert 77 | 78 | ```bash 79 | openssl s_client -connect 127.0.0.1:8500 -CAfile /etc/consul/ca/ca.crt.pem -cert /etc/consul/tls/tls.crt -key /etc/consul/tls/tls.key 80 | ``` 81 | 82 | When you use a working client cert you will see the following 83 | 84 | ```bash 85 | Start Time: 1501620123 86 | Timeout : 300 (sec) 87 | Verify return code: 0 (ok) 88 | --- 89 | 90 | HTTP/1.1 400 Bad Request 91 | Content-Type: text/plain; charset=utf-8 92 | Connection: close 93 | 94 | 400 Bad Requestclosed 95 | ``` 96 | 97 | Note there is alot more info passed to your screen. You will also see some http communication as well. This shows that with a good signed client cert you will get further in the client server exchange. 98 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | 3 | {{/* 4 | Expand the name of the chart. 5 | */}} 6 | {{- define "vault.name" -}} 7 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 8 | {{- end -}} 9 | 10 | {{/* 11 | Create a default fully qualified app name. 12 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 13 | If release name contains chart name it will be used as a full name. 14 | */}} 15 | {{- define "vault.fullname" -}} 16 | {{- if .Values.fullnameOverride -}} 17 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 18 | {{- else -}} 19 | {{- $name := default .Chart.Name .Values.nameOverride -}} 20 | {{- if contains $name .Release.Name -}} 21 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 22 | {{- else -}} 23 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 24 | {{- end -}} 25 | {{- end -}} 26 | {{- end -}} 27 | 28 | {{/* 29 | Create chart name and version as used by the chart label. 30 | */}} 31 | {{- define "vault.chart" -}} 32 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 33 | {{- end -}} 34 | 35 | {{/* 36 | Create a value that contains the vault -tls-skip-verify flag if we should skip TLS verification for vault 37 | This is used to skip verification when using self-signed certs or the LetsEncrypt staging environment 38 | */}} 39 | {{- define "tls_skip_verify" -}} 40 | {{ if and .Values.Vault.Tls.CertString .Values.Vault.Tls.KeyString }} 41 | {{- print "" -}} 42 | {{ else if or (not .Values.Vault.Tls.LetsEncrypt.Enabled) (eq .Values.Vault.Tls.LetsEncrypt.Environment "stage") }} 43 | {{- print "--tls-skip-verify" -}} 44 | {{ else }} 45 | {{- print "" -}} 46 | {{ end }} 47 | {{- end -}} 48 | 49 | {{/* 50 | Compute the maximum number of unavailable Vault replicas for the PodDisruptionBudget. 51 | This defaults to (n/2)-1 where n is the number of members of the server cluster. 52 | Special case of replica equaling 3 and allowing a minor disruption of 1 otherwise 53 | use the integer value 54 | Add a special case for replicas=1, where it should default to 0 as well. 55 | */}} 56 | {{- define "vault.pdb.maxUnavailable" -}} 57 | {{- if eq (int .Values.Vault.Replicas) 1 -}} 58 | {{ 0 }} 59 | {{- else if .Values.Vault.maxUnavailable -}} 60 | {{ .Values.Vault.maxUnavailable -}} 61 | {{- else -}} 62 | {{- if eq (int .Values.Vault.Replicas) 3 -}} 63 | {{- 1 -}} 64 | {{- else -}} 65 | {{- sub (div (int .Values.Vault.Replicas) 2) 1 -}} 66 | {{- end -}} 67 | {{- end -}} 68 | {{- end -}} 69 | 70 | {{/* 71 | Compute the maximum number of unavailable Consul replicas for the PodDisruptionBudget. 72 | This defaults to (n/2)-1 where n is the number of members of the server cluster. 73 | Special case of replica equaling 3 and allowing a minor disruption of 1 otherwise 74 | use the integer value 75 | Add a special case for replicas=1, where it should default to 0 as well. 76 | */}} 77 | {{- define "consul.pdb.maxUnavailable" -}} 78 | {{- if eq (int .Values.Consul.Replicas) 1 -}} 79 | {{ 0 }} 80 | {{- else if .Values.Consul.maxUnavailable -}} 81 | {{ .Values.Consul.maxUnavailable -}} 82 | {{- else -}} 83 | {{- if eq (int .Values.Consul.Replicas) 3 -}} 84 | {{- 1 -}} 85 | {{- else -}} 86 | {{- sub (div (int .Values.Consul.Replicas) 2) 1 -}} 87 | {{- end -}} 88 | {{- end -}} 89 | {{- end -}} -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.preInstall.job.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.PreInstall.ComponentName}}" 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{.Values.Consul.PreInstall.ComponentName}}" 10 | annotations: 11 | # This is what defines this resource as a hook. Without this line, the 12 | # job is considered part of the release. 13 | "helm.sh/hook": pre-install 14 | "helm.sh/hook-weight": "100" 15 | spec: 16 | activeDeadlineSeconds: {{.Values.Consul.PreInstall.JobDeadline}} 17 | template: 18 | metadata: 19 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.PreInstall.ComponentName}}" 20 | labels: 21 | heritage: {{ .Release.Service | quote }} 22 | release: {{ .Release.Name | quote }} 23 | chart: {{ template "vault.chart" . }} 24 | component: "{{ .Release.Name }}-{{.Values.Consul.PreInstall.ComponentName}}" 25 | spec: 26 | restartPolicy: Never 27 | initContainers: 28 | - name: copy-ro-scripts 29 | image: busybox 30 | command: ['sh', '-c', 'cp /scripts/* /etc/pre-install/'] 31 | volumeMounts: 32 | - name: scripts 33 | mountPath: /scripts 34 | - name: pre-install 35 | mountPath: /etc/pre-install 36 | containers: 37 | - name: "{{ template "vault.fullname" . }}-{{.Values.Consul.PreInstall.ComponentName}}" 38 | image: "{{.Values.Misc.kubectl.Image}}:{{.Values.Misc.kubectl.ImageTag}}" 39 | imagePullPolicy: "Always" 40 | env: 41 | - name: FULL_NAME 42 | value: {{ template "vault.fullname" . }} 43 | - name: DATACENTER 44 | value: {{.Values.Consul.Datacenter}} 45 | - name: TLS_COUNTRY_NAME 46 | value: {{.Values.Consul.PreInstall.Tls.CountryName}} 47 | - name: TLS_LOCALITY_NAME 48 | value: {{.Values.Consul.PreInstall.Tls.LocalityName}} 49 | - name: TLS_EMAIL_ADDRESS 50 | value: {{.Values.Consul.PreInstall.Tls.EmailAddress}} 51 | - name: TLS_ORGANIZATION_NAME 52 | value: {{.Values.Consul.PreInstall.Tls.OrganizationName}} 53 | - name: TLS_STATE_OR_PROVINCE_NAME 54 | value: {{.Values.Consul.PreInstall.Tls.StateOrProvinceName}} 55 | - name: TLS_ORGANIZATION_UNIT_NAME 56 | value: {{.Values.Consul.PreInstall.Tls.OrganizationUnitName}} 57 | command: 58 | - "bash" 59 | - "-ec" 60 | - | 61 | chmod -R +x /pre-install 62 | cd /pre-install 63 | ./generate-consul-tls-certs.sh 64 | ./create-consul-secrets.sh 65 | volumeMounts: 66 | - name: pre-install 67 | mountPath: /pre-install 68 | - name: podinfo 69 | mountPath: /etc/podinfo 70 | readOnly: true 71 | volumes: 72 | - name: pre-install 73 | emptyDir: {} 74 | - name: scripts 75 | configMap: 76 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.PreInstall.ComponentName}}" 77 | - name: podinfo 78 | downwardAPI: 79 | items: 80 | - path: "labels" 81 | fieldRef: 82 | fieldPath: metadata.labels 83 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/consul.statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: StatefulSet 3 | metadata: 4 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{ .Values.Consul.ComponentName }}" 10 | spec: 11 | serviceName: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 12 | podManagementPolicy: Parallel 13 | updateStrategy: 14 | {{- if (gt (int .Values.Consul.updatePartition) 0) }} 15 | type: RollingUpdate 16 | rollingUpdate: 17 | partition: {{ .Values.Consul.updatePartition }} 18 | {{- else -}} 19 | type: "OnDelete" 20 | {{- end }} 21 | replicas: {{ .Values.Consul.Replicas }} 22 | selector: 23 | matchLabels: 24 | release: {{ .Release.Name | quote }} 25 | component: "{{ .Release.Name }}-{{ .Values.Consul.ComponentName }}" 26 | template: 27 | metadata: 28 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 29 | labels: 30 | heritage: {{ .Release.Service | quote }} 31 | release: {{ .Release.Name | quote }} 32 | chart: {{ template "vault.chart" . }} 33 | component: "{{ .Release.Name }}-{{ .Values.Consul.ComponentName }}" 34 | spec: 35 | podAntiAffinity: 36 | requiredDuringSchedulingIgnoredDuringExecution: 37 | - labelSelector: 38 | matchLabels: 39 | release: {{ .Release.Name | quote }} 40 | component: "{{ .Release.Name }}-{{ .Values.Consul.ComponentName }}" 41 | topologyKey: kubernetes.io/hostname 42 | terminationGracePeriodSeconds: 10 43 | securityContext: 44 | fsGroup: 1000 45 | containers: 46 | - name: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 47 | image: "{{.Values.Consul.Image}}:{{.Values.Consul.ImageTag}}" 48 | imagePullPolicy: "{{.Values.Consul.ImagePullPolicy}}" 49 | ports: 50 | - name: http 51 | containerPort: {{.Values.Consul.HttpPort}} 52 | - name: serflan-tcp 53 | protocol: "TCP" 54 | containerPort: {{.Values.Consul.SerflanPort}} 55 | - name: serflan-udp 56 | protocol: "UDP" 57 | containerPort: {{.Values.Consul.SerflanUdpPort}} 58 | - name: serfwan-tcp 59 | protocol: "TCP" 60 | containerPort: {{.Values.Consul.SerfwanPort}} 61 | - name: serfwan-udp 62 | protocol: "UDP" 63 | containerPort: {{.Values.Consul.SerfwanUdpPort}} 64 | - name: server 65 | containerPort: {{.Values.Consul.ServerPort}} 66 | - name: consuldns 67 | containerPort: {{.Values.Consul.ConsulDnsPort}} 68 | resources: 69 | requests: 70 | cpu: "{{.Values.Consul.Cpu}}" 71 | memory: "{{.Values.Consul.Memory}}" 72 | env: 73 | - name: INITIAL_CLUSTER_SIZE 74 | value: {{ .Values.Consul.Replicas | quote }} 75 | - name: STATEFULSET_NAME 76 | value: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 77 | - name: POD_IP 78 | valueFrom: 79 | fieldRef: 80 | fieldPath: status.podIP 81 | - name: STATEFULSET_NAMESPACE 82 | valueFrom: 83 | fieldRef: 84 | fieldPath: metadata.namespace 85 | volumeMounts: 86 | - name: config 87 | mountPath: /consul/config-ro 88 | - name: gossip-json 89 | mountPath: /consul/secrets 90 | readOnly: true 91 | - name: tls 92 | mountPath: /consul/tls 93 | readOnly: true 94 | - name: ca 95 | mountPath: /consul/ca 96 | readOnly: true 97 | livenessProbe: 98 | exec: 99 | command: 100 | - consul 101 | - members 102 | - "-ca-file=/consul/ca/ca.crt.pem" 103 | - "-client-cert=/consul/tls/tls.crt" 104 | - "-client-key=/consul/tls/tls.key" 105 | - "-http-addr=https://127.0.0.1:{{.Values.Consul.HttpPort}}" 106 | - "-tls-server-name={{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 107 | initialDelaySeconds: 300 108 | timeoutSeconds: 5 109 | command: 110 | - "/bin/sh" 111 | - "-ec" 112 | - | 113 | cp /consul/config-ro/config.json /consul/config/config.json 114 | IP=$(hostname -i) 115 | 116 | for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do 117 | while true; do 118 | echo "Waiting for ${STATEFULSET_NAME}-${i}.${STATEFULSET_NAME} to come up" 119 | ping -W 1 -c 1 ${STATEFULSET_NAME}-${i}.${STATEFULSET_NAME}.${STATEFULSET_NAMESPACE}.svc.cluster.local > /dev/null && break 120 | sleep 1s 121 | done 122 | done 123 | 124 | PEERS="" 125 | for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do 126 | PEERS="${PEERS}${PEERS:+ } -retry-join $(ping -c 1 ${STATEFULSET_NAME}-${i}.${STATEFULSET_NAME}.${STATEFULSET_NAMESPACE}.svc.cluster.local | awk -F'[()]' '/PING/{print $2}')" 127 | done 128 | 129 | docker-entrypoint.sh \ 130 | consul \ 131 | agent \ 132 | -config-dir=/consul/config \ 133 | -config-dir=/consul/secrets \ 134 | -bootstrap-expect=${INITIAL_CLUSTER_SIZE} \ 135 | -advertise=${IP} \ 136 | ${PEERS} 137 | volumes: 138 | - name: config 139 | configMap: 140 | name: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 141 | items: 142 | - key: config.json 143 | path: config.json 144 | - name: gossip-json 145 | secret: 146 | secretName: {{ template "vault.fullname" . }}-consul-gossip-json 147 | - name: tls 148 | secret: 149 | secretName: {{ template "vault.fullname" . }}-consul.tls 150 | - name: ca 151 | secret: 152 | secretName: {{ template "vault.fullname" . }}-consul.ca 153 | -------------------------------------------------------------------------------- /helm_charts/vault/templates/vault.deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: {{ template "vault.fullname" . }} 5 | labels: 6 | heritage: {{ .Release.Service | quote }} 7 | release: {{ .Release.Name | quote }} 8 | chart: {{ template "vault.chart" . }} 9 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 10 | spec: 11 | replicas: {{ .Values.Vault.Replicas }} 12 | strategy: 13 | type: RollingUpdate 14 | rollingUpdate: 15 | maxUnavailable: 1 16 | selector: 17 | matchLabels: 18 | release: {{ .Release.Name | quote }} 19 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 20 | template: 21 | metadata: 22 | labels: 23 | heritage: {{ .Release.Service | quote }} 24 | release: {{ .Release.Name | quote }} 25 | chart: {{ template "vault.chart" . }} 26 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 27 | spec: 28 | {{- if .Values.Vault.HostAliases }} 29 | hostAliases: 30 | {{- range .Values.Vault.HostAliases }} 31 | - ip: {{ .ip }} 32 | hostnames: 33 | {{- range .hostnames }} 34 | - {{ . }} 35 | {{- end }} 36 | {{- end }} 37 | {{- end }} 38 | containers: 39 | - name: {{ template "vault.fullname" . }} 40 | image: "{{.Values.Vault.Image}}:{{.Values.Vault.ImageTag}}" 41 | imagePullPolicy: {{.Values.Vault.ImagePullPolicy}} 42 | command: 43 | - "/bin/sh" 44 | - "-ec" 45 | - | 46 | cp /vault/config-ro/config.json /vault/config/config.json 47 | docker-entrypoint.sh server 48 | env: 49 | - name: "VAULT_CACERT" 50 | value: "/vault/tls/ca.crt" 51 | - name: "VAULT_LOG_LEVEL" 52 | value: "{{.Values.Vault.LogLevel}}" 53 | - name: "VAULT_TLS_SERVER_NAME" 54 | value: "{{.Values.Vault.Tls.ServerName}}" 55 | {{- if .Values.Vault.AutoUnseal }} 56 | lifecycle: 57 | postStart: 58 | exec: 59 | command: 60 | - "/bin/sh" 61 | - "-ec" 62 | - | 63 | cp -R /post-start-ro /post-start 64 | chmod -R +x /post-start 65 | /post-start/unseal-vault.sh {{ template "tls_skip_verify" . }} 66 | {{- end }} 67 | livenessProbe: 68 | exec: 69 | command: 70 | - pidof 71 | - vault 72 | initialDelaySeconds: 5 73 | timeoutSeconds: 2 74 | readinessProbe: 75 | # Ready depends on preference 76 | httpGet: 77 | scheme: HTTPS 78 | path: /v1/sys/health? 79 | {{- if .Values.Vault.Readiness.readyIfStandby -}}standbycode=204&{{- end }} 80 | {{- if .Values.Vault.Readiness.readyIfSealed -}}sealedcode=204&{{- end }} 81 | {{- if .Values.Vault.Readiness.readyIfUninitialized -}}uninitcode=204&{{- end }} 82 | port: {{.Values.Vault.HttpPort}} 83 | initialDelaySeconds: 5 84 | periodSeconds: 5 85 | ports: 86 | - name: https 87 | containerPort: {{.Values.Vault.HttpPort}} 88 | - name: ha-forwarding 89 | containerPort: {{.Values.Vault.HaForwardingPort}} 90 | volumeMounts: 91 | - name: config 92 | mountPath: /vault/config-ro 93 | - name: tls 94 | mountPath: /vault/tls 95 | - name: ca-consul 96 | mountPath: /etc/vault/consul-ca 97 | {{- if .Values.Vault.AutoUnseal }} 98 | - name: vault-keys 99 | mountPath: /etc/vault/keys 100 | - name: post-start 101 | mountPath: /post-start-ro 102 | {{- end }} 103 | - name: consul-unix-socket 104 | mountPath: /consul-unix-socket 105 | {{- if .Values.Vault.HostAliases }} 106 | - name: nsswitch 107 | mountPath: /etc/nsswitch.conf 108 | subPath: nsswitch.conf 109 | {{- end }} 110 | resources: 111 | limits: 112 | cpu: {{ .Values.Vault.Cpu }} 113 | memory: {{ .Values.Vault.Memory }} 114 | requests: 115 | cpu: {{ .Values.Vault.Cpu }} 116 | memory: {{ .Values.Vault.Memory }} 117 | securityContext: 118 | capabilities: 119 | add: 120 | - IPC_LOCK 121 | - name: "{{ template "vault.fullname" . }}-{{.Values.Vault.ConsulClient.ComponentName}}" 122 | image: "{{.Values.Consul.Image}}:{{.Values.Consul.ImageTag}}" 123 | imagePullPolicy: "{{.Values.Consul.ImagePullPolicy}}" 124 | ports: 125 | - name: http 126 | containerPort: {{.Values.Consul.HttpPort}} 127 | - name: serflan-tcp 128 | protocol: "TCP" 129 | containerPort: {{.Values.Consul.SerflanPort}} 130 | - name: serflan-udp 131 | protocol: "UDP" 132 | containerPort: {{.Values.Consul.SerflanUdpPort}} 133 | - name: serfwan-tcp 134 | protocol: "TCP" 135 | containerPort: {{.Values.Consul.SerfwanPort}} 136 | - name: serfwan-udp 137 | protocol: "UDP" 138 | containerPort: {{.Values.Consul.SerfwanUdpPort}} 139 | - name: server 140 | containerPort: {{.Values.Consul.ServerPort}} 141 | - name: consuldns 142 | containerPort: {{.Values.Consul.ConsulDnsPort}} 143 | env: 144 | - name: INITIAL_CLUSTER_SIZE 145 | value: {{ .Values.Consul.Replicas | quote }} 146 | - name: STATEFULSET_NAME 147 | value: "{{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}}" 148 | - name: STATEFULSET_NAMESPACE 149 | valueFrom: 150 | fieldRef: 151 | fieldPath: metadata.namespace 152 | volumeMounts: 153 | - name: consul-config 154 | mountPath: /consul/config-ro 155 | - name: tls-consul 156 | mountPath: /consul/tls 157 | readOnly: true 158 | - name: ca-consul 159 | mountPath: /consul/ca 160 | readOnly: true 161 | - name: gossip-json 162 | mountPath: /consul/secrets 163 | readOnly: true 164 | - name: consul-unix-socket 165 | mountPath: /consul-unix-socket 166 | command: 167 | - "/bin/sh" 168 | - "-ec" 169 | - | 170 | cp /consul/config-ro/config.json /consul/config/config.json 171 | docker-entrypoint.sh \ 172 | consul \ 173 | agent \ 174 | -config-dir=/consul/config \ 175 | -config-dir=/consul/secrets \ 176 | -retry-join={{ template "vault.fullname" . }}-{{.Values.Consul.ComponentName}} 177 | livenessProbe: 178 | exec: 179 | command: 180 | - pidof 181 | - consul 182 | initialDelaySeconds: 5 183 | timeoutSeconds: 2 184 | podAntiAffinity: 185 | preferredDuringSchedulingIgnoredDuringExecution: 186 | - weight: 100 187 | podAffinityTerm: 188 | topologyKey: kubernetes.io/hostname 189 | labelSelector: 190 | matchLabels: 191 | release: {{ .Release.Name | quote }} 192 | component: "{{ .Release.Name }}-{{ .Values.Vault.ComponentName }}" 193 | volumes: 194 | - name: config 195 | configMap: 196 | name: {{ template "vault.fullname" . }} 197 | items: 198 | - key: config.json 199 | path: config.json 200 | - name: tls 201 | secret: 202 | secretName: {{ template "vault.fullname" . }}.tls 203 | - name: ca-consul 204 | secret: 205 | secretName: {{ template "vault.fullname" . }}-consul.ca 206 | - name: tls-consul 207 | secret: 208 | secretName: {{ template "vault.fullname" . }}-consul.tls 209 | {{- if .Values.Vault.AutoUnseal }} 210 | - name: vault-keys 211 | secret: 212 | secretName: {{ template "vault.fullname" . }}-vault-keys 213 | - name: post-start 214 | configMap: 215 | name: {{ template "vault.fullname" . }}-post-start 216 | {{- end }} 217 | - name: consul-config 218 | configMap: 219 | name: "{{ template "vault.fullname" . }}-{{.Values.Vault.ConsulClient.ComponentName}}" 220 | - name: gossip-json 221 | secret: 222 | secretName: {{ template "vault.fullname" . }}-consul-gossip-json 223 | - name: consul-unix-socket 224 | emptyDir: {} 225 | {{- if .Values.Vault.HostAliases }} 226 | - name: nsswitch 227 | configMap: 228 | name: "{{ template "vault.fullname" . }}" 229 | items: 230 | - key: nsswitch.conf 231 | path: nsswitch.conf 232 | {{- end }} 233 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vault 2 | 3 | > **Breaking Changes**: New releases may contain breaking changes to this chart as it evolves. Please read the [CHANGELOG](CHANGELOG.md) prior to upgrading. 4 | 5 | This project sets up [Vault by Hashicorp](https://www.vaultproject.io/) on Kubernetes using Helm in a HA configuration. It create its own private Consul backend and secures the Consul and Vault traffic. Also, optionally, a Consul UI and Vault UI can be enabled. 6 | 7 | It isn't hard to [get started](https://www.vaultproject.io/intro/getting-started/install.html) with Vault. There is also [charts](https://github.com/kubernetes/charts/tree/master/stable/consul) that will get Consul running in K8S. AWS has a solution if you want to use [CloudFormation templates](https://aws.amazon.com/quickstart/architecture/vault/). Putting this all together yourself in a secure way turns out to be much harder. This projects takes on a lot of the tasks that would normally be manual. From unsealing Vault to TLS creation for client and backend communication. Also automatically backing up the Consul data to AWS S3. The default setup will create 5 Consul replicas and 3 Vault replicas. 8 | 9 | ##### Tech stack 10 | [Vault](https://www.vaultproject.io/), [Consul](https://www.consul.io/), [K8S](https://kubernetes.io/), [Helm](https://github.com/kubernetes/helm), [AWS S3](https://aws.amazon.com/s3/), Bash, OpenSSL, [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/), [Let's Encrypt](https://letsencrypt.org/docs/) 11 | 12 | ### Contents 13 | 14 | 15 | - [Installing Vault](#installing-vault) 16 | - [Upgrading Vault](#upgrading-vault) 17 | - [Testing Vault](#testing-vault) 18 | - [Concepts (Work in progress)](#concepts-work-in-progress) 19 | - [Security](#security) 20 | - [Backup and recovery](#backup-and-recovery) 21 | - [Deleting Vault](#deleting-vault) 22 | - [Future Work](#future-work) 23 | 24 | 25 | 26 | ## Installing Vault 27 | 28 | ### Prerequisites 29 | * Kubernetes 1.5.x 30 | * Helm 2.5.x 31 | * LetsEncrypt Account and Route53 access (optional) 32 | * Amazon S3 access (optiona, for backups) 33 | 34 | ### Configuration 35 | 36 | | Parameter | Description | Default | 37 | | ----------------------- | ---------------------------------- | ---------------------------------------------------------- | 38 | | `Consul.ComponentName` | Used for resource names and labeling | `consul` | 39 | | `Consul.Cpu` | Consul container requested cpu | `100m` | 40 | | `Consul.Datacenter` | Consul datacenter name | `dc1` | 41 | | `Consul.Image` | Consul container image name | `consul` | 42 | | `Consul.ImageTag` | Consul container image tag | `0.9.2` | 43 | | `Consul.ImagePullPolicy` | Consul container pull policy | `IfNotPresent` | 44 | | `Consul.Memory` | Consul container requested memory | `256Mi` | 45 | | `Consul.Replicas` | Consul container replicas | `5` | 46 | | `Consul.maxUnavailable` | Consul container PodDisruptionBudget maxUnavailable instances | `2` | 47 | | `Consul.updatePartition` | Consul container updatePartition number for Phased Roll Outs | `0` | 48 | | `Consul.HttpPort` | Consul http listening port | `8500` | 49 | | `Consul.SerflanPort` | Consul serf lan listening port | `8301` | 50 | | `Consul.SerflanUdpPort` | Consul serf lan UDP listening port | `8301` | 51 | | `Consul.SerfwanPort` | Consul serf wan listening port | `8302` | 52 | | `Consul.SerfwanUdpPort` | Consul serf wan UDP listening port | `8302` | 53 | | `Consul.ServerPort` | Consul server listening port | `8300` | 54 | | `Consul.ConsulDnsPort` | Consul DNS listening port | `8600` | 55 | | `Consul.PreInstall.ComponentName` | Used for resource names and labeling | `consul-preinstall` | 56 | | `Consul.PreInstall.JobDeadline` | Timeout for Consul pre-install job (seconds) | `30` | 57 | | `Consul.PreInstall.Tls.CountryName` | TLS cert country name | `US` | 58 | | `Consul.PreInstall.Tls.LocalityName` | TLS cert locality name | `placeholder` | 59 | | `Consul.PreInstall.Tls.EmailAddress` | TLS cert email address | `placeholder@placeholder.com` | 60 | | `Consul.PreInstall.Tls.OrganizationName` | TLS cert organization name | `placeholder` | 61 | | `Consul.PreInstall.Tls.StateOrProvinceName` | TLS cert state | `CO` | 62 | | `Consul.PreInstall.Tls.OrganizationUnitName` | TLS cert organization unit name | `placeholder` | 63 | | `Consul.Backup.Enabled` | Consul backups enabled | `false` | 64 | | `Consul.Backup.ComponentName` | Used for resource names and labeling | `backup` | 65 | | `Consul.Backup.Replicas` | Consul backup container replicas | `1` | 66 | | `Consul.Backup.ImagePullPolicy` | Consul backup container pull policy | `Always` | 67 | | `Consul.Backup.Image` | Consul backup container image name | `thorix/consul-backup` | 68 | | `Consul.Backup.ImageTag` | Consul backup container image tag | `latest` | 69 | | `Consul.Backup.Cpu` | Consul backup requested cpu | `512m` | 70 | | `Consul.Backup.Memory` | Consul backup container requested memory | `200Mi` | 71 | | `Consul.Backup.S3URL` | Consul backups S3 bucket path, e.g `s3://my-bucket` (helm `.Release.Name` will get added to the end) | | 72 | | `Consul.Backup.SleepDuration` | Backup interval (in seconds) | `7200` | 73 | | `Consul.Restore.ComponentName` | Used for resource names and labeling | `restore` | 74 | | `Consul.Restore.S3URL` | Full restore S3 bucket path, e.g. `s3://my-bucket/vault-a/` | | 75 | | `Consul.Restore.RestoreFile` | Filename in s3 to restore from, this field triggers the restore option | | 76 | | `Consul.Restore.AwsAccessKeyId` | AWS key with access to the restore s3 location | | 77 | | `Consul.Restore.AwsSecretAccessKey` | AWS secret key with access to the restore s3 location | | 78 | | `Vault.ComponentName` | Used for resource names and labeling | `vault` | 79 | | `Vault.AutoUnseal` | Vault auto-unsealing (deprecated) | `false` | 80 | | `Vault.HttpPort` | Vault http listening port | `8200` | 81 | | `Vault.HaForwardingPort` | Vault high-availability port-forwarding port | `8201` | 82 | | `Vault.Ingress.Enabled` | Enable ingress. If enabled, will use service type ClusterIP | `true` | 83 | | `Vault.NodePort` | Vault service NodePort to open. Ignored if Ingress.Enabled = true | `30825` | 84 | | `Vault.Image` | Vault container image name | `vault` | 85 | | `Vault.ImageTag` | Vault container image tag | `0.9.0` | 86 | | `Vault.ImagePullPolicy` | Vault container pull policy | `IfNotPresent` 87 | | `Vault.LogLevel` | Set vault log level (trace, debug, info, etc.) | `info` | 88 | | `Vault.Replicas` | Vault container replicas | `3` | 89 | | `Vault.maxUnavailable` | Vault container PodDisruptionBudget maxUnavailable instances | `1` | 90 | | `Vault.Readiness.readyIfStandby` | Mark Vault pods as ready to start accepting traffic when `Standby` status is reached | `true` | 91 | | `Vault.Readiness.readyIfSealed` | Mark Vault pods as ready to start accepting traffic when `Sealed` status is reached | `false` | 92 | | `Vault.Readiness.readyIfUninitialized` | Mark Vault pods as ready to start accepting traffic when `Uninitialized` status is reached | `true` | 93 | | `Vault.Cpu` | Vault container requested cpu | `512m` | 94 | | `Vault.Memory` | Vault container requested memory | `200Mi` | 95 | | `Vault.HostAliases` | List of hostfile entries, see [docs](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) for details and format | | 96 | | `Vault.DisableConsulRegistration` | [Disable Vault registration within Consul ](https://www.vaultproject.io/docs/configuration/storage/consul.html#disable_registration) | `false` | 97 | | `Vault.DefaultLeaseTtl` | Default lease TTL for Vault tokens | `768h` | 98 | | `Vault.MaxLeaseTtl` | Max lease TTL for Vault tokens | `768h` | 99 | | `Vault.ConsulClient.ComponentName` | Used for resource names and labeling | `consul-client` | 100 | | `Vault.PreInstall.ComponentName` | Used for resource names and labeling | `vault-preinstall` | 101 | | `Vault.PreInstall.JobDeadline` | Timeout for Vault pre-install job(s) (seconds) | `100` 102 | | `Vault.Tls.ServerName` | Vault TLS/ingress hostname | `vault.consul` | 103 | | `Vault.Tls.AlternateServerNames` | Vault TLS/ingress alternative hostnames (comma separated) | `vault-alt.consul` | 104 | | `Vault.Tls.CertString` | Provide a TLS certificate file to use in Vault's listener | | 105 | | `Vault.Tls.KeyString` | Provide a TLS certificate key file to use in Vault's listener | | 106 | | `Vault.Tls.LetsEncrypt.Enabled` | Enable LetsEncrypt support | `false` | 107 | | `Vault.Tls.LetsEncrypt.Environment` | LetsEncrypt environment (production/stage) | `stage` | 108 | | `Vault.Tls.LetsEncrypt.AcmeAccountKey` | LetsEncrypt registration key location (s3 bucket), e.g. `s3://my-bucket/stage.pem` | | 109 | | `Vault.Ui.Enabled` | Enable Vault UI | `false` | 110 | 111 | Additional dependencies 112 | 113 | | Parameter | Description | Default | 114 | | ----------------------- | ---------------------------------- | ---------------------------------------------------------- | 115 | | `Misc.kubectl.Image` | Consul Preinstall Job container image name | `bartlettc/docker-kubectl` | 116 | | `Misc.kubectl.ImageTag` | Consul Preinstall Job container image tag | `latest` | 117 | | `Misc.omgwtfssl.Image` | Vault Preinstall Job container image name | `bartlettc/omgwtfssl-kubernetes` | 118 | | `Misc.omgwtfssl.ImageTag` | Vault Preinstall Job container image tag | `latest` | 119 | | `Misc.letsencrypt.Image` | Let's encrypt container image name | `bartlettc/letsencrypt-acm` | 120 | | `Misc.letsencrypt.ImageTag` | Let's encrypt container image tag | `kubernetes` | 121 | 122 | ### Helm Install 123 | 124 | Checkout the repo: 125 | ```bash 126 | git clone https://github.com/ReadyTalk/vault-helm-chart.git 127 | cd vault-helm-chart 128 | ``` 129 | 130 | To install the chart with the release name `vault` in namespace `vault` with the default configuration (see above for values definitions): 131 | 132 | ```bash 133 | helm install --name vault --namespace vault helm_charts/vault 134 | ``` 135 | 136 | > Note: By default, this chart will install with a self-signed certificate and therefore will require you to pass in the `-tls-skip-verify` to your vault commands. To pass in your own valid certificate, use the `Vault.Tls` options (see above). 137 | 138 | It will take a minute or two for the pre-installation steps to run. Then Helm will finish installing all the necessary resources. 139 | 140 | To check the status run `kubectl get po -l release=vault-prod,component=vault -n vault` and all pods should have a `Running` status. 141 | 142 | > **Notes on pre-install steps**: This Helm chart has pre-install jobs that create certain Kubernetes secrets before the rest of the application get set up. See the READMEs in [helm_charts/vault/init/preInstallConsul](helm_charts/vault/preInstallConsul) and [helm_charts/vault/init/preInstallVault](helm_charts/vault/preInstallVault) for more information on what is going on here. If there is a failure while running the install, it could be due to the pre-install scripts failing. To look at the pre-install resources: 143 | ``` 144 | kubectl get po,jobs,cm,secrets -l 'component in (consul-preinstall,vault-preinstall),release=vault-prod' -n vault --show-all 145 | ``` 146 | 147 | ### Initialize Vault 148 | If this is a new instance of Vault, it needs to be initialized. To do this, run 149 | 150 | ``` 151 | ./init/vault-init.sh vault-prod vault 152 | ``` 153 | 154 | This script does 3 things: 155 | * Initializes Vault 156 | * Displays the Vault Unseal Keys and Root Token 157 | * Saves the Vault Unseal Keys as a Kubernetes secret 158 | 159 | > **IMPORTANT**: You must save off the Unseal Keys and Root Token as this is the only time they will be shown. The Unseal Keys are needed unseal vault in the case of failure and the Root Token is needed to auth with Vault for admin tasks. 160 | 161 | ### Unseal Vault 162 | 163 | If this is a new instance of Vault, it will need to be unsealed manually. Any pod restarts in the future will auto-unseal. 164 | 165 | To manually unseal the vault, run 166 | 167 | ``` 168 | ./init/vault-unseal.sh vault-prod vault 169 | ``` 170 | 171 | ## Upgrading Vault 172 | To upgrade vault with new configuration values, run 173 | 174 | ``` 175 | helm upgrade --values=helm_charts/vault/values-prod.yaml vault-prod helm_charts/vault/ 176 | ``` 177 | 178 | > **Note when upgrading** 179 | Certain components cannot be upgraded simply with a `helm upgrade`, primarily, anything that was created with the helm pre-installation process. This includes 180 | * Consul TLS certificates 181 | * Consul Gossip Keys 182 | * LetsEncrypt TLS certificates 183 | 184 | >Also if the Consul stateful set it being modified, it must be updated with a rolling restart *-process TBD--*. 185 | 186 | ## Renewing Vault Certificate (TLS Secret) 187 | 188 | * Step 1: Connect to the correct kubernetes cluster 189 | * Step 2: Generate new certificate using appropriate tool 190 | * Step 3: Combine new certificate and intermediate certificate, in that order, into bundle file 191 | * Step 4: Update kubernetes tls secret 192 | 193 | ```bash 194 | # one liner to update kubernetes tls secret (cert and key): 195 | kubectl create secret tls \ 196 | --cert="/path/to/combined/cert.bundle" \ 197 | --key="/path/to/private.key" \ 198 | .tls --dry-run -o yaml | kubectl apply -f - 199 | # deploy by destroying one vault pod at a time 200 | kubectl delete po/ 201 | # monitor for changes to certificate date as pods recreate 202 | watch -n1 "echo | openssl s_client -connect vault.example.com:8200 2>/dev/null | openssl x509 -noout -dates" 203 | ``` 204 | 205 | 206 | ## Testing Vault 207 | 208 | Installation Variations 209 | * CA Variation 210 | ** Self-signed CA 211 | ** LetsEncrypt Stage CA 212 | * UI test 213 | 214 | Upgrade Variations 215 | * Upgraded from last minor release 216 | 217 | Tests 218 | * Consul Members 219 | * Vault Read 220 | * Vault Write 221 | * Auto Unseal 222 | * Backup and Restore 223 | 224 | 225 | ## Concepts (Work in progress) 226 | * Multiple Vaults in 1 namespace 227 | * Letsencrypt 228 | * Ingress Controller 229 | * Auto Unsealing 230 | * Readiness 231 | 232 | ## Security 233 | 234 | With this setup, all network communication is encrypted with TLS. See the figure below for details. 235 | ![alt text](vault-security-model.png "Security Model") 236 | 237 | ## Backup and recovery 238 | 239 | This Helm chart has a built-in backup/restore option which can be used to take snapshots of the Vault's encrypted backend data (from Consul) and restore it in the case of a full Consul failure. 240 | 241 | ### Backing up Vault 242 | 243 | Backups can be enabled and configured with the `Consul.Backup.*` option values. Backups work by taking a periodic [snapshot of Consul](https://www.consul.io/docs/commands/snapshot.html) and shipping those snapshots to an S3 bucket. 244 | 245 | > **Note**: At this time AWS S3 access for backups is a assumed with AWS Roles given to the K8S cluster nodes. Future work might be done to pull from vault the needed access for it's own backups. 246 | 247 | ### Restoring from Backup 248 | 249 | > **Warning**: The restore process will create an exact restore of your Vault instance at the point of the snapshot. This means that any leases that have expired since the backup was taken will be immediately revoked. It will have no knowledge of lease renewals that happened after the snapshot and therefore could unexpectedly revoke external leases (AWS, MySQL, etc). Take extreme caution when performing a restore from a production backup! 250 | 251 | > **Note**: When doing a restore, ensure that the Consul and Vault versions are compatible with version used to create the snapshot file. 252 | 253 | A restore can be configured with the `Consul.Restore.*` option values. The restore process uses a Kubernetes job to perform a [`consul snapshot restore`](https://www.consul.io/docs/commands/snapshot/restore.html) based on a provided snapshot file stored in AWS S3. A set of AWS credentials are required to be passed in to permform this action. With all the restore variables set `helm install` will pull down the snapshot and apply that to the cluster. The following is an example of how you could install vault using a backup file: 254 | 255 | > **Important Notes**: The restore process will only work on a fresh install of the helm chart. Doing a `helm upgrade` will not envoke the restore option(s). This is a design feature to prevent restoring on top of a working Consul cluster. 256 | 257 | For the restore option, specific AWS credentials with access to s3 must be provided. 258 | 259 | ```bash 260 | helm install \ 261 | --values=values-prod.yaml \ 262 | --namespace=vault \ 263 | --name=vault-prod \ 264 | --set Consul.Restore.S3URL=s3://vault-backups/prod/ \ 265 | --set Consul.Restore.RestoreFile=consul-20180101.171653.snap \ 266 | --set Consul.Restore.AwsAccessKeyId=xxxxxxxxxx \ 267 | --set Consul.Restore.AwsSecretAccessKey=xxxxxxxxxx \ 268 | vault 269 | ``` 270 | 271 | ## Deleting Vault 272 | 273 | To delete the chart: 274 | ```bash 275 | helm delete vault-prod --purge 276 | ``` 277 | 278 | Also, due to the way helm does pre-installs and the way the vault/consul secrets are generated, you'll need to clean up these extra resources by running the following command: 279 | ```bash 280 | kubectl delete po,jobs,cm,secrets -l 'component in (consul-preinstall,vault-preinstall,unseal-keys),release=vault-prod' -n vault 281 | ``` 282 | 283 | Lastly, the ingress controller creates a configmap that needs to be cleaned up as well. 284 | ``` 285 | kubectl delete cm ingress-controller-leader-nginx -n vault 286 | ``` 287 | ## Troubleshooting 288 | 289 | ### Querying Consul 290 | 291 | ``` 292 | consul members -client-cert /consul/tls/tls.crt -client-key /consul/tls/tls.key -ca-file /consul/ca/ca.crt.pem -http-addr https://localhost:8500 -tls-server-name server..consul 293 | ``` 294 | 295 | ## Future Work 296 | 297 | ### Cleanup 298 | * Add Helm default vaules 299 | * Document Helm values 300 | 301 | ### Security 302 | * Secure Consul UI 303 | ** Access to the consul ui should be restricted via some kind of auth or, at a minimum, set to read-only 304 | 305 | ### High Availability 306 | * Updating LetsEncrypt Certs 307 | * Updating Consul certs via custom CA 308 | ** The Consul server/client certs, as well as the CA, will need to be rotated out periodically. Need to figure out how to do this automatically with no downtime 309 | * Add node affinity 310 | ** Ensure that consul and vault pods are spaced out amongst all the kubernetes nodes to ensure failure in one AZ doesn't disrupt service 311 | --------------------------------------------------------------------------------