├── setup └── .keep ├── ci ├── oss-tasks │ ├── extract-tile.sh │ ├── run-errand.sh │ ├── get-version-from-plugin.sh │ ├── update-pipeline.sh │ ├── generate-pcf-manifest.sh │ ├── populate-or-rotate-vault.sh │ └── generate-manifest.sh ├── tasks │ ├── extract-tile.sh │ ├── run-errand.sh │ ├── populate-vault-deployment-properties.sh │ ├── get-version-from-plugin.sh │ ├── update-pipeline.sh │ ├── populate-or-rotate-vault.sh │ ├── generate-manifest.sh │ └── generate-pcf-manifest.sh ├── opensource-pipeline.yml └── pcf-pipeline.yml ├── .gitignore ├── samples ├── setup-vault-sample.sh ├── deployment-props-sample.json ├── oss-pipeline-vars-template.yml └── pcf-pipeline-vars-template.yml ├── init.sh ├── README.md └── LICENSE /setup/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ci/oss-tasks/extract-tile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | for TILE in $PRODUCT_DIR/*.pivotal; do 4 | unzip -d $OUTPUT_DIR $TILE 5 | done 6 | 7 | #eof 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.DS_Store 2 | **/pipeline-vars.yml 3 | **/deployment-props.json 4 | oss-pipeline-vars.yml 5 | **/deployment-net.json 6 | **/setup-vault.sh 7 | setup/ 8 | !.keep 9 | -------------------------------------------------------------------------------- /ci/tasks/extract-tile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | for TILE in $PRODUCT_DIR/*.pivotal; do 4 | unzip -d $OUTPUT_DIR $TILE 5 | done 6 | 7 | echo "we dont want the consul release" 8 | rm -fr ${OUTPUT_DIR}/releases/consul* 9 | 10 | #eof 11 | -------------------------------------------------------------------------------- /ci/tasks/run-errand.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | echo -e "$BOSH_CACERT" > ca.cert 4 | 5 | bosh --ca-cert ca.cert -n target $BOSH_TARGET 6 | bosh -n download manifest $BOSH_DEPLOYMENT_NAME $BOSH_DEPLOYMENT_NAME.yml 7 | bosh -n deployment $BOSH_DEPLOYMENT_NAME.yml 8 | bosh -n run errand $BOSH_ERRAND 9 | 10 | #eof 11 | -------------------------------------------------------------------------------- /ci/oss-tasks/run-errand.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | echo -e "$BOSH_CACERT" > ca.cert 4 | 5 | bosh --ca-cert ca.cert -n target $BOSH_TARGET 6 | bosh -n download manifest $BOSH_DEPLOYMENT_NAME $BOSH_DEPLOYMENT_NAME.yml 7 | bosh -n deployment $BOSH_DEPLOYMENT_NAME.yml 8 | bosh -n run errand $BOSH_ERRAND 9 | 10 | #eof 11 | -------------------------------------------------------------------------------- /samples/setup-vault-sample.sh: -------------------------------------------------------------------------------- 1 | export VAULT_ADDR=http://1.0.0.4:8200 2 | export VAULT_TOKEN=12345678989 3 | export VAULT_HASH=secret/cf-nonprod-props 4 | echo "requires files (rootCA.pem, director.pwd, deployment-props.json)" 5 | vault write ${VAULT_HASH} \ 6 | bosh-cacert=@rootCA.pem \ 7 | bosh-client-secret=@director.pwd \ 8 | bosh-pass=@director.pwd @deployment-props.json 9 | 10 | vault read ${VAULT_HASH} 11 | -------------------------------------------------------------------------------- /ci/tasks/populate-vault-deployment-properties.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # initializes values for vault from a given deployment-props.json file 4 | # 5 | # 6 | 7 | echo "${VAULT_PROPERTIES_JSON}" > deployment-props.json 8 | echo "${BOSH_CACERT}" > rootCA.pem 9 | 10 | echo "requires files (rootCA.pem, director.pwd, deployment-props.json)" 11 | vault write ${VAULT_HASH_MISC} \ 12 | bosh-cacert=@rootCA.pem \ 13 | bosh-client-secret="${BOSH_CLIENT_SECRET}" \ 14 | bosh-pass="${BOSH_CLIENT_SECRET}" @deployment-props.json 15 | 16 | vault read ${VAULT_HASH_MISC} 17 | 18 | 19 | #eof 20 | -------------------------------------------------------------------------------- /ci/tasks/get-version-from-plugin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | chmod +x omg-cli/omg-linux 4 | 5 | omg-cli/omg-linux register-plugin \ 6 | -type product \ 7 | -pluginpath ert-plugin/$PRODUCT_PLUGIN 8 | 9 | PRODUCT_NAME=$(printf ${PRODUCT_PLUGIN%%-*}) 10 | 11 | omg-cli/omg-linux product-meta $PRODUCT_NAME | \ 12 | awk '/^pivotal-/ {printf $NF}' > $OUTPUT_DIR/product_version 13 | 14 | omg-cli/omg-linux product-meta $PRODUCT_NAME | \ 15 | awk '/^stemcell:/ {printf $NF}' > $OUTPUT_DIR/stemcell_version 16 | 17 | printf "Pivotal $PRODUCT_NAME version: $(<$OUTPUT_DIR/product_version)\n" 18 | printf "Stemcell version: $(<$OUTPUT_DIR/stemcell_version)\n" 19 | 20 | #eof 21 | -------------------------------------------------------------------------------- /ci/oss-tasks/get-version-from-plugin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | chmod +x omg-cli/omg-linux 4 | 5 | omg-cli/omg-linux register-plugin \ 6 | -type product \ 7 | -pluginpath omg-product-bundle/$PRODUCT_PLUGIN 8 | 9 | PRODUCT_NAME=$(printf ${PRODUCT_PLUGIN%%-*}) 10 | 11 | omg-cli/omg-linux product-meta $PRODUCT_NAME | \ 12 | awk '/^pivotal-/ {printf $NF}' > $OUTPUT_DIR/product_version 13 | 14 | omg-cli/omg-linux product-meta $PRODUCT_NAME | \ 15 | awk '/^stemcell:/ {printf $NF}' > $OUTPUT_DIR/stemcell_version 16 | 17 | printf "Pivotal $PRODUCT_NAME version: $(<$OUTPUT_DIR/product_version)\n" 18 | printf "Stemcell version: $(<$OUTPUT_DIR/stemcell_version)\n" 19 | 20 | #eof 21 | -------------------------------------------------------------------------------- /ci/tasks/update-pipeline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | curl -SsL -u "$CONCOURSE_USER:$CONCOURSE_PASS" "$CONCOURSE_URL/api/v1/cli?arch=amd64&platform=linux" > fly 4 | chmod +x fly 5 | 6 | ./fly -t here login -c "$CONCOURSE_URL" -u "$CONCOURSE_USER" -p "$CONCOURSE_PASS" 7 | ./fly -t here get-pipeline -p $PIPELINE_NAME > $PIPELINE_NAME.yml 8 | 9 | sed -i.original " 10 | 11 | /^resources:$/,/^resource_types:$/ { 12 | 13 | /^- name: $PRODUCT$/,/product_version:/ { 14 | s,\(product_version:\).*,\1 $( fly 4 | chmod +x fly 5 | 6 | ./fly -t here login -c "$CONCOURSE_URL" -u "$CONCOURSE_USER" -p "$CONCOURSE_PASS" 7 | ./fly -t here get-pipeline -p $PIPELINE_NAME > $PIPELINE_NAME.yml 8 | 9 | sed -i.original " 10 | 11 | /^resources:$/,/^resource_types:$/ { 12 | 13 | /^- name: $PRODUCT$/,/product_version:/ { 14 | s,\(product_version:\).*,\1 $( manifest/deployment.yml 34 | 35 | #eof 36 | -------------------------------------------------------------------------------- /ci/oss-tasks/populate-or-rotate-vault.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Create or rotate certificates and passwords/preshared keys 4 | # in the $VAULT_HASH_KEYCERT and $VAULT_HASH_PASSWORD vault 5 | # hashes. $SYSTEM_DOMAIN is required for certificate generation. 6 | 7 | chmod +x omg-cli/omg-linux 8 | 9 | omg-cli/omg-linux register-plugin \ 10 | -type product \ 11 | -pluginpath omg-product-bundle/$PRODUCT_PLUGIN 12 | 13 | if [[ $SKIP_HAPROXY == "false" ]]; then 14 | HAPROXY_FLAG="--skip-haproxy=false" 15 | fi 16 | 17 | omg-cli/omg-linux deploy-product \ 18 | --bosh-url $(vault read -field=bosh-url $VAULT_HASH_MISC) \ 19 | --bosh-port $(vault read -field=bosh-port $VAULT_HASH_MISC) \ 20 | --bosh-user $(vault read -field=bosh-user $VAULT_HASH_MISC) \ 21 | --bosh-pass $(vault read -field=bosh-pass $VAULT_HASH_MISC) \ 22 | --print-manifest \ 23 | --ssl-ignore \ 24 | $PRODUCT_PLUGIN \ 25 | $HAPROXY_FLAG \ 26 | --infer-from-cloud \ 27 | --system-domain $SYSTEM_DOMAIN \ 28 | --vault-active \ 29 | --vault-domain $VAULT_ADDR \ 30 | --vault-hash-host $VAULT_HASH_HOSTVARS \ 31 | --vault-hash-ip $VAULT_HASH_IP \ 32 | --vault-hash-keycert $VAULT_HASH_KEYCERT \ 33 | --vault-hash-misc $VAULT_HASH_MISC \ 34 | --vault-hash-password $VAULT_HASH_PASSWORD \ 35 | --vault-rotate \ 36 | --vault-token $VAULT_TOKEN > throw-away-manifest.yml 37 | 38 | #eof 39 | -------------------------------------------------------------------------------- /init.sh: -------------------------------------------------------------------------------- 1 | if [ ! -f setup/deployment-props.json ]; then 2 | echo "Creating setup/deployment-props.json" 3 | cp samples/deployment-props-sample.json setup/deployment-props.json 4 | fi 5 | 6 | if [ ! -f setup/setup-vault.sh ]; then 7 | echo "Creating setup/setup-vault.sh" 8 | cp samples/setup-vault-sample.sh setup/setup-vault.sh 9 | fi 10 | 11 | if [ ! -f setup/oss-pipeline-vars.yml ]; then 12 | echo "Creating setup/oss-pipeline-vars.yml" 13 | cp samples/oss-pipeline-vars-template.yml setup/oss-pipeline-vars.yml 14 | fi 15 | 16 | if [ ! -f setup/pcf-pipeline-vars.yml ]; then 17 | echo "Creating setup/pcf-pipeline-vars.yml" 18 | cp samples/pcf-pipeline-vars-template.yml setup/pcf-pipeline-vars.yml 19 | fi 20 | echo "!!!!!!!!!!!!! BEFORE PROCEEDING !!!!!!!!!!!!!!!!!" 21 | echo "PLEASE MODIFY THE VALUES IN THE FILES IN the 'setup' directory TO MATCH YOUR SYSTEM" 22 | echo 23 | echo 24 | echo "to seed vault with your desired values please run:" 25 | echo "./setup/setup-vault.sh" 26 | echo 27 | echo 28 | echo "to upload the oss pipeline please run:" 29 | echo "$> fly -t CF-Concourse set-pipeline -p deploy-oss-cloudfoundry -c ci/opensource-pipeline.yml -l setup/oss-pipeline-vars.yml" 30 | echo 31 | echo "to upload the pcf pipeline please run:" 32 | echo "$> fly -t CF-Concourse set-pipeline -p deploy-pct-ert -c ci/pcf-pipeline.yml -l setup/pcf-pipeline-vars.yml" 33 | -------------------------------------------------------------------------------- /ci/tasks/populate-or-rotate-vault.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Create or rotate certificates and passwords/preshared keys 4 | # in the $VAULT_HASH_KEYCERT and $VAULT_HASH_PASSWORD vault 5 | # hashes. $SYSTEM_DOMAIN is required for certificate generation. 6 | 7 | chmod +x omg-cli/omg-linux 8 | 9 | omg-cli/omg-linux register-plugin \ 10 | -type product \ 11 | -pluginpath ert-plugin/$PRODUCT_PLUGIN 12 | 13 | if [[ $SKIP_HAPROXY == "false" ]]; then 14 | HAPROXY_FLAG="--skip-haproxy=false" 15 | fi 16 | 17 | omg-cli/omg-linux deploy-product \ 18 | --bosh-url $(vault read -field=bosh-url $VAULT_HASH_MISC) \ 19 | --bosh-port $(vault read -field=bosh-port $VAULT_HASH_MISC) \ 20 | --bosh-user $(vault read -field=bosh-user $VAULT_HASH_MISC) \ 21 | --bosh-pass $(vault read -field=bosh-pass $VAULT_HASH_MISC) \ 22 | --print-manifest \ 23 | --ssl-ignore \ 24 | $PRODUCT_PLUGIN \ 25 | $HAPROXY_FLAG \ 26 | --infer-from-cloud \ 27 | --stemcell-version $STEMCELL_VERSION \ 28 | --system-domain $SYSTEM_DOMAIN \ 29 | --vault-active \ 30 | --vault-domain $VAULT_ADDR \ 31 | --vault-hash-host $VAULT_HASH_HOSTVARS \ 32 | --vault-hash-ip $VAULT_HASH_IP \ 33 | --vault-hash-keycert $VAULT_HASH_KEYCERT \ 34 | --vault-hash-misc $VAULT_HASH_MISC \ 35 | --vault-hash-password $VAULT_HASH_PASSWORD \ 36 | --vault-rotate \ 37 | --vault-token $VAULT_TOKEN > throw-away-manifest.yml 38 | 39 | #eof 40 | -------------------------------------------------------------------------------- /ci/oss-tasks/generate-manifest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | chmod +x omg-cli/omg-linux 4 | 5 | omg-cli/omg-linux register-plugin \ 6 | -type product \ 7 | -pluginpath omg-product-bundle/$PRODUCT_PLUGIN 8 | 9 | if [[ $SKIP_HAPROXY == "false" ]]; then 10 | HAPROXY_FLAG="--skip-haproxy=false" 11 | fi 12 | 13 | omg-cli/omg-linux deploy-product \ 14 | --bosh-url $(vault read -field=bosh-url $VAULT_HASH_MISC) \ 15 | --bosh-port $(vault read -field=bosh-port $VAULT_HASH_MISC) \ 16 | --bosh-user $(vault read -field=bosh-user $VAULT_HASH_MISC) \ 17 | --bosh-pass $(vault read -field=bosh-pass $VAULT_HASH_MISC) \ 18 | --print-manifest \ 19 | --ssl-ignore \ 20 | $PRODUCT_PLUGIN \ 21 | --cf-mysql-release-version $( manifest/deployment.yml 37 | 38 | #eof 39 | -------------------------------------------------------------------------------- /ci/tasks/generate-manifest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | chmod +x omg-cli/omg-linux 4 | 5 | omg-cli/omg-linux register-plugin \ 6 | -type product \ 7 | -pluginpath ert-plugin/$PRODUCT_PLUGIN 8 | 9 | if [[ $SKIP_HAPROXY == "false" ]]; then 10 | HAPROXY_FLAG="--skip-haproxy=false" 11 | fi 12 | 13 | omg-cli/omg-linux deploy-product \ 14 | --bosh-url $(vault read -field=bosh-url $VAULT_HASH_MISC) \ 15 | --bosh-port $(vault read -field=bosh-port $VAULT_HASH_MISC) \ 16 | --bosh-user $(vault read -field=bosh-user $VAULT_HASH_MISC) \ 17 | --bosh-pass $(vault read -field=bosh-pass $VAULT_HASH_MISC) \ 18 | --print-manifest \ 19 | --ssl-ignore \ 20 | $PRODUCT_PLUGIN \ 21 | --cf-mysql-release-version $( manifest/deployment.yml 37 | 38 | #eof 39 | -------------------------------------------------------------------------------- /ci/tasks/generate-pcf-manifest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | chmod +x omg-cli/omg-linux 4 | 5 | omg-cli/omg-linux register-plugin \ 6 | -type product \ 7 | -pluginpath ert-plugin/$PRODUCT_PLUGIN 8 | 9 | UAA_LDAP_ENABLED=$(vault read -field=uaa-ldap-enabled $VAULT_HASH_MISC) 10 | ALLOW_APP_SSH_ACCESS=$(vault read -field=allow-app-ssh-access $VAULT_HASH_MISC) 11 | SKIP_HAPROXY=$(vault read -field=skip-haproxy $VAULT_HASH_MISC) 12 | 13 | if [[ $UAA_LDAP_ENABLED == "true" ]]; then 14 | UAA_LDAP_ENABLED_FLAG="--uaa-ldap-enabled=true" 15 | fi 16 | 17 | if [[ $ALLOW_APP_SSH_ACCESS == "true" ]]; then 18 | SSH_FLAG="--allow-app-ssh-access=true" 19 | fi 20 | 21 | if [[ $SKIP_HAPROXY == "false" ]]; then 22 | HAPROXY_FLAG="--skip-haproxy=false" 23 | fi 24 | 25 | omg-cli/omg-linux deploy-product \ 26 | --bosh-url $(vault read -field=bosh-url $VAULT_HASH_MISC) \ 27 | --bosh-port $(vault read -field=bosh-port $VAULT_HASH_MISC) \ 28 | --bosh-user $(vault read -field=bosh-user $VAULT_HASH_MISC) \ 29 | --bosh-pass $(vault read -field=bosh-pass $VAULT_HASH_MISC) \ 30 | --print-manifest \ 31 | --ssl-ignore \ 32 | $PRODUCT_PLUGIN \ 33 | $SSH_FLAG \ 34 | $HAPROXY_FLAG \ 35 | $UAA_LDAP_ENABLED_FLAG \ 36 | --infer-from-cloud \ 37 | --stemcell-version $STEMCELL_VERSION \ 38 | --vault-active \ 39 | --vault-domain $VAULT_ADDR \ 40 | --vault-hash-host $VAULT_HASH_HOSTVARS \ 41 | --vault-hash-ip $VAULT_HASH_IP \ 42 | --vault-hash-keycert $VAULT_HASH_KEYCERT \ 43 | --vault-hash-misc $VAULT_HASH_MISC \ 44 | --vault-hash-password $VAULT_HASH_PASSWORD \ 45 | --vault-token $VAULT_TOKEN > manifest/deployment.yml 46 | 47 | #eof 48 | -------------------------------------------------------------------------------- /samples/deployment-props-sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "app-domain": "", 3 | "bosh-port": "25555", 4 | "bosh-url": "https://x.x.x.x", 5 | "bosh-user": "director", 6 | "cc-vm-type": "large", 7 | "cc-worker-vm-type": "large", 8 | "clock-global-vm-type": "large", 9 | "consul-ip": "", 10 | "consul-vm-type": "large", 11 | "diego-brain-disk-type": "51200", 12 | "diego-brain-ip": "", 13 | "diego-brain-vm-type": "large", 14 | "diego-cell-disk-type": "51200", 15 | "diego-cell-ip": "", 16 | "diego-cell-vm-type": "large", 17 | "diego-db-ip": "", 18 | "diego-db-vm-type": "large", 19 | "deployment-name": "cf-nonprod", 20 | "doppler-ip": "", 21 | "doppler-vm-type": "large", 22 | "errand-vm-type": "large", 23 | "etcd-machine-ip": "", 24 | "etcd-vm-type": "large", 25 | "haproxy-vm-type": "large", 26 | "loggregator-traffic-controller-ip": "", 27 | "loggregator-traffic-controller-vmtype": "large", 28 | "mysql-disk-type": "51200", 29 | "mysql-ip": "", 30 | "mysql-proxy-ip": "", 31 | "mysql-proxy-vm-type": "large", 32 | "mysql-vm-type": "large", 33 | "nats-machine-ip": "", 34 | "nats-vm-type": "large", 35 | "nfs-allow-from-network-cidr": "", 36 | "nfs-disk-type": "51200", 37 | "nfs-ip": "", 38 | "nfs-vm-type": "large", 39 | "router-ip": "", 40 | "router-vm-type": "large", 41 | "system-domain": "", 42 | "uaa-vm-type": "large", 43 | "uaa-ldap-enabled": "false", 44 | "uaa-ldap-url": "ldap://", 45 | "uaa-ldap-user-dn": "", 46 | "uaa-ldap-search-base": "", 47 | "uaa-ldap-search-filter": "cn={0}", 48 | "uaa-ldap-mail-attributename": "mail", 49 | "skip-haproxy": "false", 50 | "allow-app-ssh-access": "true" 51 | } 52 | -------------------------------------------------------------------------------- /samples/oss-pipeline-vars-template.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copy this file to pipeline-vars.yml, add appropriate values, and supply 3 | # to fly when creating or updating the concourse pipeline. 4 | # 5 | # For example: 6 | # 7 | # fly -t TARGET set-pipeline \ 8 | # -p deploy-pcf \ 9 | # -c ci/pcf-pipeline.yml \ 10 | # -l pipeline-vars.yml 11 | # 12 | # Note pipeline-vars.yml is in .gitignore to help avoid checking sensitive 13 | # material into github. 14 | # 15 | # Notes on BOSH authentication: 16 | # omg and the Concourse bosh-deployment resource use two different types 17 | # of authentication mechanisms: 18 | # 19 | # 1. bosh-user and bosh-pass below below are for omg so it can authenticate 20 | # to bosh and read the cloud-config when generating the manifest. 21 | # 2. bosh-client-id and bosh-client-secret is for the Concourse bosh-deployment 22 | # resource so it can authenticate to do the deployment. 23 | # 24 | bosh-cacert: | 25 | -----BEGIN CERTIFICATE----- 26 | ... 27 | -----END CERTIFICATE----- 28 | bosh-client-id: director 29 | bosh-client-secret: BOSH_SHARED_SECRET 30 | bosh-pass: PASSWORD 31 | bosh-url: https://10.0.0.10 32 | bosh-user: director 33 | app-domain: apps.1.2.3.4.xip.io 34 | system-domain: sys.1.2.3.4.xip.io 35 | deployment-name: oss-cf-nonprod 36 | product-plugin: cloudfoundry-plugin-linux 37 | skip-haproxy: false 38 | stemcell-name: bosh-vsphere-esxi-ubuntu-trusty-go_agent 39 | stemcell-os: ubuntu-trusty 40 | vault-addr: http://10.0.0.8:8200 41 | vault-hash-hostvars: secret/cf-nonprod-hostvars 42 | vault-hash-ip: secret/cf-nonprod-ip 43 | vault-hash-keycert: secret/cf-nonprod-keycert 44 | vault-hash-misc: secret/cf-nonprod-props 45 | vault-hash-password: secret/cf-nonprod-password 46 | vault-token: VAULT_TOKEN 47 | slack-url: https://hooks.slack.com/services/XXXXXXXXX/XXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX 48 | slack-channel: my-channel 49 | slack-username: whoami 50 | slack-icon-url: img.com 51 | -------------------------------------------------------------------------------- /samples/pcf-pipeline-vars-template.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copy this file to pipeline-vars.yml, add appropriate values, and supply 3 | # to fly when creating or updating the concourse pipeline. 4 | # 5 | # For example: 6 | # 7 | # fly -t TARGET set-pipeline \ 8 | # -p deploy-pcf \ 9 | # -c ci/pcf-pipeline.yml \ 10 | # -l pipeline-vars.yml 11 | # 12 | # Note pipeline-vars.yml is in .gitignore to help avoid checking sensitive 13 | # material into github. 14 | # 15 | # Notes on BOSH authentication: 16 | # omg and the Concourse bosh-deployment resource use two different types 17 | # of authentication mechanisms: 18 | # 19 | # 1. bosh-user and bosh-pass below below are for omg so it can authenticate 20 | # to bosh and read the cloud-config when generating the manifest. 21 | # 2. bosh-client-id and bosh-client-secret is for the Concourse bosh-deployment 22 | # resource so it can authenticate to do the deployment. 23 | # 24 | bosh-cacert: | 25 | -----BEGIN CERTIFICATE----- 26 | ... 27 | -----END CERTIFICATE----- 28 | bosh-client-id: BOSH_CLIENT_ID_WITH_client_credentials_GRANT_TYPE 29 | bosh-client-secret: BOSH_SHARED_SECRET 30 | bosh-pass: PASSWORD 31 | bosh-url: https://11.1.1.11 32 | bosh-user: admin 33 | app-domain: apps.11.111.111.11.xip.io 34 | system-domain: sys.11.111.111.11.xip.io 35 | concourse-url: http://11.1.1.11:8080 36 | concourse-user: USERNAME 37 | concourse-pass: PASSWORD 38 | deployment-name: cf-nonprod 39 | product-slug: elastic-runtime 40 | product-version: 1.8.21 41 | product-plugin: ert-1-8-linux 42 | pivnet-api-token: API_TOKEN 43 | skip-haproxy: false 44 | stemcell-cpi-glob: '*vsphere*' 45 | stemcell-version: 3263.12 46 | vault-addr: http://11.1.1.111:8200 47 | vault-hash-hostvars: secret/cf-nonprod-hostvars 48 | vault-hash-ip: secret/cf-nonprod-ip 49 | vault-hash-keycert: secret/cf-nonprod-keycert 50 | vault-hash-misc: secret/cf-nonprod-props 51 | vault-hash-password: secret/cf-nonprod-password 52 | vault-token: 11111111-1111-1111-1111-111111111111 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # concourse-deploy-cloudfoundry 2 | 3 | Deploy Cloud Foundry with [omg](https://github.com/enaml-ops) in a Concourse pipeline. 4 | 5 | ## Prerequisites 6 | 7 | 1. [Git](https://git-scm.com) 8 | 1. [Vault](https://www.vaultproject.io) 9 | 1. [Concourse](http://concourse.ci) 10 | 11 | ## Steps to use this pipeline 12 | 13 | 1. Clone this repository. 14 | 15 | ``` 16 | git clone https://github.com/enaml-ops/concourse-deploy-cloudfoundry.git 17 | ``` 18 | 19 | 1. Initialize the repo for your deployment. 20 | 21 | ``` 22 | cd concourse-deploy-cloudfoundry 23 | ./init.sh 24 | ``` 25 | 26 | 1. Edit `setup/deployment-props.json`, adding the appropriate values. 27 | 28 | This file is used to populate a `vault` hash. It holds the BOSH credentials for both `omg` (username/password) and the Concourse `bosh-deployment` (UAA client) resource. 29 | 30 | ``` 31 | $EDITOR deployment-props.json 32 | ``` 33 | 34 | `omg` will also read other key/value pairs added here, matching them to command-line arguments. For example, to add the `omg` plugin parameter `--syslog-address`, you could add `"syslog-address": "10.150.12.10"` here rather than modifying the manifest generation script in `ci/tasks`. 35 | 36 | All available parameters/keys can be listed by querying the plugin. If not specified in `deployment-props.json`, default values will be used where possible. 37 | 38 | 39 | ``` 40 | omg-linux deploy-product ert-1-8-linux --help 41 | ``` 42 | 43 | 1. Edit `setup/(oss|pcf)-pipeline-vars.yml` 44 | These files are the open source and pcf equivalents of each other. Choose 45 | which type of deployment you would like and complete the values for your 46 | environment 47 | 48 | 1. Create or update the pipeline, either opensource or PCF. 49 | 50 | ``` 51 | fly -t CF-Concourse set-pipeline -p deploy-oss-cloudfoundry -c ci/opensource-pipeline.yml -l setup/oss-pipeline-vars.yml --var "vault-json-string=$(cat setup/deployment-props.json)" 52 | fly -t CF-Concourse unpause-pipeline -p deploy-oss-cloudfoundry 53 | ``` 54 | 55 | _or_ 56 | 57 | ``` 58 | fly -t CF-Concourse set-pipeline -p deploy-pcf-ert-1.8 -c ci/pcf-pipeline.yml -l setup/pcf-pipeline-vars.yml --var "vault-json-string=$(cat setup/deployment-props.json)" 59 | fly -t CF-Concourse unpause-pipeline -p deploy-pcf-ert-1.8 60 | fly -t CF-Concourse trigger-job -j deploy-pcf-ert-1.8/load-vault-properties -w 61 | ``` 62 | 63 | 1. Delete or move `setup/*` to a secure location. 64 | These files are gitignored, but might contain sensitive information and 65 | great care should be taken in how/where/if these are kept 66 | 67 | 1. Trigger the deployment job and observe the output. 68 | 69 | ``` 70 | fly -t TARGET trigger-job -j deploy-pcf/deploy -w 71 | ``` 72 | 73 | -------------------------------------------------------------------------------- /ci/opensource-pipeline.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: all 3 | jobs: 4 | - populate-or-rotate-vault 5 | - deploy 6 | - name: deploy 7 | jobs: 8 | - populate-or-rotate-vault 9 | - deploy 10 | 11 | resources: 12 | - name: cf-release 13 | type: bosh-io-release 14 | check_every: 1h 15 | source: 16 | repository: cloudfoundry/cf-release 17 | - name: garden-release 18 | type: bosh-io-release 19 | check_every: 1h 20 | source: 21 | repository: cloudfoundry/garden-linux-release 22 | - name: diego-release 23 | type: bosh-io-release 24 | check_every: 1h 25 | source: 26 | repository: cloudfoundry/diego-release 27 | - name: etcd-release 28 | type: bosh-io-release 29 | check_every: 1h 30 | source: 31 | repository: cloudfoundry-incubator/etcd-release 32 | - name: cf-mysql-release 33 | type: bosh-io-release 34 | check_every: 1h 35 | source: 36 | repository: cloudfoundry/cf-mysql-release 37 | - name: cflinuxfs2-release 38 | type: bosh-io-release 39 | check_every: 1h 40 | source: 41 | repository: cloudfoundry/cflinuxfs2-rootfs-release 42 | - name: stemcell 43 | type: bosh-io-stemcell 44 | check_every: 1h 45 | source: 46 | force_regular: true 47 | name: {{stemcell-name}} 48 | - name: concourse-deploy-cloudfoundry 49 | type: git 50 | check_every: 1h 51 | source: 52 | uri: https://github.com/enaml-ops/concourse-deploy-cloudfoundry 53 | branch: master 54 | - name: omg-cli 55 | type: github-release 56 | check_every: 1h 57 | source: 58 | user: enaml-ops 59 | repository: omg-cli 60 | - name: omg-product-bundle 61 | type: github-release 62 | check_every: 1h 63 | source: 64 | user: enaml-ops 65 | repository: omg-product-bundle 66 | - name: pipeline-tasks 67 | type: git 68 | check_every: 1h 69 | source: 70 | uri: https://github.com/18f/cg-pipeline-tasks.git 71 | branch: master 72 | - name: {{deployment-name}} 73 | type: bosh-deployment 74 | source: 75 | ca_cert: {{bosh-cacert}} 76 | client_id: {{bosh-client-id}} 77 | client_secret: {{bosh-client-secret}} 78 | deployment: {{deployment-name}} 79 | target: {{bosh-url}} 80 | - name: slack 81 | type: slack-notification 82 | source: 83 | url: {{slack-url}} 84 | 85 | resource_types: 86 | - name: slack-notification 87 | type: docker-image 88 | source: 89 | repository: cfcommunity/slack-notification-resource 90 | 91 | jobs: 92 | - name: populate-or-rotate-vault 93 | plan: 94 | - get: concourse-deploy-cloudfoundry 95 | - get: omg-cli 96 | params: 97 | globs: 98 | - omg-linux 99 | - get: omg-product-bundle 100 | trigger: true 101 | params: 102 | globs: 103 | - {{product-plugin}} 104 | - task: populate-or-rotate 105 | config: 106 | platform: linux 107 | image_resource: 108 | type: docker-image 109 | source: 110 | repository: virtmerlin/deploy-worker 111 | run: 112 | path: concourse-deploy-cloudfoundry/ci/oss-tasks/populate-or-rotate-vault.sh 113 | params: 114 | STEMCELL_OS: {{stemcell-os}} 115 | SKIP_HAPROXY: {{skip-haproxy}} 116 | PRODUCT_PLUGIN: {{product-plugin}} 117 | SYSTEM_DOMAIN: {{system-domain}} 118 | VAULT_ADDR: {{vault-addr}} 119 | VAULT_HASH_IP: {{vault-hash-ip}} 120 | VAULT_HASH_HOSTVARS: {{vault-hash-hostvars}} 121 | VAULT_HASH_KEYCERT: {{vault-hash-keycert}} 122 | VAULT_HASH_MISC: {{vault-hash-misc}} 123 | VAULT_HASH_PASSWORD: {{vault-hash-password}} 124 | VAULT_TOKEN: {{vault-token}} 125 | inputs: 126 | - name: concourse-deploy-cloudfoundry 127 | - name: omg-cli 128 | - name: omg-product-bundle 129 | outputs: 130 | - name: deploy 131 | plan: 132 | - aggregate: 133 | - get: cf-release 134 | - get: garden-release 135 | - get: diego-release 136 | - get: etcd-release 137 | - get: cf-mysql-release 138 | - get: cflinuxfs2-release 139 | - get: stemcell 140 | - get: concourse-deploy-cloudfoundry 141 | - get: omg-cli 142 | params: 143 | globs: 144 | - omg-linux 145 | - get: omg-product-bundle 146 | params: 147 | globs: 148 | - oss-cf-plugin-linux 149 | trigger: true 150 | - task: generate-manifest 151 | config: 152 | platform: linux 153 | image_resource: 154 | type: docker-image 155 | source: 156 | repository: virtmerlin/deploy-worker 157 | run: 158 | path: concourse-deploy-cloudfoundry/ci/oss-tasks/generate-manifest.sh 159 | params: 160 | STEMCELL_OS: {{stemcell-os}} 161 | SKIP_HAPROXY: {{skip-haproxy}} 162 | BOSH_CLIENT: {{bosh-client-id}} 163 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 164 | BOSH_CACERT: {{bosh-cacert}} 165 | PRODUCT_PLUGIN: {{product-plugin}} 166 | VAULT_ADDR: {{vault-addr}} 167 | VAULT_HASH_IP: {{vault-hash-ip}} 168 | VAULT_HASH_HOSTVARS: {{vault-hash-hostvars}} 169 | VAULT_HASH_KEYCERT: {{vault-hash-keycert}} 170 | VAULT_HASH_MISC: {{vault-hash-misc}} 171 | VAULT_HASH_PASSWORD: {{vault-hash-password}} 172 | VAULT_TOKEN: {{vault-token}} 173 | inputs: 174 | - name: cf-release 175 | - name: garden-release 176 | - name: diego-release 177 | - name: etcd-release 178 | - name: cf-mysql-release 179 | - name: cflinuxfs2-release 180 | - name: stemcell 181 | - name: concourse-deploy-cloudfoundry 182 | - name: omg-cli 183 | - name: omg-product-bundle 184 | outputs: 185 | - name: manifest 186 | path: "" 187 | - put: {{deployment-name}} 188 | params: 189 | manifest: manifest/deployment.yml 190 | stemcells: 191 | - stemcell/*.tgz 192 | releases: 193 | - cf-release/*.tgz 194 | - garden-release/*.tgz 195 | - diego-release/*.tgz 196 | - etcd-release/*.tgz 197 | - cf-mysql-release/*.tgz 198 | - cflinuxfs2-release/*.tgz 199 | # on_success: 200 | # put: slack 201 | # params: 202 | # text: | 203 | # :white_check_mark: Successfully deployed {{deployment-name}} CF 204 | # 205 | # channel: {{slack-channel}} 206 | # username: {{slack-username}} 207 | # icon_url: {{slack-icon-url}} 208 | # on_failure: 209 | # put: slack 210 | # params: 211 | # text: | 212 | # :x: FAILED to deploy {{deployment-name}} CF 213 | # 214 | # channel: {{slack-channel}} 215 | # username: {{slack-username}} 216 | # icon_url: {{slack-icon-url}} 217 | - name: run-smoke-tests 218 | plan: 219 | - aggregate: 220 | - get: pipeline-tasks 221 | trigger: false 222 | - get: {{deployment-name}} 223 | passed: [deploy] 224 | trigger: true 225 | - task: run-errand 226 | file: pipeline-tasks/bosh-errand.yml 227 | params: 228 | BOSH_TARGET: {{bosh-url}} 229 | BOSH_USERNAME: {{bosh-user}} 230 | BOSH_PASSWORD: {{bosh-pass}} 231 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 232 | BOSH_ERRAND: smoke_tests 233 | BOSH_CACERT: {{bosh-cacert}} 234 | on_success: 235 | put: slack 236 | params: 237 | text: | 238 | :white_check_mark: Smoke tests for {{deployment-name}} CF passed 239 | 240 | channel: {{slack-channel}} 241 | username: {{slack-username}} 242 | icon_url: {{slack-icon-url}} 243 | on_failure: 244 | put: slack 245 | params: 246 | text: | 247 | :x: Smoke tests for {{deployment-name}} CF failed 248 | 249 | channel: {{slack-channel}} 250 | username: {{slack-username}} 251 | icon_url: {{slack-icon-url}} 252 | - name: run-acceptance-tests 253 | plan: 254 | - aggregate: 255 | - get: pipeline-tasks 256 | trigger: false 257 | - get: {{deployment-name}} 258 | passed: [run-smoke-tests] 259 | trigger: true 260 | - task: run-errand 261 | file: pipeline-tasks/bosh-errand.yml 262 | params: 263 | BOSH_TARGET: {{bosh-url}} 264 | BOSH_USERNAME: {{bosh-user}} 265 | BOSH_PASSWORD: {{bosh-pass}} 266 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 267 | BOSH_ERRAND: smoke_tests 268 | BOSH_CACERT: {{bosh-cacert}} 269 | on_success: 270 | put: slack 271 | params: 272 | text: | 273 | :white_check_mark: Acceptance tests for {{deployment-name}} CF passed 274 | 275 | channel: {{slack-channel}} 276 | username: {{slack-username}} 277 | icon_url: {{slack-icon-url}} 278 | on_failure: 279 | put: slack 280 | params: 281 | text: | 282 | :x: Acceptance tests for {{deployment-name}} CF failed 283 | 284 | channel: {{slack-channel}} 285 | username: {{slack-username}} 286 | icon_url: {{slack-icon-url}} 287 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /ci/pcf-pipeline.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: all 3 | jobs: 4 | - populate-or-rotate-vault 5 | - get-product-version 6 | - deploy 7 | - smoke-tests 8 | - acceptance-tests 9 | - push-apps-manager 10 | - notifications 11 | - notifications-ui 12 | - autoscaling 13 | - autoscaling-register-broker 14 | - load-vault-properties 15 | - name: vault-interactions 16 | jobs: 17 | - populate-or-rotate-vault 18 | - load-vault-properties 19 | - name: deploy 20 | jobs: 21 | - get-product-version 22 | - deploy 23 | - name: tests 24 | jobs: 25 | - smoke-tests 26 | - acceptance-tests 27 | - name: errands 28 | jobs: 29 | - push-apps-manager 30 | - notifications 31 | - notifications-ui 32 | - autoscaling 33 | - autoscaling-register-broker 34 | 35 | resources: 36 | - name: daily 37 | type: time 38 | source: {interval: 24h} 39 | - name: {{product-slug}} 40 | type: pivnet 41 | check_every: 4h 42 | source: 43 | api_token: {{pivnet-api-token}} 44 | product_slug: {{product-slug}} 45 | product_version: {{product-version}} 46 | sort_by: semver 47 | - name: every-4hrs 48 | type: time 49 | source: {interval: 4h} 50 | - name: concourse-deploy-cloudfoundry 51 | type: git 52 | check_every: 4h 53 | source: 54 | uri: https://github.com/enaml-ops/concourse-deploy-cloudfoundry 55 | branch: master 56 | - name: omg-cli 57 | type: github-release 58 | check_every: 4h 59 | source: 60 | user: enaml-ops 61 | repository: omg-cli 62 | - name: ert-plugin 63 | type: github-release 64 | check_every: 4h 65 | source: 66 | user: enaml-ops 67 | repository: ert-plugin 68 | - name: {{deployment-name}} 69 | type: bosh-deployment 70 | source: 71 | ca_cert: {{bosh-cacert}} 72 | client_id: {{bosh-client-id}} 73 | client_secret: {{bosh-client-secret}} 74 | deployment: {{deployment-name}} 75 | target: {{bosh-url}} 76 | - name: stemcells 77 | type: pivnet 78 | source: 79 | api_token: {{pivnet-api-token}} 80 | product_slug: stemcells 81 | product_version: {{stemcell-version}} 82 | 83 | resource_types: 84 | - name: pivnet 85 | type: docker-image 86 | source: 87 | repository: pivotalcf/pivnet-resource 88 | tag: latest-final 89 | 90 | jobs: 91 | - name: load-vault-properties 92 | plan: 93 | - aggregate: 94 | - get: concourse-deploy-cloudfoundry 95 | - task: load-vault-properties 96 | config: 97 | platform: linux 98 | image_resource: 99 | type: docker-image 100 | source: 101 | repository: virtmerlin/deploy-worker 102 | run: 103 | path: concourse-deploy-cloudfoundry/ci/tasks/populate-vault-deployment-properties.sh 104 | params: 105 | VAULT_ADDR: {{vault-addr}} 106 | VAULT_HASH_MISC: {{vault-hash-misc}} 107 | VAULT_TOKEN: {{vault-token}} 108 | VAULT_PROPERTIES_JSON: {{vault-json-string}} 109 | BOSH_CLIENT_SECRET: {{bosh-pass}} 110 | BOSH_CACERT: {{bosh-cacert}} 111 | inputs: 112 | - name: concourse-deploy-cloudfoundry 113 | - name: populate-or-rotate-vault 114 | plan: 115 | - aggregate: 116 | - get: concourse-deploy-cloudfoundry 117 | - get: omg-cli 118 | params: 119 | globs: 120 | - omg-linux 121 | - get: ert-plugin 122 | params: 123 | globs: 124 | - ert-1-8-linux 125 | - task: populate-or-rotate 126 | config: 127 | platform: linux 128 | image_resource: 129 | type: docker-image 130 | source: 131 | repository: virtmerlin/deploy-worker 132 | run: 133 | path: concourse-deploy-cloudfoundry/ci/tasks/populate-or-rotate-vault.sh 134 | params: 135 | PRODUCT_PLUGIN: {{product-plugin}} 136 | SYSTEM_DOMAIN: {{system-domain}} 137 | VAULT_ADDR: {{vault-addr}} 138 | VAULT_HASH_IP: {{vault-hash-ip}} 139 | VAULT_HASH_HOSTVARS: {{vault-hash-hostvars}} 140 | VAULT_HASH_KEYCERT: {{vault-hash-keycert}} 141 | VAULT_HASH_MISC: {{vault-hash-misc}} 142 | VAULT_HASH_PASSWORD: {{vault-hash-password}} 143 | VAULT_TOKEN: {{vault-token}} 144 | STEMCELL_VERSION: {{stemcell-version}} 145 | inputs: 146 | - name: concourse-deploy-cloudfoundry 147 | - name: omg-cli 148 | - name: ert-plugin 149 | outputs: 150 | - name: get-product-version 151 | plan: 152 | - aggregate: 153 | - get: concourse-deploy-cloudfoundry 154 | - get: omg-cli 155 | params: 156 | globs: 157 | - omg-linux 158 | - get: ert-plugin 159 | trigger: true 160 | params: 161 | globs: 162 | - ert-1-8-linux 163 | - task: get-product-version 164 | config: 165 | platform: linux 166 | image_resource: 167 | type: docker-image 168 | source: 169 | repository: virtmerlin/deploy-worker 170 | run: 171 | path: concourse-deploy-cloudfoundry/ci/tasks/get-version-from-plugin.sh 172 | params: 173 | PRODUCT_PLUGIN: {{product-plugin}} 174 | OUTPUT_DIR: versions 175 | inputs: 176 | - name: concourse-deploy-cloudfoundry 177 | - name: omg-cli 178 | - name: ert-plugin 179 | outputs: 180 | - name: versions 181 | path: "" 182 | # - task: update-pipeline 183 | # config: 184 | # platform: linux 185 | # image_resource: 186 | # type: docker-image 187 | # source: 188 | # repository: virtmerlin/deploy-worker 189 | # run: 190 | # path: concourse-deploy-cloudfoundry/ci/tasks/update-pipeline.sh 191 | # params: 192 | # PRODUCT: {{product-slug}} 193 | # PIPELINE_NAME: deploy-pcf 194 | # CONCOURSE_URL: {{concourse-url}} 195 | # CONCOURSE_USER: {{concourse-user}} 196 | # CONCOURSE_PASS: {{concourse-pass}} 197 | # inputs: 198 | # - name: concourse-deploy-cloudfoundry 199 | # - name: versions 200 | # outputs: 201 | - name: deploy 202 | plan: 203 | - aggregate: 204 | - get: concourse-deploy-cloudfoundry 205 | - get: omg-cli 206 | passed: [get-product-version] 207 | params: 208 | globs: 209 | - omg-linux 210 | - get: ert-plugin 211 | passed: [get-product-version] 212 | trigger: true 213 | params: 214 | globs: 215 | - ert-1-8-linux 216 | - get: {{product-slug}} 217 | params: 218 | globs: 219 | - cf-* 220 | - get: stemcells 221 | trigger: true 222 | params: 223 | globs: 224 | - {{stemcell-cpi-glob}} 225 | - task: generate-manifest 226 | config: 227 | platform: linux 228 | image_resource: 229 | type: docker-image 230 | source: 231 | repository: virtmerlin/deploy-worker 232 | run: 233 | path: concourse-deploy-cloudfoundry/ci/tasks/generate-pcf-manifest.sh 234 | params: 235 | BOSH_CLIENT: {{bosh-user}} 236 | BOSH_CLIENT_SECRET: {{bosh-pass}} 237 | BOSH_CACERT: {{bosh-cacert}} 238 | PRODUCT_PLUGIN: {{product-plugin}} 239 | SKIP_HAPROXY: {{skip-haproxy}} 240 | STEMCELL_VERSION: {{stemcell-version}} 241 | VAULT_ADDR: {{vault-addr}} 242 | VAULT_HASH_IP: {{vault-hash-ip}} 243 | VAULT_HASH_HOSTVARS: {{vault-hash-hostvars}} 244 | VAULT_HASH_KEYCERT: {{vault-hash-keycert}} 245 | VAULT_HASH_MISC: {{vault-hash-misc}} 246 | VAULT_HASH_PASSWORD: {{vault-hash-password}} 247 | VAULT_TOKEN: {{vault-token}} 248 | OUTPUT_DIR: manifest 249 | inputs: 250 | - name: concourse-deploy-cloudfoundry 251 | - name: omg-cli 252 | - name: ert-plugin 253 | outputs: 254 | - name: manifest 255 | path: "" 256 | - task: extract-tile 257 | config: 258 | platform: linux 259 | image_resource: 260 | type: docker-image 261 | source: 262 | repository: virtmerlin/deploy-worker 263 | run: 264 | path: concourse-deploy-cloudfoundry/ci/tasks/extract-tile.sh 265 | params: 266 | PRODUCT_DIR: {{product-slug}} 267 | OUTPUT_DIR: product-extracted 268 | inputs: 269 | - name: concourse-deploy-cloudfoundry 270 | - name: {{product-slug}} 271 | outputs: 272 | - name: product-extracted 273 | path: "" 274 | - put: {{deployment-name}} 275 | params: 276 | manifest: manifest/deployment.yml 277 | stemcells: 278 | - stemcells/bosh-stemcell*.tgz 279 | releases: 280 | - product-extracted/releases/cf*.tgz 281 | - product-extracted/releases/diego*.tgz 282 | - product-extracted/releases/garden-runc*.tgz 283 | - product-extracted/releases/cflinuxfs2*.tgz 284 | - product-extracted/releases/etcd*.tgz 285 | - product-extracted/releases/mysql-backup*.tgz 286 | - product-extracted/releases/push-apps-manager*.tgz 287 | - product-extracted/releases/cf-autoscaling*.tgz 288 | - product-extracted/releases/notifications*.tgz 289 | - name: smoke-tests 290 | plan: 291 | - aggregate: 292 | - get: every-4hrs 293 | trigger: true 294 | - get: concourse-deploy-cloudfoundry 295 | - get: {{deployment-name}} 296 | passed: [deploy] 297 | - task: smoke-tests 298 | config: 299 | platform: linux 300 | image_resource: 301 | type: docker-image 302 | source: 303 | repository: virtmerlin/deploy-worker 304 | run: 305 | path: concourse-deploy-cloudfoundry/ci/tasks/run-errand.sh 306 | params: 307 | BOSH_CACERT: {{bosh-cacert}} 308 | BOSH_CLIENT: {{bosh-client-id}} 309 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 310 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 311 | BOSH_ERRAND: smoke-tests 312 | BOSH_TARGET: {{bosh-url}} 313 | inputs: 314 | - name: concourse-deploy-cloudfoundry 315 | outputs: 316 | - name: acceptance-tests 317 | plan: 318 | - aggregate: 319 | - get: daily 320 | trigger: true 321 | - get: concourse-deploy-cloudfoundry 322 | - get: {{deployment-name}} 323 | passed: [deploy] 324 | - task: acceptance-tests 325 | config: 326 | platform: linux 327 | image_resource: 328 | type: docker-image 329 | source: 330 | repository: virtmerlin/deploy-worker 331 | run: 332 | path: concourse-deploy-cloudfoundry/ci/tasks/run-errand.sh 333 | params: 334 | BOSH_CACERT: {{bosh-cacert}} 335 | BOSH_CLIENT: {{bosh-client-id}} 336 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 337 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 338 | BOSH_ERRAND: acceptance-tests 339 | BOSH_TARGET: {{bosh-url}} 340 | inputs: 341 | - name: concourse-deploy-cloudfoundry 342 | outputs: 343 | - name: push-apps-manager 344 | plan: 345 | - aggregate: 346 | - get: concourse-deploy-cloudfoundry 347 | - get: {{deployment-name}} 348 | passed: [deploy] 349 | trigger: true 350 | - task: push-apps-manager 351 | config: 352 | platform: linux 353 | image_resource: 354 | type: docker-image 355 | source: 356 | repository: virtmerlin/deploy-worker 357 | run: 358 | path: concourse-deploy-cloudfoundry/ci/tasks/run-errand.sh 359 | params: 360 | BOSH_CACERT: {{bosh-cacert}} 361 | BOSH_CLIENT: {{bosh-client-id}} 362 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 363 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 364 | BOSH_ERRAND: push-apps-manager 365 | BOSH_TARGET: {{bosh-url}} 366 | inputs: 367 | - name: concourse-deploy-cloudfoundry 368 | outputs: 369 | - name: notifications 370 | plan: 371 | - aggregate: 372 | - get: concourse-deploy-cloudfoundry 373 | - get: {{deployment-name}} 374 | passed: [push-apps-manager] 375 | trigger: true 376 | - task: notifications 377 | config: 378 | platform: linux 379 | image_resource: 380 | type: docker-image 381 | source: 382 | repository: virtmerlin/deploy-worker 383 | run: 384 | path: concourse-deploy-cloudfoundry/ci/tasks/run-errand.sh 385 | params: 386 | BOSH_CACERT: {{bosh-cacert}} 387 | BOSH_CLIENT: {{bosh-client-id}} 388 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 389 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 390 | BOSH_ERRAND: notifications 391 | BOSH_TARGET: {{bosh-url}} 392 | inputs: 393 | - name: concourse-deploy-cloudfoundry 394 | outputs: 395 | - name: notifications-ui 396 | plan: 397 | - aggregate: 398 | - get: concourse-deploy-cloudfoundry 399 | - get: {{deployment-name}} 400 | passed: [notifications] 401 | trigger: true 402 | - task: notifications-ui 403 | config: 404 | platform: linux 405 | image_resource: 406 | type: docker-image 407 | source: 408 | repository: virtmerlin/deploy-worker 409 | run: 410 | path: concourse-deploy-cloudfoundry/ci/tasks/run-errand.sh 411 | params: 412 | BOSH_CACERT: {{bosh-cacert}} 413 | BOSH_CLIENT: {{bosh-client-id}} 414 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 415 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 416 | BOSH_ERRAND: notifications-ui 417 | BOSH_TARGET: {{bosh-url}} 418 | inputs: 419 | - name: concourse-deploy-cloudfoundry 420 | outputs: 421 | - name: autoscaling 422 | plan: 423 | - aggregate: 424 | - get: concourse-deploy-cloudfoundry 425 | - get: {{deployment-name}} 426 | passed: [notifications-ui] 427 | trigger: true 428 | - task: autoscaling 429 | config: 430 | platform: linux 431 | image_resource: 432 | type: docker-image 433 | source: 434 | repository: virtmerlin/deploy-worker 435 | run: 436 | path: concourse-deploy-cloudfoundry/ci/tasks/run-errand.sh 437 | params: 438 | BOSH_CACERT: {{bosh-cacert}} 439 | BOSH_CLIENT: {{bosh-client-id}} 440 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 441 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 442 | BOSH_ERRAND: autoscaling 443 | BOSH_TARGET: {{bosh-url}} 444 | inputs: 445 | - name: concourse-deploy-cloudfoundry 446 | outputs: 447 | - name: autoscaling-register-broker 448 | plan: 449 | - aggregate: 450 | - get: concourse-deploy-cloudfoundry 451 | - get: {{deployment-name}} 452 | passed: [autoscaling] 453 | trigger: true 454 | - task: autoscaling-register-broker 455 | config: 456 | platform: linux 457 | image_resource: 458 | type: docker-image 459 | source: 460 | repository: virtmerlin/deploy-worker 461 | run: 462 | path: concourse-deploy-cloudfoundry/ci/tasks/run-errand.sh 463 | params: 464 | BOSH_CACERT: {{bosh-cacert}} 465 | BOSH_CLIENT: {{bosh-client-id}} 466 | BOSH_CLIENT_SECRET: {{bosh-client-secret}} 467 | BOSH_DEPLOYMENT_NAME: {{deployment-name}} 468 | BOSH_ERRAND: autoscaling-register-broker 469 | BOSH_TARGET: {{bosh-url}} 470 | inputs: 471 | - name: concourse-deploy-cloudfoundry 472 | outputs: 473 | --------------------------------------------------------------------------------