├── .envrc ├── .github └── dependabot.yml ├── LICENSE ├── NOTICE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── bbl-destroy ├── task └── task.yml ├── bbl-up ├── task └── task.yml ├── bosh-cleanup ├── task └── task.yml ├── bosh-delete-deployment ├── task └── task.yml ├── bosh-deploy-with-created-release ├── task └── task.yml ├── bosh-deploy-with-updated-release-submodule ├── task └── task.yml ├── bosh-deploy ├── task └── task.yml ├── bosh-upload-stemcell-from-cf-deployment ├── bosh-upload-stemcells ├── task └── task.yml ├── bump_docker_images.sh ├── ci ├── configure ├── pipeline.md ├── pipeline.yml └── tasks │ ├── bump-cf-d-ct-docker-image │ ├── task │ └── task.yml │ ├── cf-decota-release-notes-template │ ├── Gemfile │ ├── Gemfile.lock │ ├── Guardfile │ ├── binary_changes.rb │ ├── binary_changes_spec.rb │ ├── renderer.rb │ ├── renderer_spec.rb │ ├── task │ ├── task.rb │ ├── task.yml │ ├── task_finder.rb │ ├── task_finder_spec.rb │ ├── task_updates.rb │ └── task_updates_spec.rb │ ├── generate-major-version-file │ ├── task │ └── task.yml │ └── update-cf-d-ct-dockerfile-versions │ ├── task │ └── task.yml ├── collect-ops-files ├── task └── task.yml ├── dockerfiles └── cf-deployment-concourse-tasks │ └── Dockerfile ├── open-asgs-for-bosh-instance-group ├── create_asgs_json.rb ├── task └── task.yml ├── run-cats-with-provided-cli ├── run-cats ├── task └── task.yml ├── run-errand ├── task └── task.yml ├── set-feature-flags ├── task └── task.yml ├── shared-functions └── update-integration-configs ├── task └── task.yml /.envrc: -------------------------------------------------------------------------------- 1 | export PROJECT_DIR=$PWD 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "docker" 4 | directory: "/dockerfiles/cf-deployment-concourse-tasks" 5 | schedule: 6 | interval: "daily" 7 | 8 | - package-ecosystem: "bundler" 9 | directory: "/ci/tasks/cf-decota-release-notes-template" 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. 2 | 3 | This project is licensed to you under the Apache License, Version 2.0 (the "License"). 4 | You may not use this project except in compliance with the License. 5 | 6 | This project may include a number of subcomponents with separate copyright notices 7 | and license terms. Your use of these subcomponents is subject to the terms and 8 | conditions of the subcomponent's license, as noted in the LICENSE file. 9 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### What is this change about? 2 | 3 | _Describe the change and why it's needed._ 4 | 5 | 6 | ### Please provide contextual information. 7 | 8 | _Include any links to other PRs, stories, slack discussions, etc... that will help establish context._ 9 | 10 | 11 | 12 | ### Please check all that apply for this PR: 13 | - [ ] introduces a new task 14 | - [ ] changes an existing task 15 | - [ ] changes the Dockerfile 16 | - [ ] introduces a breaking change (other users will need to make manual changes when this is released) 17 | 18 | 19 | 20 | ### Did you update the README as appropriate for this change? 21 | - [ ] YES 22 | - [ ] N/A 23 | 24 | 25 | 26 | ### How should this change be described in release notes? 27 | 28 | _Something brief that conveys the change and is written with the component author audience in mind._ 29 | 30 | 31 | 32 | ### What is the level of urgency for publishing this change? 33 | 34 | - [ ] **Urgent** - unblocks current or future work 35 | - [ ] **Slightly Less than Urgent** 36 | 37 | 38 | 39 | ### Tag your pair, your PM, and/or team! 40 | _It's helpful to tag a few other folks on your team or your team alias in case we need to follow up later._ 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cf-deployment-concourse-tasks 2 | This repo contains concourse tasks for use with [cf-deployment][cf-deployment-repo]. 3 | If you are trying to deploy to IAAS, you may wish to check the [Deployment Guide][deployment-guide]. 4 | Each task is in its own directory. 5 | A task will generally be composed of a task.yml to be referenced in pipelines, 6 | a supporting task file, and a Dockerfile. 7 | The Dockerfile is built and pushed to Dockerhub regularly 8 | in CI maintained by the CF Release Integration team [here][cf-deployment-concourse-tasks-ci]. 9 | 10 | It should be clear how to use each task 11 | from the task.yml 12 | and the description below. 13 | If you find that it is not, 14 | please contact the Release Integration team 15 | in our [Slack channel dedicated to supporting users of `cf-deployment`][cf-deployment-slack-channel]. 16 | Alternatively, you can [open an issue][issues-page]. 17 | 18 | ### Versioning of this repo 19 | Development updates to the repo are made to the `master` branch, 20 | so untested or backwards incompatible changes may be present there. 21 | Once changes have been tested and all stories accepted, 22 | we add new version tags such as `v1.6` to the approprate commit. 23 | 24 | We use a bare-bones type of [semantic versioning](http://semver.org/) for this repo. 25 | Backwards incompatible changes warrant a major version bump (e.g. `v1.6` to `v2.0`), 26 | while other changes will simply add a minor version bump (e.g. `v2.0` to `v2.1`). 27 | 28 | In Concourse, you can pretty easily lock to a major version, 29 | meaning that your pipeline will take minor (i.e. backwards compatible) changes only. 30 | Here's an example from our [nats release pipeline](https://github.com/cloudfoundry/runtime-ci/blob/5e4d8a384c9e9fc7ddc052cd8c21503d40d29851/pipelines/nats-release.yml#L91-L96): 31 | ``` 32 | - name: cf-deployment-concourse-tasks 33 | type: git 34 | source: 35 | branch: main 36 | uri: https://github.com/cloudfoundry/cf-deployment-concourse-tasks.git 37 | tag_filter: v3.* 38 | ``` 39 | 40 | When you're ready to take the backwards _incompatible_ changes, 41 | you can take any necessary manual steps to upgrade, 42 | and then change the major version in your pipeline configuration. 43 | 44 | 45 | ## Tasks 46 | Tasks are listed here alphabetically, 47 | along with a brief description 48 | meant to be used alongside the `task.yml` within each task directory 49 | to understand the tasks' 50 | purpose, interface, and options. 51 | Each title is also a link 52 | to the appropriate task.yml. 53 | 54 | ### [bbl-destroy][bbl-destroy-task-yaml] 55 | This destroys the director 56 | and infrastructure 57 | created by [bbl](https://github.com/cloudfoundry/bosh-bootloader). 58 | Debug output 59 | is written to 60 | bbl_destroy.txt 61 | to help debug failures 62 | in this task. 63 | 64 | ### [bbl-up][bbl-up-task-yaml] 65 | This uses [bbl](https://github.com/cloudfoundry/bosh-bootloader) 66 | to create your infrastructure 67 | and deploy a BOSH director. 68 | Debug output 69 | is written to 70 | `bbl_plan.txt` and 71 | `bbl_up.txt` 72 | to help debug failures 73 | in this task. 74 | This task requires 75 | a certificate and key 76 | (unless you are `bbl`ing up a bosh-lite environment) 77 | which can be generated using 78 | the commands specified [here][deployment-guide-on-certificates]. 79 | 80 | ### [bosh-cleanup][bosh-cleanup-task-yaml] 81 | This performs a BOSH cleanup 82 | which is necessary 83 | from time to time 84 | to avoid 85 | running out of space. 86 | 87 | ### [bosh-delete-deployment][bosh-delete-deployment-task-yaml] 88 | This deletes a BOSH deployment. 89 | If you want to delete all of the available BOSH deployments you can set the `DELETE_ALL_DEPLOYMENTS` flag to `true`. 90 | 91 | ### [bosh-deploy][bosh-deploy-task-yaml] 92 | This performs a BOSH upload-stemcell and BOSH deployment. 93 | Optionally, operations files may be applied 94 | to the deployment manifest. 95 | 96 | It's also configurable to 97 | regenerate deployment credentials 98 | on each deployment 99 | though this is not the default behavior. 100 | This is helpful for testing 101 | changes to variable generation, 102 | but is only expected to work 103 | with fresh deployments. 104 | This also automatically uploads the stemcells present in deployment. 105 | If you're deploying to **bosh-lite** environment you need to set the 106 | `BOSH_LITE` flag to `true` so the task uploads the correct stemcell(s). 107 | 108 | ### [bosh-deploy-with-created-release][bosh-deploy-with-created-release-task-yaml] 109 | This creates and applies an 110 | additional operations file to `cf-deployment.yml`, 111 | which causes BOSH to 112 | create, upload, and use a dev release 113 | from the provided release folder 114 | in place of the version specified in `cf-deployment.yml`. 115 | This is useful for testing an upstream component. 116 | Otherwise identical to the `bosh-deploy` task above. 117 | 118 | ### [bosh-deploy-with-updated-release-submodule](bosh-deploy-with-updated-release-submodule/task.yml) 119 | This takes as input 120 | a concourse resource 121 | for the submodule version bumped 122 | when creating a dev release 123 | from the provided release folder. 124 | Otherwise identical to the `bosh-deploy-with-created-release` task above. 125 | 126 | ### [bosh-upload-stemcells][bosh-upload-stemcells-task-yaml] 127 | This uploads stemcell(s) associated with the manifest and/or ops files provided. 128 | This task can be used to upload stemcells within jobs that do not contain a bosh-deploy* task (which handles uploading stemcells as well as executing the bosh deployment). 129 | 130 | ### [collect-ops-files][collect-ops-files] 131 | This collects 132 | two sets of operations files. 133 | The first set is the "base" set, 134 | to which the second ("new") set is added. 135 | 136 | If there is a name conflict, 137 | the operations file 138 | from the second ("new") set 139 | wins. 140 | 141 | ### [run-cats][run-cats-task-yaml] 142 | This runs [CF Acceptance Tests](https://github.com/cloudfoundry/cf-acceptance-tests) 143 | against a CF environment specified by the CATs integration file. 144 | If desired, you can use the optional cf-cli input to provide your own CF CLI 145 | binary. 146 | 147 | ### [run-errand][run-errand-yaml] 148 | This runs a bosh errand 149 | with the provided deployment and errand name. 150 | 151 | ### [set-feature-flags][set-feature-flags-task-yaml] 152 | This will 153 | toggle 154 | the specified feature-flags 155 | based on their boolean values. 156 | 157 | ### [update-integration-configs][update-integration-configs-task-yaml] 158 | This updates integration files 159 | to be consumed by CATs and RATs 160 | with credentials drawn from 161 | CredHub. 162 | 163 | ### [open-asgs-for-bosh-instance-group][open-asgs-for-bosh-instance-group-task-yaml] 164 | This opens Application Security Groups for BOSH 165 | instance groups. 166 | 167 | [bbl-destroy-task-yaml]: bbl-destroy/task.yml 168 | [bbl-up-task-yaml]: bbl-up/task.yml 169 | [bosh-cleanup-task-yaml]: bosh-cleanup/task.yml 170 | [bosh-deploy-task-yaml]: bosh-deploy/task.yml 171 | [bosh-deploy-with-created-release-task-yaml]: bosh-deploy-with-created-release/task.yml 172 | [bosh-delete-deployment-task-yaml]: bosh-delete-deployment/task.yml 173 | [bosh-upload-stemcells-task-yaml]: bosh-upload-stemcells/task.yml 174 | [bosh-upload-stemcell-from-cf-deployment-task-yaml]: bosh-upload-stemcell-from-cf-deployment/task.yml 175 | [cf-deployment-repo]: https://github.com/cloudfoundry/cf-deployment 176 | [cf-deployment-slack-channel]: https://cloudfoundry.slack.com/messages/cf-deployment/ 177 | [collect-ops-files]: collect-ops-files/task.yml 178 | [deploy-with-created-lines]: bosh-deploy-with-created-release/task#L49-L55 179 | [deployment-guide]: https://github.com/cloudfoundry/cf-deployment/blob/master/texts/deployment-guide.md 180 | [deployment-guide-on-certificates]: https://github.com/cloudfoundry/cf-deployment/blob/master/texts/deployment-guide.md#on-certificates 181 | [issues-page]: https://github.com/cloudfoundry/cf-deployment-concourse-tasks/issues 182 | [run-cats-task-yaml]: run-cats/task.yml 183 | [run-errand-yaml]: run-errand/task.yml 184 | [cf-deployment-concourse-tasks-ci]: https://concourse.wg-ard.ci.cloudfoundry.org/teams/main/pipelines/cf-deployment-concourse-tasks 185 | [set-feature-flags-task-yaml]: set-feature-flags/task.yml 186 | [update-integration-configs-task-yaml]: update-integration-configs/task.yml 187 | [open-asgs-for-bosh-instance-group-task-yaml]: open-asgs-for-bosh-instance-group/task.yml 188 | -------------------------------------------------------------------------------- /bbl-destroy/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xeu 2 | 3 | # shellcheck disable=SC1091 4 | source cf-deployment-concourse-tasks/shared-functions 5 | 6 | function main() { 7 | local root_dir 8 | root_dir="${1}" 9 | 10 | load_from_json_config 11 | 12 | check_bbl_input_params 13 | 14 | pushd "bbl-state/${BBL_STATE_DIR}" 15 | local bbl_iaas 16 | bbl_iaas=$(jq -r .iaas bbl-state.json) 17 | 18 | if [ "${bbl_iaas}" == "gcp" ]; then 19 | pushd "${root_dir}/bbl-state" 20 | write_gcp_service_account_key 21 | popd 22 | fi 23 | 24 | bbl \ 25 | --debug \ 26 | destroy \ 27 | --no-confirm > "${root_dir}"/bbl_destroy.txt 28 | popd 29 | } 30 | 31 | if [[ "${STORE_BBL_STATE_AS_TARBALL}" == "true" ]]; then 32 | trap "rm -f ${PWD}/bbl-state/bbl-state.tgz ${PWD}/bbl-state/bbl-state.tgz.uncompressed; tar czf '${PWD}/updated-bbl-state/bbl-state.tgz' -C '${PWD}/bbl-state' ." EXIT 33 | else 34 | trap "commit_bbl_state_dir ${PWD} 'Remove bbl state dir'" EXIT 35 | fi 36 | 37 | main "$PWD" 38 | -------------------------------------------------------------------------------- /bbl-destroy/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks # - This repo 12 | - name: bbl-state # - The env repo containing the Director's bbl state dir 13 | - name: pool-lock 14 | optional: true 15 | 16 | outputs: 17 | - name: updated-bbl-state 18 | # - The bbl-state resource, with changes git committed, 19 | # ready to be `put` to a git resource. 20 | # - This output _must_ be persisted for these tasks to function properly 21 | 22 | run: 23 | path: cf-deployment-concourse-tasks/bbl-destroy/task 24 | 25 | params: 26 | BBL_STATE_DIR: bbl-state 27 | # - Path to the directory containing the `bbl-state.json` file 28 | # - The path is relative to the `bbl-state` input 29 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 30 | 31 | GIT_COMMIT_EMAIL: "cf-release-integration@pivotal.io" 32 | GIT_COMMIT_USERNAME: "CI Bot" 33 | # - Optional 34 | # - You may choose the git committer username and email address by setting these 35 | 36 | # AWS Configuration Params 37 | # - Required for AWS 38 | BBL_AWS_ACCESS_KEY_ID: 39 | BBL_AWS_SECRET_ACCESS_KEY: 40 | 41 | # Azure Configuration Params 42 | # - Required for Azure 43 | BBL_AZURE_CLIENT_ID: 44 | BBL_AZURE_CLIENT_SECRET: 45 | BBL_AZURE_TENANT_ID: 46 | BBL_AZURE_SUBSCRIPTION_ID: 47 | 48 | # GCP Configuration Params 49 | # - Required for GCP 50 | BBL_GCP_SERVICE_ACCOUNT_KEY: 51 | # - Key content or path to the file containing credentials downloaded from GCP 52 | # - Path is relative to the `bbl-state` input 53 | 54 | # OpenStack Configuration Params 55 | # - Required for OpenStack 56 | BBL_OPENSTACK_AUTH_URL: 57 | BBL_OPENSTACK_AZ: 58 | BBL_OPENSTACK_NETWORK_ID: 59 | BBL_OPENSTACK_NETWORK_NAME: 60 | BBL_OPENSTACK_PASSWORD: 61 | BBL_OPENSTACK_USERNAME: 62 | BBL_OPENSTACK_PROJECT: 63 | BBL_OPENSTACK_DOMAIN: 64 | BBL_OPENSTACK_REGION: 65 | 66 | STORE_BBL_STATE_AS_TARBALL: false 67 | # - Optional 68 | # - Creates a tarball of the bbl-state directory, instead of treating it 69 | # - as a git repo and committing. 70 | # - This is useful if you want to store your state file in S3 or GCS. 71 | 72 | BBL_JSON_CONFIG: 73 | # - Optional 74 | # - For if you need a dynamic config for BBL 75 | -------------------------------------------------------------------------------- /bbl-up/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xeu 2 | 3 | set -o pipefail 4 | 5 | # shellcheck disable=SC1091 6 | source cf-deployment-concourse-tasks/shared-functions 7 | 8 | function check_fast_fails() { 9 | set +x 10 | 11 | if [ "${SKIP_LB_CREATION}" == "false" ]; then 12 | if [ -z "${LB_DOMAIN}" ]; then 13 | echo "\$LB_DOMAIN is a required parameter. Please set the domain." 14 | exit 1 15 | fi 16 | 17 | if [ -z "${BBL_LB_CERT}" ]; then 18 | echo "\$BBL_LB_CERT is a required parameter" 19 | exit 1 20 | fi 21 | 22 | if [ -z "${BBL_LB_KEY}" ]; then 23 | echo "\$BBL_LB_KEY is a required parameter" 24 | exit 1 25 | fi 26 | fi 27 | 28 | check_bbl_input_params 29 | 30 | set -x 31 | } 32 | 33 | function write_bbl_certs() { 34 | set +x 35 | if [ -f "${BBL_LB_CERT}" ]; then 36 | echo "cert found at ${BBL_LB_CERT}" 37 | bbl_cert_path="${BBL_LB_CERT}" 38 | else 39 | echo "${BBL_LB_CERT}" > /tmp/bbl-cert 40 | bbl_cert_path="/tmp/bbl-cert" 41 | fi 42 | if [ -f "${BBL_LB_CERT_CHAIN}" ]; then 43 | bbl_cert_chain_flag="--lb-chain=${BBL_LB_CERT_CHAIN}" 44 | elif [ ! -z "${BBL_LB_CERT_CHAIN// }" ]; then 45 | echo "${BBL_LB_CERT_CHAIN}" > /tmp/bbl-cert-chain 46 | bbl_cert_chain_flag="--lb-chain=/tmp/bbl-cert-chain" 47 | fi 48 | if [ -f "${BBL_LB_KEY}" ]; then 49 | echo "key found at ${BBL_LB_KEY}" 50 | bbl_key_path="${BBL_LB_KEY}" 51 | else 52 | echo "${BBL_LB_KEY}" > /tmp/bbl-key 53 | bbl_key_path="/tmp/bbl-key" 54 | fi 55 | set -x 56 | } 57 | 58 | function main() { 59 | local root_dir 60 | root_dir="${1}" 61 | 62 | load_from_json_config 63 | 64 | check_fast_fails 65 | 66 | if [ "${BBL_IAAS}" == "gcp" ]; then 67 | pushd "bbl-state" 68 | write_gcp_service_account_key 69 | popd 70 | fi 71 | 72 | mkdir -p "bbl-state/${BBL_STATE_DIR}" 73 | pushd "bbl-state/${BBL_STATE_DIR}" 74 | bbl version 75 | 76 | local name_flag 77 | name_flag="" 78 | 79 | if [ -n "${BBL_ENV_NAME}" -a ! -f bbl-state.json ]; then 80 | name_flag="--name ${BBL_ENV_NAME}" 81 | fi 82 | 83 | local lb_flags 84 | lb_flags="" 85 | 86 | if [ "${SKIP_LB_CREATION}" == "false" ]; then 87 | local bbl_cert_chain_flag 88 | bbl_cert_chain_flag="" 89 | local bbl_cert_path 90 | write_bbl_certs 91 | 92 | lb_flags="--lb-type=cf --lb-cert=${bbl_cert_path} ${bbl_cert_chain_flag} --lb-key=${bbl_key_path} --lb-domain=${LB_DOMAIN}" 93 | fi 94 | 95 | set -o pipefail 96 | local drain 97 | drain=">" 98 | 99 | if [ "${DEBUG_MODE}" == "true" ] ; then 100 | drain="| tee" 101 | fi 102 | 103 | eval bbl plan --debug \ 104 | ${name_flag} \ 105 | ${lb_flags} "2>&1" ${drain} "${root_dir}/bbl_plan.log" 106 | 107 | if [ -n "${BBL_CONFIG_DIR}" ]; then 108 | cp -r ${root_dir}/bbl-config/${BBL_CONFIG_DIR}/. . 109 | fi 110 | 111 | eval bbl --debug up \ 112 | ${name_flag} \ 113 | ${lb_flags} "2>&1" ${drain} "${root_dir}/bbl_up.log" 114 | 115 | ns_record_advisory="For DNS delegation to work, please ensure an NS record for ${LB_DOMAIN} is created in its parent DNS zone with the following nameservers:" 116 | if [[ "$BBL_IAAS" == "aws" ]]; then 117 | echo "$ns_record_advisory" 118 | bbl outputs | yq '.env_dns_zone_name_servers[0:4]' --output-format yaml | cut -d' ' -f2 119 | elif [[ "$BBL_IAAS" == "gcp" ]]; then 120 | echo "$ns_record_advisory" 121 | bbl outputs | yq '.system_domain_dns_servers' --output-format yaml | cut -d' ' -f2 122 | fi 123 | 124 | if [[ "${DELETE_TERRAFORM_PLUGINS}" == "true" ]]; then 125 | rm -rf "terraform/.terraform" 126 | else 127 | pushd "terraform" 128 | terraform init 129 | popd 130 | fi 131 | popd 132 | } 133 | 134 | if [[ "${STORE_BBL_STATE_AS_TARBALL}" == "true" ]]; then 135 | trap "rm -f ${PWD}/bbl-state/bbl-state.tgz ${PWD}/bbl-state/bbl-state.tgz.uncompressed; tar czf '${PWD}/updated-bbl-state/bbl-state.tgz' -C '${PWD}/bbl-state' ." EXIT 136 | else 137 | trap "commit_bbl_state_dir ${PWD} '${GIT_COMMIT_MESSAGE}'" EXIT 138 | fi 139 | 140 | main "$PWD" 141 | -------------------------------------------------------------------------------- /bbl-up/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks # - This repo 12 | - name: bbl-state # - The env repo containing the Director's bbl state dir 13 | - name: bbl-config # - The repo containing bbl plan patches 14 | # - If no plan patches are necessary in your case, 15 | # we suggest you use `input_mapping` 16 | # to map your `bbl-state` resource to `bbl-config` 17 | # in order to satisfy this required input. 18 | - name: pool-lock 19 | optional: true 20 | 21 | outputs: 22 | - name: updated-bbl-state 23 | # - The bbl-state resource, with changes git committed, 24 | # - ready to be `put` to a git resource. 25 | # - This output _must_ be persisted for these tasks to function properly 26 | 27 | run: 28 | path: cf-deployment-concourse-tasks/bbl-up/task 29 | 30 | params: 31 | # AWS Configuration Params 32 | # - Required for AWS 33 | BBL_AWS_ACCESS_KEY_ID: 34 | BBL_AWS_SECRET_ACCESS_KEY: 35 | BBL_AWS_REGION: 36 | 37 | # Azure Configuration Params 38 | # - Required for Azure 39 | BBL_AZURE_CLIENT_ID: 40 | BBL_AZURE_CLIENT_SECRET: 41 | BBL_AZURE_TENANT_ID: 42 | BBL_AZURE_SUBSCRIPTION_ID: 43 | BBL_AZURE_REGION: 44 | 45 | # GCP Configuration Params 46 | # - Required for GCP 47 | BBL_GCP_SERVICE_ACCOUNT_KEY: 48 | # - Key content or path to the file containing credentials downloaded from GCP 49 | # - Path is relative to the `bbl-state` input 50 | BBL_GCP_REGION: 51 | 52 | # OpenStack Configuration Params 53 | # - Required for OpenStack 54 | BBL_OPENSTACK_AUTH_URL: 55 | BBL_OPENSTACK_AZ: 56 | BBL_OPENSTACK_NETWORK_ID: 57 | BBL_OPENSTACK_NETWORK_NAME: 58 | BBL_OPENSTACK_PASSWORD: 59 | BBL_OPENSTACK_USERNAME: 60 | BBL_OPENSTACK_PROJECT: 61 | BBL_OPENSTACK_DOMAIN: 62 | BBL_OPENSTACK_REGION: 63 | 64 | # vSphere Configuration Params 65 | # - Required for vSphere 66 | BBL_VSPHERE_VCENTER_USER: 67 | BBL_VSPHERE_VCENTER_PASSWORD: 68 | BBL_VSPHERE_VCENTER_IP: 69 | BBL_VSPHERE_VCENTER_DC: 70 | BBL_VSPHERE_VCENTER_CLUSTER: 71 | BBL_VSPHERE_VCENTER_RP: 72 | BBL_VSPHERE_NETWORK: 73 | BBL_VSPHERE_VCENTER_DS: 74 | BBL_VSPHERE_SUBNET_CIDR: 75 | BBL_VSPHERE_VCENTER_DISKS: 76 | BBL_VSPHERE_VCENTER_TEMPLATES: 77 | BBL_VSPHERE_VCENTER_VMS: 78 | 79 | BBL_STATE_DIR: bbl-state 80 | # - Path to the directory containing the `bbl-state.json` file 81 | # - The path is relative to the `bbl-state` input 82 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 83 | # - If the specified path does not exist, it will be created 84 | 85 | BBL_CONFIG_DIR: 86 | # - Optional 87 | # - Path to the directory containing bbl plan patches 88 | # - The path is relative to the `bbl-config` input 89 | # - If left empty, no customizations (i.e. ops-files, terraform templates, etc.) will be applied during `bbl up` 90 | 91 | BBL_IAAS: 92 | # - Required 93 | # - The target IAAS which bbl will create infrastructure 94 | # - Must be `aws` or `azure` or `gcp` or `openstack` or `vsphere` 95 | 96 | BBL_LB_CERT: 97 | # - Required if `SKIP_LB_CREATION` is false 98 | # - PEM encoded certificate to be associated with the load balancer 99 | # for SSL termination 100 | # - Can be either the certificate content or a path to the certificate 101 | # - If a path is provided, path is relative to the BBL_STATE_DIR 102 | 103 | BBL_LB_CERT_CHAIN: 104 | # - Optional 105 | # - PEM encoded certificate to be associated with the load balancer 106 | # certificate chain 107 | # - Can be either the certificate content or a path to the certificate 108 | # - If a path is provided, path is relative to the BBL_STATE_DIR 109 | 110 | BBL_LB_KEY: 111 | # - Required if `SKIP_LB_CREATION` is false 112 | # - PEM-encoded private key to be used for TLS termination on the load 113 | # balancer. 114 | # - Can be either the key content or a path to the key 115 | # - If a path is provided, path is relative to the BBL_STATE_DIR 116 | 117 | LB_DOMAIN: 118 | # - Required if `SKIP_LB_CREATION` is false 119 | # - The domain which bbl will register 120 | 121 | BBL_ENV_NAME: 122 | # - Optional 123 | # - A label to apply to the bbl environment 124 | # - Label will appear in the IaaS metadata/interface 125 | 126 | GIT_COMMIT_EMAIL: "cf-release-integration@pivotal.io" 127 | GIT_COMMIT_USERNAME: "CI Bot" 128 | GIT_COMMIT_MESSAGE: "Update bbl state dir" 129 | # - Optional 130 | # - You may choose the git committer 131 | # username, email address and message by setting these 132 | 133 | SKIP_LB_CREATION: false 134 | # - Optional 135 | # - Set to `true` to skip load balancer creation, 136 | # - for example, a BOSH Lite environment 137 | 138 | DEBUG_MODE: false 139 | # - Optional 140 | # - BE CAREFUL! For public pipelines it can cause credentials disclosure. 141 | # - With `false` all output from BBL calls will be saved in log files inside concourse job only. 142 | # - Set this to `true` to see all output straight in the pipeline. 143 | 144 | STORE_BBL_STATE_AS_TARBALL: false 145 | # - Optional 146 | # - Creates a tarball of the bbl-state directory, instead of treating it 147 | # - as a git repo and committing. 148 | # - This is useful if you want to store your state file in S3 or GCS. 149 | 150 | DELETE_TERRAFORM_PLUGINS: true 151 | # - Optional 152 | # - Deletes Terraform plugins in the bbl state directory before pushing or uploading state 153 | # - These plugins are very large (around 50 MB for the Google plugin, for instance) 154 | # - bbl will replace them on its next operation 155 | # - Set this to `false` to keep these plugins 156 | 157 | BBL_JSON_CONFIG: 158 | # - Optional 159 | # - For if you need a dynamic config for BBL 160 | 161 | TRUSTED_CA: 162 | # - Optional 163 | # - For if you need an additional CA cert to be trusted by BOSH VMs 164 | -------------------------------------------------------------------------------- /bosh-cleanup/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exu 3 | 4 | # shellcheck disable=SC1091 5 | source cf-deployment-concourse-tasks/shared-functions 6 | 7 | function bosh_clean_up() { 8 | if [ -z "${CLEAN_ALL}" ] || [ "${CLEAN_ALL}" = true ]; then 9 | bosh -n clean-up --all 10 | else 11 | bosh -n clean-up 12 | fi 13 | } 14 | 15 | function main() { 16 | setup_bosh_env_vars 17 | bosh_clean_up 18 | } 19 | 20 | main 21 | -------------------------------------------------------------------------------- /bosh-cleanup/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks 12 | - name: bbl-state # - The repo containing the Director's bbl state dir 13 | optional: true 14 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 15 | optional: true 16 | 17 | run: 18 | path: cf-deployment-concourse-tasks/bosh-cleanup/task 19 | 20 | params: 21 | BBL_STATE_DIR: bbl-state 22 | # - Path to the directory containing the `bbl-state.json` file 23 | # - The path is relative to the `bbl-state` input 24 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 25 | # - If the specified path does not exist, it will be created 26 | # - This input and parameter will be ignored if toolsmiths-env is provided 27 | 28 | CLEAN_ALL: true 29 | # - Optional 30 | # - Boolean determines whether to use `--all` parameter to 31 | # - `bosh clean-up` 32 | -------------------------------------------------------------------------------- /bosh-delete-deployment/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exu 3 | 4 | # shellcheck disable=SC1091 5 | source cf-deployment-concourse-tasks/shared-functions 6 | 7 | function bosh_delete_all_deployments() { 8 | local deployments 9 | deployments=$(bosh deployments --json | jq -r '.Tables[].Rows[].name') 10 | 11 | local force_flag="" 12 | 13 | if [ "$IGNORE_ERRORS" = true ]; then 14 | force_flag="--force" 15 | fi 16 | 17 | for deployment in ${deployments}; do 18 | if [ -n "${deployment}" ]; then 19 | echo "Deleting deployment: ${deployment}" 20 | bosh \ 21 | -n \ 22 | delete-deployment -d "${deployment}" \ 23 | ${force_flag} 24 | fi 25 | done 26 | } 27 | 28 | function check_delete_deployment_params() { 29 | if [ -z "$DEPLOYMENT_NAME" ]; then 30 | echo "DEPLOYMENT_NAME has not been set" 31 | exit 1 32 | fi 33 | } 34 | 35 | function bosh_delete_deployment() { 36 | local force_flag="" 37 | 38 | if [ "$IGNORE_ERRORS" = true ]; then 39 | force_flag="--force" 40 | fi 41 | 42 | bosh \ 43 | -n \ 44 | delete-deployment -d ${DEPLOYMENT_NAME} \ 45 | ${force_flag} 46 | } 47 | 48 | function main() { 49 | load_from_json_config 50 | 51 | setup_bosh_env_vars 52 | if [ "$DELETE_ALL_DEPLOYMENTS" = true ]; then 53 | bosh_delete_all_deployments 54 | else 55 | check_delete_deployment_params 56 | bosh_delete_deployment 57 | fi 58 | } 59 | 60 | main 61 | -------------------------------------------------------------------------------- /bosh-delete-deployment/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: bbl-state # - The repo containing the Director's bbl state dir 12 | optional: true 13 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 14 | optional: true 15 | - name: cf-deployment-concourse-tasks # - This repo 16 | - name: pool-lock 17 | optional: true 18 | 19 | run: 20 | path: cf-deployment-concourse-tasks/bosh-delete-deployment/task 21 | 22 | params: 23 | BBL_STATE_DIR: bbl-state 24 | # - Path to the directory containing the `bbl-state.json` file 25 | # - The path is relative to the `bbl-state` input 26 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 27 | # - If the specified path does not exist, it will be created 28 | # - This input and parameter will be ignored if toolsmiths-env is provided 29 | 30 | DELETE_ALL_DEPLOYMENTS: false 31 | # - Optional 32 | # - When true, deletes ALL BOSH deployments 33 | 34 | DEPLOYMENT_NAME: cf 35 | 36 | IGNORE_ERRORS: false 37 | # - Optional 38 | # - When true, passes the `--force` flag to the delete-deployment BOSH command 39 | 40 | BBL_JSON_CONFIG: 41 | # - Optional 42 | # - For if you need a dynamic config for BBL 43 | -------------------------------------------------------------------------------- /bosh-deploy-with-created-release/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | # shellcheck disable=SC1091 4 | source cf-deployment-concourse-tasks/shared-functions 5 | 6 | optional_submodule_bump() { 7 | set +x 8 | if [ -z "${BUMP_SUBMODULE}" ] && ! [ -z "${SUBMODULE_BRANCH}" ]; then 9 | echo "SUBMODULE_BRANCH must not be set without a path set for BUMP_SUBMODULE" 10 | exit 1 11 | fi 12 | 13 | if [ -z "${SUBMODULE_BRANCH}" ] && ! [ -z "${BUMP_SUBMODULE}" ]; then 14 | echo "BUMP_SUBMODULE must not be set without a ref set for SUBMODULE_BRANCH" 15 | exit 1 16 | fi 17 | set -x 18 | 19 | if ! [ -z "${BUMP_SUBMODULE}" ]; then 20 | pushd "release/${BUMP_SUBMODULE}" 21 | git checkout "${SUBMODULE_BRANCH}" 22 | git pull 23 | popd 24 | fi 25 | } 26 | 27 | main() { 28 | local root_dir 29 | root_dir="${1}" 30 | 31 | check_input_params 32 | optional_submodule_bump 33 | setup_bosh_env_vars 34 | 35 | local release_name 36 | release_name="$(yq -r '.name' release/config/final.yml)" 37 | if [[ "${release_name}" == "null" ]]; then 38 | release_name="$(yq -r '.final_name' release/config/final.yml)" 39 | fi 40 | if [[ "${release_name}" == "null" ]]; then 41 | echo "Expected non-empty 'name' or 'final_name' in release/config/final.yml" 42 | exit 1 43 | fi 44 | 45 | if [[ -r "${root_dir}/release_tarball_name/name" ]]; then 46 | bosh_interpolate "${root_dir}" "${release_name}" "$(cat ${root_dir}/release_tarball_name/name)" 47 | else 48 | bosh_interpolate "${root_dir}" "${release_name}" 49 | fi 50 | 51 | if [ "$REGENERATE_CREDENTIALS" == true ]; then 52 | remove_credentials_from_credhub 53 | fi 54 | upload_stemcells 55 | # shellcheck disable=SC2086 56 | bosh_deploy ${BOSH_DEPLOY_ARGS} 57 | } 58 | 59 | main "${PWD}" 60 | -------------------------------------------------------------------------------- /bosh-deploy-with-created-release/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: bbl-state # - The repo containing the Director's bbl state dir 12 | optional: true 13 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 14 | optional: true 15 | - name: cf-deployment # - The cf-deployment manifest 16 | - name: ops-files # - Operations files to be made available 17 | optional: true 18 | - name: vars-files # - Variable files to be made available 19 | optional: true 20 | - name: release 21 | # - BOSH release source repo or directory containing a release tarball 22 | # - If the input is a git repository, a dev release will be created from 23 | # this repo and used in the deployment. If the input is a directory 24 | # containing a pre-built release tarball, then the release_tarball_name 25 | # input needs to point at a directory containing a "name" file containing 26 | # the full name of the tarball to be used. 27 | - name: release_tarball_name 28 | optional: true 29 | - name: cf-deployment-concourse-tasks # - This repo 30 | 31 | run: 32 | path: cf-deployment-concourse-tasks/bosh-deploy-with-created-release/task 33 | 34 | params: 35 | BBL_STATE_DIR: bbl-state 36 | # - Path to the directory containing the `bbl-state.json` file 37 | # - The path is relative to the `bbl-state` input 38 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 39 | # - If the specified path does not exist, it will be created 40 | # - This input and parameter will be ignored if toolsmiths-env is provided 41 | 42 | MANIFEST_FILE: cf-deployment.yml 43 | # - Required 44 | # - Filepath to the manifest file within the cf-deployment resource 45 | # - The path is relative to root of the `cf-deployment` input 46 | 47 | SYSTEM_DOMAIN: 48 | # - Required unless toolsmiths-env optional input is provided 49 | # - CF system base domain e.g. `my-cf.com` 50 | 51 | OPS_FILES: 52 | # - Optional 53 | # - List of operations files to be applied to the manifest 54 | # - Quoted and space-separated 55 | # - Ops will be applied in the order they're listed 56 | # - Paths are relative to root of the `ops-files` input 57 | 58 | VARS_FILES: 59 | # - Optional 60 | # - List of variable files to be applied to the manifest 61 | # - Quoted and space-separated 62 | # - Variable files will be applied in the order they're listed 63 | # - Paths are relative to root of the `vars-files` input 64 | 65 | REGENERATE_CREDENTIALS: false 66 | # - Optional 67 | # - Delete credentials associated with this deployment from CredHub 68 | # - This allows the credentials to be freshly generated each time 69 | # - This is helpful for testing changes around variable generation 70 | # - Works well with fresh deployments 71 | # - Upgrade deployments are not expected to work with total cred rotation 72 | 73 | BUMP_SUBMODULE: "" 74 | # - Optional 75 | # - Set to the path of a submodule you want bumped 76 | # prior to creating the dev release. 77 | # - Also requires that you set `SUBMODULE_BRANCH` 78 | SUBMODULE_BRANCH: "" 79 | # - Ignored unless `BUMP_SUBMODULE` is set 80 | # - Required if `BUMP_SUBMODULE` is set 81 | # - Set to the ref you want the submodule checked out to. 82 | # This ref will be used as an argument to `git checkout` 83 | # when updating your submodule. 84 | 85 | DEPLOY_WITH_UPTIME_MEASUREMENTS: false 86 | # - Optional 87 | # - This will measure uptime during the bosh deploy task using [uptimer](https://github.com/cloudfoundry/uptimer) 88 | 89 | FAIL_ON_DOWNTIME: false 90 | # - Optional 91 | # - This will cause the task to fail if you are not using the DEPLOY_WITH_UPTIME_MEASUREMENTS flag 92 | # - It will cause the concourse task to fail if you do not have perfect uptime 93 | 94 | MEASURE_SYSLOG_AVAILABILITY: false 95 | # - Optional 96 | # - Requires DEPLOY_WITH_UPTIME_MEASUREMENTS to be true. 97 | # - This will measure availability of app syslog drain during deployment. 98 | 99 | MEASURE_TCP_AVAILABILITY: false 100 | # - Optional 101 | # - Requires DEPLOY_WITH_UPTIME_MEASUREMENTS to be true. 102 | # - This will measure availability of TCP routing during deployment. 103 | 104 | TCP_DOMAIN: 105 | # - Required if MEASURE_TCP_AVAILABILITY or MEASURE_SYSLOG_AVAILABILITY is set to true, 106 | # unless toolsmiths-env is provided. With toolsmiths-env, this value is automatically calculated. 107 | # - Domain used by CF for creating tcp routes to apps, syslog. 108 | # - Usually of the form `tcp.[SYSTEM_DOMAIN]` (e.g. `tcp.my-cf.com`). 109 | 110 | TCP_PORT: 111 | # - Required if MEASURE_TCP_AVAILABILITY is set to true. 112 | # - Available port within the 'reservable_ports' range, 113 | # - used to create a tcp route for TCP routing availability measurement. 114 | 115 | AVAILABLE_PORT: 116 | # - Required if MEASURE_SYSLOG_AVAILABILITY is set to true. 117 | # - Available port within the 'reservable_ports' range, 118 | # - used to create a tcp route for the syslog availability measurement. 119 | 120 | APP_PUSHABILITY_THRESHOLD: 0 121 | # - Optional 122 | # - This sets the maximum number of allowed uptimer failures for app pushability 123 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 124 | 125 | APP_STATS_THRESHOLD: 0 126 | # - Optional 127 | # - This sets the maximum number of allowed uptimer failures for app stats 128 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 129 | 130 | HTTP_AVAILABILITY_THRESHOLD: 0 131 | # - Optional 132 | # - This sets the maximum number of allowed uptimer failures for http availability 133 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 134 | 135 | TCP_AVAILABILITY_THRESHOLD: 0 136 | # - Optional 137 | # - This sets the maximum number of allowed uptimer failures for tcp availability 138 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 139 | 140 | RECENT_LOGS_THRESHOLD: 0 141 | # - Optional 142 | # - This sets the maximum number of allowed uptimer failures for recent logs 143 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 144 | 145 | STREAMING_LOGS_THRESHOLD: 0 146 | # - Optional 147 | # - This sets the maximum number of allowed uptimer failures for streaming logs 148 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 149 | 150 | APP_SYSLOG_AVAILABILITY_THRESHOLD: 0 151 | # - Optional 152 | # - This sets the maximum number of allowed uptimer failures for app syslog availability 153 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 154 | 155 | USE_SINGLE_APP_INSTANCE: false 156 | # - Optional 157 | # - Configures uptimer to deploy with a single app instance rather than two. 158 | # - This is primarily used by Diego to validate new uptime features. 159 | 160 | BOSH_DEPLOY_ARGS: 161 | # - Optional 162 | # - Supplies additional arguments to the bosh deploy command 163 | # - For example, '--recreate' 164 | 165 | BOSH_LITE: false 166 | # - Optional 167 | # - Used to determine whether we should upload a warden stemcell 168 | # - Boolean 169 | -------------------------------------------------------------------------------- /bosh-deploy-with-updated-release-submodule/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | # shellcheck disable=SC1091 4 | source cf-deployment-concourse-tasks/shared-functions 5 | 6 | submodule_bump() { 7 | rm -rf "release/${BUMP_SUBMODULE}" 8 | git clone submodule "release/${BUMP_SUBMODULE}" 9 | } 10 | 11 | main() { 12 | local root_dir 13 | root_dir="${1}" 14 | 15 | if [ -z "$BUMP_SUBMODULE" ]; then 16 | echo "BUMP_SUBMODULE has not been set" 17 | exit 1 18 | fi 19 | 20 | check_input_params 21 | 22 | submodule_bump 23 | setup_bosh_env_vars 24 | bosh_interpolate "${root_dir}" "$(grep final_name release/config/final.yml | awk '{print $2}')" 25 | if [ "$REGENERATE_CREDENTIALS" == true ]; then 26 | remove_credentials_from_credhub 27 | fi 28 | upload_stemcells 29 | # shellcheck disable=SC2086 30 | bosh_deploy ${BOSH_DEPLOY_ARGS} 31 | } 32 | 33 | main "${PWD}" 34 | -------------------------------------------------------------------------------- /bosh-deploy-with-updated-release-submodule/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: bbl-state # - The repo containing the Director's bbl state dir 12 | optional: true 13 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 14 | optional: true 15 | - name: cf-deployment # - The cf-deployment manifest 16 | - name: ops-files # - Operations files to be made available 17 | optional: true 18 | - name: vars-files # - Variable files to be made available 19 | optional: true 20 | - name: release 21 | # - BOSH release source repo 22 | # - A dev release will be created from this repo and used in the deployment 23 | - name: cf-deployment-concourse-tasks # - This repo 24 | - name: submodule # - The resource that will be placed at the BUMP_SUBMODULE path 25 | 26 | run: 27 | path: cf-deployment-concourse-tasks/bosh-deploy-with-updated-release-submodule/task 28 | 29 | params: 30 | BBL_STATE_DIR: bbl-state 31 | # - Path to the directory containing the `bbl-state.json` file 32 | # - The path is relative to the `bbl-state` input 33 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 34 | # - If the specified path does not exist, it will be created 35 | # - This input and parameter will be ignored if toolsmiths-env is provided 36 | 37 | MANIFEST_FILE: cf-deployment.yml 38 | # - Required 39 | # - Filepath to the manifest file within the cf-deployment resource 40 | # - The path is relative to root of the `cf-deployment` input 41 | 42 | SYSTEM_DOMAIN: 43 | # - Required unless toolsmiths-env optional input is provided 44 | # - CF system base domain e.g. `my-cf.com` 45 | 46 | OPS_FILES: 47 | # - Optional 48 | # - List of operations files to be applied to the manifest 49 | # - Quoted and space-separated 50 | # - Ops will be applied in the order they're listed 51 | # - Paths are relative to root of the `ops-files` input 52 | 53 | VARS_FILES: 54 | # - Optional 55 | # - List of variable files to be applied to the manifest 56 | # - Quoted and space-separated 57 | # - Variable files will be applied in the order they're listed 58 | # - Paths are relative to root of the `vars-files` input 59 | 60 | REGENERATE_CREDENTIALS: false 61 | # - Optional 62 | # - Delete credentials associated with this deployment from CredHub 63 | # - This allows the credentials to be freshly generated each time 64 | # - This is helpful for testing changes around variable generation 65 | # - Works well with fresh deployments 66 | # - Upgrade deployments are not expected to work with total cred rotation 67 | 68 | BUMP_SUBMODULE: "" 69 | # - Required 70 | # - Set to the path of a submodule you want bumped 71 | # prior to creating the dev release. 72 | 73 | DEPLOY_WITH_UPTIME_MEASUREMENTS: false 74 | # - Optional 75 | # - This will measure uptime during the bosh deploy task using [uptimer](https://github.com/cloudfoundry/uptimer) 76 | 77 | FAIL_ON_DOWNTIME: false 78 | # - Optional 79 | # - This will cause the task to fail if you are not using the DEPLOY_WITH_UPTIME_MEASUREMENTS flag 80 | # - It will cause the concourse task to fail if you do not have perfect uptime 81 | 82 | MEASURE_SYSLOG_AVAILABILITY: false 83 | # - Optional 84 | # - Requires DEPLOY_WITH_UPTIME_MEASUREMENTS to be true. 85 | # - This will measure availability of app syslog drain during deployment. 86 | 87 | MEASURE_TCP_AVAILABILITY: false 88 | # - Optional 89 | # - Requires DEPLOY_WITH_UPTIME_MEASUREMENTS to be true. 90 | # - This will measure availability of TCP routing during deployment. 91 | 92 | TCP_DOMAIN: 93 | # - Required if MEASURE_TCP_AVAILABILITY or MEASURE_SYSLOG_AVAILABILITY is set to true, 94 | # unless toolsmiths-env is provided. With toolsmiths-env, this value is automatically calculated. 95 | # - Domain used by CF for creating tcp routes to apps, syslog. 96 | # - Usually of the form `tcp.[SYSTEM_DOMAIN]` (e.g. `tcp.my-cf.com`). 97 | 98 | TCP_PORT: 99 | # - Required if MEASURE_TCP_AVAILABILITY is set to true. 100 | # - Available port within the 'reservable_ports' range, 101 | # - used to create a tcp route for TCP routing availability measurement. 102 | 103 | 104 | AVAILABLE_PORT: 105 | # - Required if MEASURE_SYSLOG_AVAILABILITY is set to true. 106 | # - Available port within the 'reservable_ports' range, 107 | # - used to create a tcp route for the syslog availability measurement. 108 | 109 | APP_PUSHABILITY_THRESHOLD: 0 110 | # - Optional 111 | # - This sets the maximum number of allowed uptimer failures for app pushability 112 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 113 | 114 | APP_STATS_THRESHOLD: 0 115 | # - Optional 116 | # - This sets the maximum number of allowed uptimer failures for app stats 117 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 118 | 119 | HTTP_AVAILABILITY_THRESHOLD: 0 120 | # - Optional 121 | # - This sets the maximum number of allowed uptimer failures for http availability 122 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 123 | 124 | TCP_AVAILABILITY_THRESHOLD: 0 125 | # - Optional 126 | # - This sets the maximum number of allowed uptimer failures for tcp availability 127 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 128 | 129 | RECENT_LOGS_THRESHOLD: 0 130 | # - Optional 131 | # - This sets the maximum number of allowed uptimer failures for recent logs 132 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 133 | 134 | STREAMING_LOGS_THRESHOLD: 0 135 | # - Optional 136 | # - This sets the maximum number of allowed uptimer failures for streaming logs 137 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 138 | 139 | APP_SYSLOG_AVAILABILITY_THRESHOLD: 0 140 | # - Optional 141 | # - This sets the maximum number of allowed uptimer failures for app syslog availability 142 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 143 | 144 | USE_SINGLE_APP_INSTANCE: false 145 | # - Optional 146 | # - Configures uptimer to deploy with a single app instance rather than two. 147 | # - This is primarily used by Diego to validate new uptime features. 148 | 149 | BOSH_DEPLOY_ARGS: 150 | # - Optional 151 | # - Supplies additional arguments to the bosh deploy command 152 | # - For example, '--recreate' 153 | 154 | BOSH_LITE: false 155 | # - Optional 156 | # - Used to determine whether we should upload a warden stemcell 157 | # - Boolean 158 | -------------------------------------------------------------------------------- /bosh-deploy/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | # shellcheck disable=SC1091 4 | source cf-deployment-concourse-tasks/shared-functions 5 | 6 | function main() { 7 | load_from_json_config 8 | 9 | check_input_params 10 | setup_bosh_env_vars 11 | bosh_interpolate 12 | if [ "$REGENERATE_CREDENTIALS" == true ]; then 13 | remove_credentials_from_credhub 14 | fi 15 | if [ ! "$SKIP_STEMCELL_UPLOAD" == true ]; then 16 | upload_stemcells 17 | fi 18 | # shellcheck disable=SC2086 19 | bosh_deploy ${BOSH_DEPLOY_ARGS} 20 | } 21 | 22 | main 23 | -------------------------------------------------------------------------------- /bosh-deploy/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: bbl-state # - The repo containing the Director's bbl state dir 12 | optional: true 13 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 14 | optional: true 15 | - name: cf-deployment # - The cf-deployment manifest 16 | - name: ops-files # - Operations files to be made available 17 | optional: true 18 | - name: vars-files # - Variable files to be made available 19 | optional: true 20 | - name: cf-deployment-concourse-tasks # - This repo 21 | - name: pool-lock 22 | optional: true 23 | 24 | run: 25 | path: cf-deployment-concourse-tasks/bosh-deploy/task 26 | 27 | params: 28 | BBL_STATE_DIR: bbl-state 29 | # - Path to the directory containing the `bbl-state.json` file 30 | # - The path is relative to the `bbl-state` input 31 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 32 | # - If the specified path does not exist, it will be created 33 | # - This input and parameter will be ignored if toolsmiths-env is provided 34 | 35 | MANIFEST_FILE: cf-deployment.yml 36 | # - Required 37 | # - Filepath to the manifest file within the cf-deployment resource 38 | # - The path is relative to root of the `cf-deployment` input 39 | 40 | SYSTEM_DOMAIN: 41 | # - Required unless toolsmiths-env optional input is provided 42 | # - CF system base domain e.g. `my-cf.com` 43 | 44 | OPS_FILES: 45 | # - Optional 46 | # - List of operations files to be applied to the manifest 47 | # - Quoted and space-separated 48 | # - Ops will be applied in the order they're listed 49 | # - Paths are relative to root of the `ops-files` input 50 | 51 | VARS_FILES: 52 | # - Optional 53 | # - List of variable files to be applied to the manifest 54 | # - Quoted and space-separated 55 | # - Variable files will be applied in the order they're listed 56 | # - Paths are relative to root of the `vars-files` input 57 | 58 | REGENERATE_CREDENTIALS: false 59 | # - Optional 60 | # - Delete credentials associated with this deployment from CredHub 61 | # - This allows the credentials to be freshly generated each time 62 | # - This is helpful for testing changes around variable generation 63 | # - Works well with fresh deployments 64 | # - Upgrade deployments are not expected to work with total cred rotation 65 | 66 | SKIP_STEMCELL_UPLOAD: false 67 | # - Optional 68 | # - Skip stemcell upload, if stemcell has already been uploaded to BOSH director. 69 | 70 | 71 | # Uptimer Params 72 | # https://github.com/cloudfoundry/uptimer 73 | DEPLOY_WITH_UPTIME_MEASUREMENTS: false 74 | # - Optional 75 | # - This will measure uptime during the bosh deploy task. 76 | 77 | MEASURE_SYSLOG_AVAILABILITY: false 78 | # - Optional 79 | # - Requires DEPLOY_WITH_UPTIME_MEASUREMENTS to be true. 80 | # - This will measure availability of app syslog drain during deployment. 81 | 82 | MEASURE_TCP_AVAILABILITY: false 83 | # - Optional 84 | # - Requires DEPLOY_WITH_UPTIME_MEASUREMENTS to be true. 85 | # - This will measure availability of TCP routing during deployment. 86 | 87 | TCP_DOMAIN: 88 | # - Required if MEASURE_TCP_AVAILABILITY or MEASURE_SYSLOG_AVAILABILITY is set to true, 89 | # unless toolsmiths-env is provided. With toolsmiths-env, this value is automatically calculated. 90 | # - Domain used by CF for creating tcp routes to apps, syslog. 91 | # - Usually of the form `tcp.[SYSTEM_DOMAIN]` (e.g. `tcp.my-cf.com`). 92 | 93 | TCP_PORT: 94 | # - Required if MEASURE_TCP_AVAILABILITY is set to true. 95 | # - Available port within the 'reservable_ports' range, 96 | # - used to create a tcp route for TCP routing availability measurement. 97 | 98 | AVAILABLE_PORT: 99 | # - Required if MEASURE_SYSLOG_AVAILABILITY is set to true. 100 | # - Available port within the 'reservable_ports' range, 101 | # - used to create a tcp route for the syslog availability measurement. 102 | 103 | FAIL_ON_DOWNTIME: false 104 | # - Optional 105 | # - This will cause the task to fail if you are not using the DEPLOY_WITH_UPTIME_MEASUREMENTS flag 106 | # - It will cause the concourse task to fail if you do not have perfect uptime 107 | 108 | APP_PUSHABILITY_THRESHOLD: 0 109 | # - Optional 110 | # - This sets the maximum number of allowed uptimer failures for app pushability 111 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 112 | 113 | APP_STATS_THRESHOLD: 0 114 | # - Optional 115 | # - This sets the maximum number of allowed uptimer failures for app stats 116 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 117 | 118 | HTTP_AVAILABILITY_THRESHOLD: 0 119 | # - Optional 120 | # - This sets the maximum number of allowed uptimer failures for http availability 121 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 122 | 123 | TCP_AVAILABILITY_THRESHOLD: 0 124 | # - Optional 125 | # - This sets the maximum number of allowed uptimer failures for tcp availability 126 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 127 | 128 | RECENT_LOGS_THRESHOLD: 0 129 | # - Optional 130 | # - This sets the maximum number of allowed uptimer failures for recent logs 131 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 132 | 133 | STREAMING_LOGS_THRESHOLD: 0 134 | # - Optional 135 | # - This sets the maximum number of allowed uptimer failures for streaming logs 136 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 137 | 138 | APP_SYSLOG_AVAILABILITY_THRESHOLD: 0 139 | # - Optional 140 | # - This sets the maximum number of allowed uptimer failures for app syslog availability 141 | # - The default threshold value of 0 is our working value for cf-deployment, and is likely to be updated 142 | 143 | USE_SINGLE_APP_INSTANCE: false 144 | # - Optional 145 | # - Configures uptimer to deploy with a single app instance rather than two. 146 | # - This is primarily used by Diego to validate new uptime features. 147 | 148 | BOSH_DEPLOY_ARGS: 149 | # - Optional 150 | # - Supplies additional arguments to the bosh deploy command 151 | # - For example, '--recreate' 152 | 153 | BOSH_LITE: false 154 | # - Optional 155 | # - Used to determine whether we should upload a warden stemcell 156 | # - Boolean 157 | 158 | BBL_JSON_CONFIG: 159 | # - Optional 160 | # - For if you need a dynamic config for BBL 161 | -------------------------------------------------------------------------------- /bosh-upload-stemcell-from-cf-deployment: -------------------------------------------------------------------------------- 1 | bosh-upload-stemcells/ -------------------------------------------------------------------------------- /bosh-upload-stemcells/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | 4 | # shellcheck disable=SC1091 5 | source cf-deployment-concourse-tasks/shared-functions 6 | 7 | check_upload_stemcell_params() { 8 | if [ -z "$INFRASTRUCTURE" ]; then 9 | echo "INFRASTRUCTURE has not been set" 10 | exit 1 11 | fi 12 | local supported_infrastructures 13 | supported_infrastructures=("aws" "google" "boshlite" "bosh-lite" "vsphere" "azure" "openstack") 14 | any_matched=false 15 | for iaas in ${supported_infrastructures[*]}; do 16 | if [ "${INFRASTRUCTURE}" == "${iaas}" ]; then 17 | any_matched=true 18 | break 19 | fi 20 | done 21 | if [ "$any_matched" = false ]; then 22 | echo "${INFRASTRUCTURE} is not supported; please choose a value from ${supported_infrastructures[*]}" 23 | exit 1 24 | fi 25 | } 26 | 27 | verify_ops_file_modifies_stemcell_section() { 28 | local ops_file 29 | ops_file=$1 30 | 31 | set +e 32 | if ! grep 'path: /stemcells/' ops-files/${ops_file} > /dev/null; then 33 | echo "Error: Could not find any stemcells to be uploaded from ${ops_file} ops file." 34 | exit 1 35 | fi 36 | set -e 37 | } 38 | 39 | upload_stemcell() { 40 | # Read potentially variable stemcell paramaters out of cf-deployment with bosh 41 | local os 42 | os=$1 43 | local version 44 | version=$2 45 | 46 | # Hardcode a couple of stable stemcell paramaters 47 | local stemcells_url 48 | stemcells_url="https://bosh.io/d/stemcells" 49 | 50 | # Ask bosh if it already has our OS / version stemcell combination 51 | # As of this writing, the new bosh cli doesn't have --skip-if-exists 52 | set +e 53 | local existing_stemcell 54 | existing_stemcell=$(bosh stemcells | grep "${os}" | awk '{print $2}' | tr -d "\*" | grep ^"${version}"$ ) 55 | set -e 56 | 57 | local stemcell_name 58 | 59 | if [ "$INFRASTRUCTURE" = "aws" ]; then 60 | stemcell_name="bosh-aws-xen-hvm" 61 | elif [ "$INFRASTRUCTURE" = "google" ]; then 62 | stemcell_name="bosh-google-kvm" 63 | elif [ "$INFRASTRUCTURE" = "boshlite" ]; then 64 | stemcell_name="bosh-warden-boshlite" 65 | elif [ "$INFRASTRUCTURE" = "bosh-lite" ]; then 66 | stemcell_name="bosh-warden-boshlite" 67 | elif [ "$INFRASTRUCTURE" = "vsphere" ]; then 68 | stemcell_name="bosh-vsphere-esxi" 69 | elif [ "$INFRASTRUCTURE" = "azure" ]; then 70 | stemcell_name="bosh-azure-hyperv" 71 | elif [ "$INFRASTRUCTURE" = "openstack" ]; then 72 | stemcell_name="bosh-openstack-kvm" 73 | fi 74 | 75 | stemcell_name="${stemcell_name}-${os}-go_agent" 76 | if [ "$version" = "latest" ]; then 77 | full_stemcell_url="${stemcells_url}/${stemcell_name}" 78 | else 79 | full_stemcell_url="${stemcells_url}/${stemcell_name}?v=${version}" 80 | fi 81 | 82 | # If bosh already has our stemcell, exit 0 83 | if [ "${existing_stemcell}" ]; then 84 | echo "Task bosh-upload-stemcell-from-cf-deployment:" 85 | echo "Stemcell '${stemcell_name}/${version}' already exists. Skipping..." 86 | return 87 | fi 88 | 89 | # If bosh.io doesn't have our stemcell, exit 2 90 | set +e 91 | local stemcell_exists 92 | wget -S --spider "${full_stemcell_url}" > /dev/null 2>&1 93 | stemcell_exists=$? 94 | if [ ${stemcell_exists} -ne 0 ]; then 95 | echo "Error: Could not find a '$os' stemcell for IaaS '$INFRASTRUCTURE' on bosh.io. Please double-check that the IaaS/OS combination is supported." 96 | exit 2 #POSIX 'No such file or directory' 97 | fi 98 | set -e 99 | 100 | # ... otherwise, begin the upload process 101 | bosh \ 102 | -n \ 103 | upload-stemcell \ 104 | "${full_stemcell_url}" 105 | } 106 | 107 | function upload_stemcells() { 108 | local arguments='' 109 | for op in ${OPS_FILES} 110 | do 111 | verify_ops_file_modifies_stemcell_section ${op} 112 | arguments="${arguments} -o ops-files/${op}" 113 | done 114 | 115 | bosh interpolate ${arguments} cf-deployment/${MANIFEST_FILE} > /tmp/cf.yml 116 | 117 | local stemcells_json=$(ruby -rjson -ryaml -e "puts YAML.load_file('/tmp/cf.yml').to_json" | jq .stemcells) 118 | local size=$(echo ${stemcells_json} | jq 'length') 119 | if [ ${size} -eq 0 ]; then 120 | echo "Error: Did not find any stemcells to upload." 121 | exit 1 122 | fi 123 | 124 | for i in `seq 0 $((size - 1))` 125 | do 126 | local os=$(echo ${stemcells_json} | jq -r .[$i].os) 127 | local version=$(echo ${stemcells_json} | jq -r .[$i].version) 128 | upload_stemcell ${os} ${version} 129 | done 130 | } 131 | 132 | main() { 133 | echo "This task has been deprecated! Stemcells are now uploaded as part of 'bosh-deploy' task." 134 | check_upload_stemcell_params 135 | setup_bosh_env_vars 136 | upload_stemcells 137 | } 138 | 139 | main 140 | -------------------------------------------------------------------------------- /bosh-upload-stemcells/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks # - This repo 12 | - name: cf-deployment # - The cf-deployment manifest 13 | - name: bbl-state # - The repo containing the Director's bbl state dir 14 | optional: true 15 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 16 | optional: true 17 | - name: ops-files # - Oplitonal operations files to be made available 18 | optional: true 19 | 20 | run: 21 | path: cf-deployment-concourse-tasks/bosh-upload-stemcells/task 22 | 23 | params: 24 | BBL_STATE_DIR: bbl-state 25 | # - Path to the directory containing the `bbl-state.json` file 26 | # - The path is relative to the `bbl-state` input 27 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 28 | # - If the specified path does not exist, it will be created 29 | # - This input and parameter will be ignored if toolsmiths-env is provided 30 | 31 | INFRASTRUCTURE: google 32 | # - Required 33 | # - Used to determine which stemcell will be uploaded 34 | # - Must be one of: aws, bosh-lite, google, vsphere, azure or openstack 35 | 36 | MANIFEST_FILE: cf-deployment.yml 37 | # - Required 38 | # - Filepath to the manifest file within the cf-deployment resource 39 | # - The path is relative to root of the `cf-deployment` input 40 | 41 | OPS_FILES: 42 | # - Optional 43 | # - List of operations files to be applied to the manifest. 44 | # - Additional stemcells specified by operations will also be uploaded. 45 | # - Quoted and space-separated 46 | # - Ops will be applied in the order they're listed 47 | # - Paths are relative to root of the `ops-files` input 48 | -------------------------------------------------------------------------------- /bump_docker_images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | OLD_VERSION="${1?: Please define an old version}" 4 | NEW_VERSION="${2?: Please define a new version}" 5 | 6 | MAJOR_VERSION="$(echo "$NEW_VERSION" | cut -d'.' -f1)" 7 | 8 | docker tag cloudfoundry/cf-deployment-concourse-tasks:latest cloudfoundry/cf-deployment-concourse-tasks:$NEW_VERSION 9 | docker tag cloudfoundry/cf-deployment-concourse-tasks:latest cloudfoundry/cf-deployment-concourse-tasks:$MAJOR_VERSION 10 | 11 | docker push cloudfoundry/cf-deployment-concourse-tasks:$NEW_VERSION 12 | docker push cloudfoundry/cf-deployment-concourse-tasks:$MAJOR_VERSION 13 | 14 | perl -pi -e "s/$OLD_VERSION/$NEW_VERSION/g" **/task.yml 15 | git add */task.yml 16 | -------------------------------------------------------------------------------- /ci/configure: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | function main() { 4 | fly -t ard sp -p cf-deployment-concourse-tasks -c "${PROJECT_DIR}/ci/pipeline.yml" 5 | } 6 | 7 | main "$@" 8 | -------------------------------------------------------------------------------- /ci/pipeline.md: -------------------------------------------------------------------------------- 1 | # cf-deployment-concourse-tasks 2 | 3 | ## Context 4 | Changes to the tasks are actually tested by having our cf-d pipeline consume cf-dt/master so we get to see problems early. 5 | 6 | notify-bbl-updates; this job notifies the team if/when bbl (and in this pipeline we are interested in bbl cli) changes occur. -------------------------------------------------------------------------------- /ci/pipeline.yml: -------------------------------------------------------------------------------- 1 | resources: 2 | - name: cf-deployment-concourse-tasks-dockerfile 3 | type: git 4 | icon: github 5 | source: 6 | branch: main 7 | uri: git@github.com:cloudfoundry/cf-deployment-concourse-tasks.git 8 | private_key: ((ard_wg_gitbot_ssh_key.private_key)) 9 | paths: 10 | - dockerfiles/cf-deployment-concourse-tasks/Dockerfile 11 | 12 | - name: bbl-github-release 13 | type: github-release 14 | icon: github 15 | source: 16 | user: cloudfoundry 17 | repository: bosh-bootloader 18 | access_token: ((ard_wg_gitbot_token)) 19 | 20 | - name: bosh-cli-github-release 21 | type: github-release 22 | icon: github 23 | source: 24 | user: cloudfoundry 25 | repository: bosh-cli 26 | access_token: ((ard_wg_gitbot_token)) 27 | 28 | - name: cf-cli-github-release 29 | type: github-release 30 | icon: github 31 | source: 32 | user: cloudfoundry 33 | repository: cli 34 | access_token: ((ard_wg_gitbot_token)) 35 | tag_filter: 'v(8\.[^v].*)' 36 | 37 | - name: credhub-cli-github-release 38 | type: github-release 39 | icon: github 40 | source: 41 | user: cloudfoundry-incubator 42 | repository: credhub-cli 43 | access_token: ((ard_wg_gitbot_token)) 44 | 45 | - name: log-cache-cli-github-release 46 | type: github-release 47 | icon: github 48 | source: 49 | user: cloudfoundry 50 | repository: log-cache-cli 51 | access_token: ((ard_wg_gitbot_token)) 52 | 53 | - name: terraform-github-release 54 | type: github-release 55 | icon: github 56 | source: 57 | user: hashicorp 58 | repository: terraform 59 | access_token: ((ard_wg_gitbot_token)) 60 | 61 | - name: yq-github-release 62 | type: github-release 63 | icon: github 64 | source: 65 | user: mikefarah 66 | repository: yq 67 | access_token: ((ard_wg_gitbot_token)) 68 | 69 | - name: uptimer 70 | type: git 71 | icon: github 72 | source: 73 | uri: https://github.com/cloudfoundry/uptimer.git 74 | branch: main 75 | 76 | - name: cf-deployment-concourse-tasks-pipeline-image 77 | type: docker-image 78 | icon: docker 79 | source: 80 | repository: cloudfoundry/cf-deployment-concourse-tasks 81 | username: ((dockerhub.username)) 82 | password: ((dockerhub.password)) 83 | email: cf-release-integration+dockerhub-push-bot@pivotal.io 84 | 85 | - name: cf-deployment-concourse-tasks-version 86 | type: semver 87 | source: 88 | driver: git 89 | uri: git@github.com:cloudfoundry/cf-relint-ci-semver.git 90 | branch: main 91 | private_key: ((ard_wg_gitbot_ssh_key.private_key)) 92 | file: cf-deployment-concourse-tasks-version 93 | 94 | - name: cf-deployment-concourse-tasks 95 | type: git 96 | icon: github 97 | source: 98 | branch: main 99 | uri: git@github.com:cloudfoundry/cf-deployment-concourse-tasks.git 100 | private_key: ((ard_wg_gitbot_ssh_key.private_key)) 101 | 102 | - name: cf-deployment-concourse-tasks-latest-release 103 | type: git 104 | icon: github 105 | source: 106 | uri: git@github.com:cloudfoundry/cf-deployment-concourse-tasks.git 107 | private_key: ((ard_wg_gitbot_ssh_key.private_key)) 108 | tag_filter: '*' 109 | 110 | - name: cf-deployment-concourse-tasks-github-release 111 | type: github-release 112 | source: 113 | owner: cloudfoundry 114 | repository: cf-deployment-concourse-tasks 115 | access_token: ((ard_wg_gitbot_token)) 116 | drafts: true 117 | 118 | jobs: 119 | - name: build-docker-image 120 | serial: true 121 | public: true 122 | plan: 123 | - in_parallel: 124 | - get: bbl-github-release 125 | trigger: true 126 | - get: bosh-cli-github-release 127 | trigger: true 128 | - get: cf-cli-github-release 129 | trigger: true 130 | - get: cf-deployment-concourse-tasks-dockerfile 131 | trigger: true 132 | - get: credhub-cli-github-release 133 | trigger: true 134 | - get: log-cache-cli-github-release 135 | trigger: true 136 | - get: terraform-github-release 137 | trigger: true 138 | - get: uptimer 139 | trigger: true 140 | - get: yq-github-release 141 | trigger: true 142 | - task: update-dockerfile-versions 143 | file: cf-deployment-concourse-tasks-dockerfile/ci/tasks/update-cf-d-ct-dockerfile-versions/task.yml 144 | input_mapping: 145 | cf-cli: cf-cli-github-release 146 | cf-deployment-concourse-tasks: cf-deployment-concourse-tasks-dockerfile 147 | - put: cf-deployment-concourse-tasks-pipeline-image 148 | params: 149 | build: cf-deployment-concourse-tasks-updated/dockerfiles/cf-deployment-concourse-tasks 150 | build_args: 151 | RELINT_CA_CERTIFICATE: ((relint_ca.certificate)) 152 | PREVIOUS_RELINT_CA_CERTIFICATE: "" 153 | push: true 154 | - put: cf-deployment-concourse-tasks-dockerfile 155 | params: 156 | repository: cf-deployment-concourse-tasks-updated 157 | rebase: true 158 | 159 | - name: ship-it-major 160 | public: true 161 | plan: 162 | - in_parallel: 163 | - get: cf-deployment-concourse-tasks-version 164 | params: 165 | bump: major 166 | - get: cf-deployment-concourse-tasks 167 | - get: cf-deployment-concourse-tasks-latest-release 168 | - get: cf-deployment-concourse-tasks-pipeline-image 169 | params: 170 | save: true 171 | passed: 172 | - build-docker-image 173 | ## push to docker with a tagged version 174 | - task: generate-major-version-file 175 | file: cf-deployment-concourse-tasks/ci/tasks/generate-major-version-file/task.yml 176 | input_mapping: 177 | semantic_version: cf-deployment-concourse-tasks-version 178 | - put: cf-deployment-concourse-tasks-pipeline-image 179 | params: 180 | load: cf-deployment-concourse-tasks-pipeline-image 181 | tag_file: major_version/version 182 | tag_prefix: v 183 | get_params: 184 | save: true 185 | - put: cf-deployment-concourse-tasks-pipeline-image 186 | params: 187 | load: cf-deployment-concourse-tasks-pipeline-image 188 | tag_file: cf-deployment-concourse-tasks-version/version 189 | tag_prefix: v 190 | ## update task yml to a updated dockerimage version and tag 191 | - task: update-docker-image-version-for-release 192 | file: cf-deployment-concourse-tasks/ci/tasks/bump-cf-d-ct-docker-image/task.yml 193 | input_mapping: 194 | version: cf-deployment-concourse-tasks-version 195 | - put: cf-deployment-concourse-tasks 196 | params: 197 | branch: main 198 | repository: cf-deployment-concourse-tasks-updated 199 | tag: cf-deployment-concourse-tasks-version/version 200 | tag_prefix: v 201 | ## update task yml to point back to latest 202 | - task: update-docker-image-version-back-to-latest 203 | file: cf-deployment-concourse-tasks/ci/tasks/bump-cf-d-ct-docker-image/task.yml 204 | params: 205 | NEW_VERSION: latest 206 | - put: cf-deployment-concourse-tasks 207 | params: 208 | repository: cf-deployment-concourse-tasks-updated 209 | rebase: true 210 | - put: cf-deployment-concourse-tasks-version 211 | params: 212 | bump: major 213 | ## generate release notes and push draft release 214 | - task: generate-release-notes-template 215 | file: cf-deployment-concourse-tasks/ci/tasks/cf-decota-release-notes-template/task.yml 216 | input_mapping: 217 | release-version: cf-deployment-concourse-tasks-version 218 | - put: cf-deployment-concourse-tasks-github-release 219 | params: 220 | name: release-notes-template/name 221 | tag: cf-deployment-concourse-tasks-version/version 222 | tag_prefix: v 223 | body: release-notes-template/template 224 | 225 | - name: ship-it-minor 226 | public: true 227 | plan: 228 | - in_parallel: 229 | - get: cf-deployment-concourse-tasks-version 230 | params: 231 | bump: minor 232 | - get: cf-deployment-concourse-tasks 233 | - get: cf-deployment-concourse-tasks-latest-release 234 | - get: cf-deployment-concourse-tasks-pipeline-image 235 | params: 236 | save: true 237 | passed: 238 | - build-docker-image 239 | ## push to docker with a tagged version 240 | - task: generate-major-version-file 241 | file: cf-deployment-concourse-tasks/ci/tasks/generate-major-version-file/task.yml 242 | input_mapping: 243 | semantic_version: cf-deployment-concourse-tasks-version 244 | - put: cf-deployment-concourse-tasks-pipeline-image 245 | params: 246 | load: cf-deployment-concourse-tasks-pipeline-image 247 | tag_file: major_version/version 248 | tag_prefix: v 249 | get_params: 250 | save: true 251 | - put: cf-deployment-concourse-tasks-pipeline-image 252 | params: 253 | load: cf-deployment-concourse-tasks-pipeline-image 254 | tag_file: cf-deployment-concourse-tasks-version/version 255 | tag_prefix: v 256 | ## update task yml to a updated dockerimage version and tag 257 | - task: update-docker-image-version-for-release 258 | file: cf-deployment-concourse-tasks/ci/tasks/bump-cf-d-ct-docker-image/task.yml 259 | input_mapping: 260 | version: cf-deployment-concourse-tasks-version 261 | - put: cf-deployment-concourse-tasks 262 | params: 263 | repository: cf-deployment-concourse-tasks-updated 264 | tag: cf-deployment-concourse-tasks-version/version 265 | tag_prefix: v 266 | ## update task yml to point back to latest 267 | - task: update-docker-image-version-back-to-latest 268 | file: cf-deployment-concourse-tasks/ci/tasks/bump-cf-d-ct-docker-image/task.yml 269 | params: 270 | NEW_VERSION: latest 271 | - put: cf-deployment-concourse-tasks 272 | params: 273 | repository: cf-deployment-concourse-tasks-updated 274 | rebase: true 275 | - put: cf-deployment-concourse-tasks-version 276 | params: 277 | bump: minor 278 | ## generate release notes and push draft release 279 | - task: generate-release-notes-template 280 | file: cf-deployment-concourse-tasks/ci/tasks/cf-decota-release-notes-template/task.yml 281 | input_mapping: 282 | release-version: cf-deployment-concourse-tasks-version 283 | - put: cf-deployment-concourse-tasks-github-release 284 | params: 285 | name: release-notes-template/name 286 | tag: cf-deployment-concourse-tasks-version/version 287 | tag_prefix: v 288 | body: release-notes-template/template 289 | -------------------------------------------------------------------------------- /ci/tasks/bump-cf-d-ct-docker-image/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | # Not able to resolve our import via shellcheck, so disable warning 4 | # shellcheck disable=SC1091 5 | function commit_with_message() { 6 | local repo_dir="${1}" 7 | local commit_message_file="${2}" 8 | 9 | pushd "${repo_dir}" > /dev/null 10 | if [[ -z "$(git status --porcelain)" ]]; then 11 | popd > /dev/null 12 | return 13 | fi 14 | 15 | git config user.name "ARD WG Bot" 16 | git config user.email "app-deployments@cloudfoundry.org" 17 | 18 | git add . 19 | 20 | local commit_message 21 | commit_message="$(cat "${commit_message_file}")" 22 | 23 | if [[ -z "${commit_message}" ]]; then 24 | commit_message="No changes" 25 | fi 26 | 27 | git commit -m "${commit_message}" 28 | 29 | git status 30 | git show --color | cat 31 | popd > /dev/null 32 | } 33 | 34 | function main() { 35 | local new_version 36 | if [[ -n "${NEW_VERSION}" ]]; then 37 | new_version="${NEW_VERSION}" 38 | elif [[ -d version ]]; then 39 | new_version="v$(cat version/version)" 40 | fi 41 | 42 | if [[ -z "${new_version}" ]]; then 43 | echo "version is empty" 44 | exit 1 45 | fi 46 | 47 | git clone cf-deployment-concourse-tasks cf-deployment-concourse-tasks-updated 48 | 49 | pushd cf-deployment-concourse-tasks-updated 50 | local old_version 51 | old_version="$(bosh int bbl-up/task.yml --path /image_resource/source/tag)" 52 | 53 | sed -i -e "s/tag: ${old_version}/tag: ${new_version}/g" ./*/task.yml 54 | 55 | commit_with_message "${PWD}" <(echo "Update docker image to ${new_version}") 56 | popd 57 | } 58 | 59 | main 60 | -------------------------------------------------------------------------------- /ci/tasks/bump-cf-d-ct-docker-image/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: docker-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | 9 | inputs: 10 | - name: cf-deployment-concourse-tasks 11 | - name: version 12 | optional: true 13 | 14 | outputs: 15 | - name: cf-deployment-concourse-tasks-updated 16 | 17 | params: 18 | NEW_VERSION: 19 | 20 | run: 21 | path: cf-deployment-concourse-tasks/ci/tasks/bump-cf-d-ct-docker-image/task 22 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'hashdiff' 4 | gem 'rspec' 5 | 6 | group :development do 7 | gem 'guard' 8 | gem 'guard-rspec', require: false 9 | end 10 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | coderay (1.1.3) 5 | diff-lcs (1.6.2) 6 | ffi (1.17.1) 7 | formatador (1.1.0) 8 | guard (2.19.1) 9 | formatador (>= 0.2.4) 10 | listen (>= 2.7, < 4.0) 11 | logger (~> 1.6) 12 | lumberjack (>= 1.0.12, < 2.0) 13 | nenv (~> 0.1) 14 | notiffany (~> 0.0) 15 | ostruct (~> 0.6) 16 | pry (>= 0.13.0) 17 | shellany (~> 0.0) 18 | thor (>= 0.18.1) 19 | guard-compat (1.2.1) 20 | guard-rspec (4.7.3) 21 | guard (~> 2.1) 22 | guard-compat (~> 1.1) 23 | rspec (>= 2.99.0, < 4.0) 24 | hashdiff (1.2.0) 25 | listen (3.9.0) 26 | rb-fsevent (~> 0.10, >= 0.10.3) 27 | rb-inotify (~> 0.9, >= 0.9.10) 28 | logger (1.6.4) 29 | lumberjack (1.2.10) 30 | method_source (1.1.0) 31 | nenv (0.3.0) 32 | notiffany (0.1.3) 33 | nenv (~> 0.1) 34 | shellany (~> 0.0) 35 | ostruct (0.6.1) 36 | pry (0.15.2) 37 | coderay (~> 1.1) 38 | method_source (~> 1.0) 39 | rb-fsevent (0.11.2) 40 | rb-inotify (0.11.1) 41 | ffi (~> 1.0) 42 | rspec (3.13.1) 43 | rspec-core (~> 3.13.0) 44 | rspec-expectations (~> 3.13.0) 45 | rspec-mocks (~> 3.13.0) 46 | rspec-core (3.13.4) 47 | rspec-support (~> 3.13.0) 48 | rspec-expectations (3.13.5) 49 | diff-lcs (>= 1.2.0, < 2.0) 50 | rspec-support (~> 3.13.0) 51 | rspec-mocks (3.13.5) 52 | diff-lcs (>= 1.2.0, < 2.0) 53 | rspec-support (~> 3.13.0) 54 | rspec-support (3.13.4) 55 | shellany (0.0.1) 56 | thor (1.3.2) 57 | 58 | PLATFORMS 59 | ruby 60 | 61 | DEPENDENCIES 62 | guard 63 | guard-rspec 64 | hashdiff 65 | rspec 66 | 67 | BUNDLED WITH 68 | 2.5.17 69 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/Guardfile: -------------------------------------------------------------------------------- 1 | # vi:syn=ruby 2 | 3 | rspec_options = { 4 | cmd: 'bundle exec rspec', 5 | spec_paths: ['.'], 6 | all_on_start: true 7 | } 8 | 9 | guard :rspec, rspec_options do 10 | watch(%r{^.+\.rb$}) do |m| 11 | puts "#{m[0]} changed." 12 | '.' 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/binary_changes.rb: -------------------------------------------------------------------------------- 1 | require 'hashdiff' 2 | 3 | class HashdiffChange 4 | attr_accessor :value 5 | 6 | OPERATION = 0 7 | VALUE = 2 8 | 9 | def initialize(hash_diff) 10 | @operation = hash_diff[OPERATION] 11 | @value = hash_diff[VALUE] 12 | end 13 | 14 | def isAddition? 15 | @operation == '+' 16 | end 17 | 18 | def isDeletion? 19 | @operation == '-' 20 | end 21 | end 22 | 23 | class BinaryUpdate 24 | attr_accessor :old_version, :new_version 25 | end 26 | 27 | class BinaryUpdates 28 | NAME_INDEX = 0 29 | VERSION_INDEX = 1 30 | 31 | def initialize(latest_path, master_path) 32 | @updates = {} 33 | 34 | binary_current_list = collect_binaries(master_path) 35 | binary_latest_release_list = collect_binaries(latest_path) 36 | diff_list = Hashdiff.diff(binary_latest_release_list, binary_current_list) 37 | change_list = diff_list.map { |diff| HashdiffChange.new(diff) } 38 | change_list.each do |change| 39 | update_binary_changes(change) 40 | end 41 | end 42 | 43 | def get_update_by_name(binary_name) 44 | @updates[binary_name] 45 | end 46 | 47 | def count 48 | @updates.count 49 | end 50 | 51 | def each 52 | @updates.sort.each do |binary_name, binary_update| 53 | yield binary_name, binary_update 54 | end 55 | end 56 | 57 | def merge!(updates2) 58 | updates2.each do |binary_name, binary_update| 59 | @updates[binary_name] = binary_update 60 | end 61 | end 62 | 63 | private 64 | 65 | def collect_binaries(dockerfile_path) 66 | binaries = `grep -i 'ENV .*_version' #{dockerfile_path} | awk '{print tolower($0)}' | awk 'gsub("_version", "")' | awk '{print $2 ":" $3}'` 67 | binaries.split("\n").map do |b| 68 | { 69 | "name" => b.split(":")[NAME_INDEX], 70 | "version" => b.split(":")[VERSION_INDEX] 71 | } 72 | end 73 | end 74 | 75 | def update_binary_changes(change) 76 | if !change.isAddition? && !change.isDeletion? 77 | return 78 | end 79 | 80 | name = change.value['name'] 81 | version = change.value['version'] 82 | 83 | binary_update = @updates[name] || BinaryUpdate.new 84 | 85 | if change.isAddition? 86 | binary_update.new_version = version 87 | elsif change.isDeletion? 88 | binary_update.old_version = version 89 | end 90 | 91 | @updates[name] = binary_update 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/binary_changes_spec.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | require 'rspec' 3 | require 'fileutils' 4 | require_relative './binary_changes.rb' 5 | 6 | describe 'BinaryUpdates' do 7 | before(:all) do 8 | @current_work_dir = Dir.pwd 9 | @tmp_work_dir = Dir.mktmpdir('cf-deployment-concourse-tasks-test') 10 | 11 | Dir.chdir(@tmp_work_dir) 12 | end 13 | 14 | after(:all) do 15 | Dir.chdir(@current_work_dir) 16 | FileUtils.rm_rf(@tmp_work_dir) if File.exists?(@tmp_work_dir) 17 | end 18 | 19 | let(:ru) { BinaryUpdates.new } 20 | 21 | describe 'load_from_file' do 22 | before(:all) do 23 | FileUtils.mkdir_p('cf-deployment-concourse-tasks-latest-release/dockerfiles/cf-deployment-concourse-tasks/') 24 | FileUtils.mkdir_p('cf-deployment-concourse-tasks/dockerfiles/cf-deployment-concourse-tasks/') 25 | end 26 | 27 | subject(:updates) do 28 | BinaryUpdates.new( 29 | "cf-deployment-concourse-tasks-latest-release/#{filename}", 30 | "cf-deployment-concourse-tasks/#{filename}" 31 | ) 32 | end 33 | 34 | before do 35 | File.open(File.join('cf-deployment-concourse-tasks-latest-release', filename), 'w') do |f| 36 | f.write(file_contents_latest_release) 37 | end 38 | 39 | File.open(File.join('cf-deployment-concourse-tasks', filename), 'w') do |f| 40 | f.write(file_contents_current) 41 | end 42 | end 43 | 44 | let(:filename) { 'dockerfiles/cf-deployment-concourse-tasks/Dockerfile' } 45 | let(:file_contents_latest_release) do 46 | <<-HEREDOC 47 | ENV JQ_VERSION 1.5 48 | ENV go_version 1.11.1 49 | ENV cf_cli_version 6.40.0 50 | ENV bosh_cli_version 5.3.1 51 | ENV bbl_version 6.10.18 52 | ENV terraform_version 0.11.10 53 | ENV credhub_cli_version 2.1.0 54 | ENV git_crypt_version 0.6.0 55 | HEREDOC 56 | end 57 | 58 | let(:file_contents_current) do 59 | <<-HEREDOC 60 | ENV JQ_VERSION 1.6 61 | ENV go_version 1.12.5 62 | ENV cf_cli_version 6.43.0 63 | ENV bosh_cli_version 5.5.1 64 | ENV bbl_version 8.0.0 65 | ENV terraform_version 0.12.0 66 | ENV credhub_cli_version 2.4.0 67 | ENV git_crypt_version 0.6.0 68 | HEREDOC 69 | end 70 | 71 | it 'reads the given file in the two inputs, and returns the binary updates' do 72 | expect(updates.count).to eq 7 73 | 74 | go_release_update = updates.get_update_by_name('go') 75 | 76 | expect(go_release_update.old_version).to eq '1.11.1' 77 | expect(go_release_update.new_version).to eq '1.12.5' 78 | 79 | cf_cli_release_update = updates.get_update_by_name('cf_cli') 80 | 81 | expect(cf_cli_release_update.old_version).to eq '6.40.0' 82 | expect(cf_cli_release_update.new_version).to eq '6.43.0' 83 | 84 | bosh_cli_release_update = updates.get_update_by_name('bosh_cli') 85 | 86 | expect(bosh_cli_release_update.old_version).to eq '5.3.1' 87 | expect(bosh_cli_release_update.new_version).to eq '5.5.1' 88 | 89 | bbl_update = updates.get_update_by_name('bbl') 90 | 91 | expect(bbl_update.old_version).to eq '6.10.18' 92 | expect(bbl_update.new_version).to eq '8.0.0' 93 | 94 | terraform_update = updates.get_update_by_name('terraform') 95 | 96 | expect(terraform_update.old_version).to eq '0.11.10' 97 | expect(terraform_update.new_version).to eq '0.12.0' 98 | 99 | credhub_cli_update = updates.get_update_by_name('credhub_cli') 100 | 101 | expect(credhub_cli_update.old_version).to eq '2.1.0' 102 | expect(credhub_cli_update.new_version).to eq '2.4.0' 103 | 104 | jq_update = updates.get_update_by_name('jq') 105 | 106 | expect(jq_update.old_version).to eq '1.5' 107 | expect(jq_update.new_version).to eq '1.6' 108 | end 109 | 110 | context 'when the old version of the file is empty' do 111 | let(:file_contents_latest_release) { "" } 112 | context 'and the new version is not empty' do 113 | let(:file_contents_current) do 114 | <<-HEREDOC 115 | ENV go_version 1.12.5 116 | HEREDOC 117 | end 118 | 119 | it 'views the newly-introduced binary as additive updates' do 120 | binary_update = updates.get_update_by_name('go') 121 | expect(binary_update.old_version).to eq nil 122 | expect(binary_update.new_version).to eq "1.12.5" 123 | end 124 | end 125 | 126 | context 'and the new version is empty' do 127 | let(:file_contents_current) { "" } 128 | 129 | it 'includes no information about the binary' do 130 | expect(updates.get_update_by_name('go')).to be_nil 131 | end 132 | end 133 | end 134 | 135 | context 'when the new version of the file is empty' do 136 | let(:file_contents_current) { "" } 137 | context 'and the old version is not empty' do 138 | let(:file_contents_latest_release) do 139 | <<-HEREDOC 140 | ENV go_version 1.12.5 141 | HEREDOC 142 | end 143 | 144 | it 'views the removed binary as negative updates' do 145 | binary_update = updates.get_update_by_name('go') 146 | expect(binary_update.old_version).to eq "1.12.5" 147 | expect(binary_update.new_version).to eq nil 148 | end 149 | end 150 | 151 | context 'and the new version is empty' do 152 | let(:file_contents_latest_release) { "" } 153 | 154 | it 'includes no information about the binary' do 155 | expect(updates.get_update_by_name('go')).to be_nil 156 | end 157 | end 158 | end 159 | end 160 | end 161 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/renderer.rb: -------------------------------------------------------------------------------- 1 | class Renderer 2 | def render(binary_updates:, task_updates:) 3 | binaries_table = render_table(binary_updates) 4 | 5 | <<-HEREDOC 6 | ## Notices 7 | ### :point_right: New Tasks :point_left: 8 | #{render_tasks(task_updates.new_tasks)} 9 | 10 | ### :point_right: Updated Tasks :point_left: 11 | #{render_tasks(task_updates.updated_tasks)} 12 | 13 | ### :point_right: Deleted Tasks :point_left: 14 | #{render_tasks(task_updates.deleted_tasks)} 15 | 16 | ## Binary Updates 17 | #{binaries_table} 18 | HEREDOC 19 | end 20 | 21 | private 22 | 23 | def render_table(binary_updates) 24 | header = <<-HEADER 25 | | Binary | Old Version | New Version | 26 | | ------- | ----------- | ----------- | 27 | HEADER 28 | 29 | table = "" 30 | binary_updates.each do |binary_name, binary_update| 31 | table << "| #{binary_name} | #{render_version(binary_update, 'old')} | #{render_version(binary_update, 'new')} |\n" 32 | end 33 | "#{header}#{table}" 34 | end 35 | 36 | def render_version(binary_update, type) 37 | return binary_update.send(type + '_version') 38 | end 39 | 40 | def render_tasks(tasks) 41 | if tasks.nil? 42 | return "" 43 | end 44 | 45 | tasks.sort.map do |task| 46 | "-**`#{task}`**" 47 | end.join("\n") 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/renderer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require_relative './renderer.rb' 3 | 4 | describe 'Renderer' do 5 | describe '#render' do 6 | subject(:renderer) { Renderer.new } 7 | 8 | let(:binary_update_1) do 9 | update = double('BinaryUpdate') 10 | allow(update).to receive(:old_version) { '1.1.0' } 11 | allow(update).to receive(:new_version) { '1.3.0' } 12 | update 13 | end 14 | 15 | let(:binary_update_2) do 16 | update = double('BinaryUpdate') 17 | allow(update).to receive(:old_version) { '1.2.0' } 18 | allow(update).to receive(:new_version) { '1.4.0' } 19 | update 20 | end 21 | 22 | let(:binary_updates) do 23 | updates = double('BinaryUpdates') 24 | allow(updates).to receive(:each).and_yield('binary-1', binary_update_1).and_yield('binary-2', binary_update_2) 25 | updates 26 | end 27 | 28 | let(:task_updates) do 29 | updates = double('TaskUpdates') 30 | allow(updates).to receive(:new_tasks).and_return(['new-task']) 31 | allow(updates).to receive(:deleted_tasks).and_return(['deleted-task']) 32 | allow(updates).to receive(:updated_tasks).and_return(['updated-task']) 33 | updates 34 | end 35 | 36 | it 'includes a section header for Notices' do 37 | rendered_output = renderer.render(binary_updates: binary_updates, task_updates: task_updates) 38 | expect(rendered_output).to include ("## Notices") 39 | end 40 | 41 | it 'includes a section header for Notices, as well as sub-headers for New and Updated Tasks' do 42 | rendered_output = renderer.render(binary_updates: binary_updates, task_updates: task_updates) 43 | expect(rendered_output).to include ( 44 | <<-NOTICES 45 | ## Notices 46 | ### :point_right: New Tasks :point_left: 47 | -**`new-task`** 48 | 49 | ### :point_right: Updated Tasks :point_left: 50 | -**`updated-task`** 51 | 52 | ### :point_right: Deleted Tasks :point_left: 53 | -**`deleted-task`** 54 | 55 | NOTICES 56 | ) 57 | end 58 | 59 | it 'inlcudes a section header for Binary Updates' do 60 | rendered_output = renderer.render(binary_updates: binary_updates, task_updates: task_updates) 61 | expect(rendered_output).to include ("## Binary Updates\n") 62 | end 63 | 64 | describe 'Binary table' do 65 | it 'includes a header' do 66 | rendered_output = renderer.render(binary_updates: binary_updates, task_updates: task_updates) 67 | expect(rendered_output).to include( 68 | <<-HEADER 69 | | Binary | Old Version | New Version | 70 | | ------- | ----------- | ----------- | 71 | HEADER 72 | ) 73 | end 74 | 75 | it 'places the table header immediately after the section header' do 76 | rendered_output = renderer.render(binary_updates: binary_updates, task_updates: task_updates) 77 | expect(rendered_output).to include ("## Binary Updates\n| Binary | Old Version | New Version |") 78 | end 79 | 80 | it 'shows the binary name, old version, and new version for each binary' do 81 | rendered_output = renderer.render(binary_updates: binary_updates, task_updates: task_updates) 82 | expect(rendered_output).to include('| binary-1 | 1.1.0 | 1.3.0 |') 83 | expect(rendered_output).to include('| binary-2 | 1.2.0 | 1.4.0 |') 84 | end 85 | 86 | context 'when some versions are nil' do 87 | let(:binary_update_1) do 88 | update = double('BinaryUpdate') 89 | allow(update).to receive(:old_version) { '1.1.0' } 90 | allow(update).to receive(:new_version) { nil } 91 | update 92 | end 93 | 94 | let(:binary_update_2) do 95 | update = double('BinaryUpdate') 96 | allow(update).to receive(:old_version) { nil } 97 | allow(update).to receive(:new_version) { '1.4.0' } 98 | update 99 | end 100 | 101 | it 'renders them as empty strings' do 102 | rendered_output = renderer.render(binary_updates: binary_updates, task_updates: task_updates) 103 | expect(rendered_output).to include('| binary-1 | 1.1.0 | |') 104 | expect(rendered_output).to include('| binary-2 | | 1.4.0 |') 105 | end 106 | end 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -exu 3 | 4 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 5 | 6 | pushd $SCRIPT_DIR 7 | PARSED_BUNDLE_VERSION=$(sed -n '/BUNDLED WITH/{;n;p;}' Gemfile.lock) 8 | gem uninstall bundler 9 | gem install bundler --version "$PARSED_BUNDLE_VERSION" 10 | bundle install --without development 11 | popd 12 | 13 | BUNDLE_GEMFILE=$SCRIPT_DIR/Gemfile \ 14 | bundle exec ruby $SCRIPT_DIR/task.rb 15 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/task.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'hashdiff' 4 | require_relative './binary_changes.rb' 5 | require_relative './task_updates.rb' 6 | require_relative './renderer.rb' 7 | 8 | template = Renderer.new.render( 9 | binary_updates: BinaryUpdates.new( 10 | 'cf-deployment-concourse-tasks-latest-release/dockerfiles/cf-deployment-concourse-tasks/Dockerfile', 11 | 'cf-deployment-concourse-tasks/dockerfiles/cf-deployment-concourse-tasks/Dockerfile' 12 | ), 13 | task_updates: TaskUpdates.new( 14 | 'cf-deployment-concourse-tasks-latest-release', 15 | 'cf-deployment-concourse-tasks' 16 | ) 17 | ) 18 | 19 | puts template 20 | 21 | output_folder = 'release-notes-template' 22 | File.write("#{output_folder}/template", template) 23 | 24 | version = File.read("release-version/version") 25 | File.write("#{output_folder}/name", "v#{version}") 26 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: docker-image 6 | source: 7 | repository: ruby 8 | tag: 3.1-slim 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks 12 | - name: cf-deployment-concourse-tasks-latest-release 13 | - name: release-version 14 | 15 | outputs: 16 | - name: release-notes-template 17 | 18 | params: 19 | GH_ACCESS_TOKEN: ~ 20 | 21 | run: 22 | path: cf-deployment-concourse-tasks/ci/tasks/cf-decota-release-notes-template/task 23 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/task_finder.rb: -------------------------------------------------------------------------------- 1 | class TaskFinder 2 | def self.find_tasks(repo_dir) 3 | tasks = Dir.glob( 4 | File.join(repo_dir, '*', 'task.yml') 5 | ) 6 | task_list = tasks.select { |fd| File.file?(fd) } 7 | task_list.map { |task| task.gsub!("#{repo_dir}/", '').gsub!("/task.yml", '') } 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/task_finder_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require_relative './task_finder.rb' 3 | 4 | describe 'TaskFinder' do 5 | subject { TaskFinder.find_tasks('test-cf-deployment-concourse-tasks') } 6 | 7 | before(:all) do 8 | @current_work_dir = Dir.pwd 9 | @tmp_work_dir = Dir.mktmpdir('test-cf-deployment-concourse-tasks') 10 | 11 | Dir.chdir(@tmp_work_dir) 12 | FileUtils.mkdir_p('test-cf-deployment-concourse-tasks/task1') 13 | FileUtils.mkdir_p('test-cf-deployment-concourse-tasks/task2') 14 | FileUtils.mkdir_p('test-cf-deployment-concourse-tasks/dockerfiles') 15 | end 16 | 17 | after(:all) do 18 | Dir.chdir(@current_work_dir) 19 | FileUtils.rm_rf(@tmp_work_dir) if File.exist?(@tmp_work_dir) 20 | end 21 | 22 | context 'when there are tasks in the lop-level directory' do 23 | before do 24 | File.open('test-cf-deployment-concourse-tasks/task1/task.yml', 'w') 25 | File.open('test-cf-deployment-concourse-tasks/task2/task.yml', 'w') 26 | File.open('test-cf-deployment-concourse-tasks/dockerfiles/Dockerfile', 'w') 27 | File.open('test-cf-deployment-concourse-tasks/README.md', 'w') 28 | end 29 | 30 | it 'returns the file names of the tasks without any additional path' do 31 | expect(subject).to contain_exactly 'task1', 'task2' 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/task_updates.rb: -------------------------------------------------------------------------------- 1 | require 'digest' 2 | 3 | require_relative './task_finder.rb' 4 | 5 | class TaskUpdates 6 | attr_accessor :new_tasks, :deleted_tasks, :updated_tasks 7 | 8 | def initialize(latest_release_dir, master_dir) 9 | latest_release_task_list = TaskFinder.find_tasks(latest_release_dir) 10 | master_task_list = TaskFinder.find_tasks(master_dir) 11 | 12 | @new_tasks = master_task_list - latest_release_task_list 13 | @deleted_tasks = latest_release_task_list - master_task_list 14 | common_task_list = latest_release_task_list & master_task_list 15 | @updated_tasks = [] 16 | 17 | common_task_list.each do |task| 18 | latest_release_task_shasum = task_shasum(latest_release_dir, task) 19 | master_task_shasum = task_shasum(master_dir, task) 20 | if latest_release_task_shasum != master_task_shasum 21 | @updated_tasks << task 22 | end 23 | end 24 | end 25 | 26 | private 27 | 28 | def task_shasum(dir, task) 29 | Digest::SHA1.hexdigest( 30 | Dir.glob( 31 | File.join(dir, task, '*') 32 | ).map do |file| 33 | Digest::SHA1.hexdigest(read_file(file)) 34 | end.join 35 | ) 36 | end 37 | 38 | def read_file(file) 39 | if !file.match(/.*task.yml/) 40 | return File.read(file) 41 | end 42 | 43 | File.readlines(file).reject do |line| 44 | line.match(/.*tag: v\d+.\d+.\d+/) 45 | end.join 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /ci/tasks/cf-decota-release-notes-template/task_updates_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require_relative './task_updates.rb' 3 | 4 | describe 'TaskUpdates' do 5 | subject { TaskUpdates.new('latest-release', 'master') } 6 | 7 | before(:all) do 8 | @current_work_dir = Dir.pwd 9 | @tmp_work_dir = Dir.mktmpdir('test-cf-deployment-concourse-tasks') 10 | 11 | Dir.chdir(@tmp_work_dir) 12 | FileUtils.mkdir_p('master/new-task') 13 | FileUtils.mkdir_p('master/updated-task') 14 | FileUtils.mkdir_p('latest-release/updated-task') 15 | FileUtils.mkdir_p('latest-release/deleted-task') 16 | end 17 | 18 | after(:all) do 19 | Dir.chdir(@current_work_dir) 20 | FileUtils.rm_rf(@tmp_work_dir) if File.exist?(@tmp_work_dir) 21 | end 22 | 23 | context 'when there are tasks in the lop-level directory' do 24 | before do 25 | File.open('master/updated-task/task.yml', 'w') 26 | File.open('latest-release/updated-task/task.yml', 'w') 27 | end 28 | 29 | context 'and only task.yml docker image tag version has changed' do 30 | before do 31 | File.open('master/updated-task/task.yml', 'w') do |f| 32 | f.write(task_yml_content("2.2.2")) 33 | end 34 | File.open('latest-release/updated-task/task.yml', 'w') do |f| 35 | f.write(task_yml_content("1.1.1")) 36 | end 37 | end 38 | 39 | def task_yml_content(tag_version) 40 | <<-HEREDOC 41 | --- 42 | platform: linux 43 | image_resource: 44 | type: docker-image 45 | source: 46 | repository: cloudfoundry/cf-deployment-concourse-tasks 47 | tag: v#{tag_version} 48 | inputs: 49 | - name: cf-deployment-concourse-tasks 50 | run: 51 | path: cf-deployment-concourse-tasks/bbl-destroy/task 52 | params: 53 | BBL_STATE_DIR: bbl-state 54 | HEREDOC 55 | end 56 | 57 | it 'returns a map with no changes' do 58 | expect(subject.new_tasks).to be_empty 59 | expect(subject.deleted_tasks).to be_empty 60 | expect(subject.updated_tasks).to be_empty 61 | end 62 | end 63 | 64 | context 'and there valid changes to the task' do 65 | before do 66 | File.open('master/new-task/task.yml', 'w') 67 | File.open('latest-release/deleted-task/task.yml', 'w') 68 | File.open('latest-release/updated-task/task.sh', 'w') 69 | end 70 | 71 | it 'returns a map with three lists of changes' do 72 | expect(subject.new_tasks).to contain_exactly('new-task') 73 | expect(subject.deleted_tasks).to contain_exactly('deleted-task') 74 | expect(subject.updated_tasks).to contain_exactly('updated-task') 75 | end 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /ci/tasks/generate-major-version-file/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | function main() { 4 | if [[ ! -r semantic_version/version ]]; then 5 | echo "Missing semantic_version/version input file." 6 | exit 1 7 | fi 8 | 9 | echo "Extracting major version from $(cat semantic_version/version)..." 10 | cat semantic_version/version | cut -d'.' -f1 > major_version/version 11 | } 12 | 13 | main 14 | -------------------------------------------------------------------------------- /ci/tasks/generate-major-version-file/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: docker-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | 9 | inputs: 10 | - name: cf-deployment-concourse-tasks 11 | - name: semantic_version 12 | 13 | outputs: 14 | - name: major_version 15 | 16 | run: 17 | path: cf-deployment-concourse-tasks/ci/tasks/generate-major-version-file/task 18 | -------------------------------------------------------------------------------- /ci/tasks/update-cf-d-ct-dockerfile-versions/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | dockerfile="cf-deployment-concourse-tasks/dockerfiles/cf-deployment-concourse-tasks/Dockerfile" 4 | 5 | sed -i "s/ENV bbl_version.*$/ENV bbl_version $(cat bbl-github-release/version)/" $dockerfile 6 | sed -i "s/ENV bosh_cli_version.*$/ENV bosh_cli_version $(cat bosh-cli-github-release/version)/" $dockerfile 7 | sed -i "s/ENV cf_cli_version.*$/ENV cf_cli_version $(cat cf-cli/version)/" $dockerfile 8 | sed -i "s/ENV credhub_cli_version.*$/ENV credhub_cli_version $(cat credhub-cli-github-release/version)/" $dockerfile 9 | sed -i "s/ENV log_cache_cli_version.*$/ENV log_cache_cli_version $(cat log-cache-cli-github-release/version)/" $dockerfile 10 | sed -i "s/ENV terraform_version.*$/ENV terraform_version $(cat terraform-github-release/version)/" $dockerfile 11 | sed -i "s/ENV uptimer_version.*$/ENV uptimer_version $(git -C uptimer rev-parse HEAD)/" $dockerfile 12 | sed -i "s/ENV yq_version.*$/ENV yq_version $(cat yq-github-release/version)/" $dockerfile 13 | 14 | pushd cf-deployment-concourse-tasks 15 | if [[ -n $(git status --porcelain) ]]; then 16 | git config user.name "ARD WG Bot" 17 | git config user.email "app-deployments@cloudfoundry.org" 18 | git add . 19 | git commit --allow-empty \ 20 | -m "Update Dockerfile" 21 | fi 22 | popd 23 | 24 | git clone cf-deployment-concourse-tasks cf-deployment-concourse-tasks-updated 25 | -------------------------------------------------------------------------------- /ci/tasks/update-cf-d-ct-dockerfile-versions/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: docker-image 6 | source: 7 | repository: cloudfoundry/relint-base 8 | 9 | inputs: 10 | - name: bbl-github-release 11 | - name: bosh-cli-github-release 12 | - name: cf-cli 13 | - name: cf-deployment-concourse-tasks 14 | - name: credhub-cli-github-release 15 | - name: log-cache-cli-github-release 16 | - name: terraform-github-release 17 | - name: uptimer 18 | - name: yq-github-release 19 | 20 | outputs: 21 | - name: cf-deployment-concourse-tasks-updated 22 | 23 | run: 24 | path: cf-deployment-concourse-tasks/ci/tasks/update-cf-d-ct-dockerfile-versions/task 25 | -------------------------------------------------------------------------------- /collect-ops-files/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | check_collect_opsfiles_params() { 4 | if [ ! -d ${ROOT_DIR}/base-ops-files/${BASE_OPS_FILE_DIR} ]; then 5 | echo "[ERROR] base-ops-files/${BASE_OPS_FILE_DIR} must exist in order to collect operations files." 6 | exit 1 7 | fi 8 | 9 | if [ ! -z "${NEW_OPS_FILES}" ]; then 10 | for OPERATION in ${NEW_OPS_FILES}; do 11 | if [ ! -e "${ROOT_DIR}/new-ops-files/${OPERATION}" ]; then 12 | echo "[ERROR] All files specified in NEW_OPS_FILES must exist. [ new-ops-files/${OPERATION} ] does not." 13 | exit 1 14 | fi 15 | done 16 | fi 17 | } 18 | 19 | # Create collection directory (task output) 20 | create_collection_directory() { 21 | mkdir -p ${ROOT_DIR}/collected-ops-files/${BASE_OPS_FILE_DIR} 22 | } 23 | 24 | set_collection_path() { 25 | if [ -z "${BASE_OPS_FILE_DIR}" ]; then 26 | COLLECTION_PATH="${ROOT_DIR}/collected-ops-files" 27 | else 28 | COLLECTION_PATH="${ROOT_DIR}/collected-ops-files/${BASE_OPS_FILE_DIR}" 29 | fi 30 | } 31 | 32 | # Copy all base operations to collection 33 | collect_base_operations() { 34 | cp -r ${ROOT_DIR}/base-ops-files/${BASE_OPS_FILE_DIR}/ ${ROOT_DIR}/collected-ops-files 35 | } 36 | 37 | # Copy each new operation to collection 38 | collect_new_operations() { 39 | if [ -z "${NEW_OPS_FILES}" ]; then 40 | # If we don't have a list of specific ops files to collect, then collect 41 | # all of the files at the root 42 | find ${ROOT_DIR}/new-ops-files -type f -maxdepth 1 | xargs -n1 -I{} cp {} ${COLLECTION_PATH} 43 | else 44 | # ... otherwise, add them one by one 45 | for OPERATION in ${NEW_OPS_FILES}; do 46 | # Get only the filename part of the specified ops file since we're putting all of them in 47 | # the same "collected" directory 48 | local target_filename 49 | target_filename=$(basename "${OPERATION}") 50 | 51 | cp ${ROOT_DIR}/new-ops-files/${OPERATION} ${COLLECTION_PATH}/${target_filename} 52 | done 53 | fi 54 | } 55 | 56 | main() { 57 | ROOT_DIR="${1}" 58 | 59 | check_collect_opsfiles_params 60 | create_collection_directory 61 | set_collection_path 62 | collect_base_operations 63 | collect_new_operations 64 | } 65 | 66 | main ${PWD} 67 | -------------------------------------------------------------------------------- /collect-ops-files/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks 12 | - name: base-ops-files 13 | - name: new-ops-files 14 | 15 | outputs: 16 | - name: collected-ops-files 17 | 18 | params: 19 | BASE_OPS_FILE_DIR: 20 | # - Optional 21 | # - Directory in the `base-ops-files` input that contains the base ops files 22 | # - If unspecified, input `base-ops-files` will be the root for finding 23 | # the base set of operations files 24 | # - If specified, the path must exist in the base-ops-files resource. 25 | 26 | NEW_OPS_FILES: 27 | # - Optional 28 | # - List of paths in the `new-ops-files` input that contains the new ops files 29 | # - If unspecified, every file in the root directory of the input `new-ops-files` will be 30 | # collected - any ops files in subdirectories will not be collected. 31 | # - If specified, each ops file listed will be copied to the `collected-ops-files` output. 32 | # For example, if the path for the ops-file is `new-ops-files/my-env/ops.yml`, 33 | # the user must include `my-env/ops.yml` in the value for `NEW_OPS_FILES`. The path in the output 34 | # will be `collected-ops-files/BASE_OPS_FILE_DIR/ops.yml`. 35 | 36 | run: 37 | path: cf-deployment-concourse-tasks/collect-ops-files/task 38 | -------------------------------------------------------------------------------- /dockerfiles/cf-deployment-concourse-tasks/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.24.4-bookworm 2 | 3 | ENV bbl_version 9.0.35 4 | ENV bosh_cli_version 7.9.6 5 | ENV cf_cli_version 8.14.0 6 | ENV credhub_cli_version 2.9.46 7 | ENV log_cache_cli_version 6.2.1 8 | ENV terraform_version 1.12.1 9 | ENV uptimer_version 4082e518d9cf64a0301482c1baf730accb753268 10 | ENV yq_version 4.45.4 11 | 12 | RUN apt-get update 13 | 14 | # useful utils 15 | RUN set -eux; \ 16 | apt-get -y install \ 17 | git-crypt \ 18 | jq \ 19 | netcat-openbsd \ 20 | unzip \ 21 | vim; 22 | 23 | # ruby 24 | RUN set -eux; \ 25 | apt-get -y install ruby-full; \ 26 | ruby --version | grep 3\.1 27 | 28 | # yq 29 | RUN set -eux; \ 30 | url="https://github.com/mikefarah/yq/releases/download/v${yq_version}/yq_linux_amd64"; \ 31 | wget "${url}" -O /usr/local/bin/yq; \ 32 | chmod +x /usr/local/bin/yq; \ 33 | yq --version 34 | 35 | # bosh 36 | RUN set -eux; \ 37 | apt-get -y install \ 38 | build-essential \ 39 | libreadline-dev \ 40 | libsqlite3-dev \ 41 | libssl-dev \ 42 | libxml2-dev \ 43 | libxslt1-dev \ 44 | libyaml-dev \ 45 | openssl \ 46 | sqlite3 \ 47 | zlib1g-dev; \ 48 | url="https://github.com/cloudfoundry/bosh-cli/releases/download/v${bosh_cli_version}/bosh-cli-${bosh_cli_version}-linux-amd64"; \ 49 | wget "${url}" -O /usr/local/bin/bosh; \ 50 | chmod +x /usr/local/bin/bosh; \ 51 | bosh --version 52 | 53 | # cf 54 | RUN set -eux; \ 55 | mkdir -p /tmp/cf; \ 56 | url="https://packages.cloudfoundry.org/stable?release=linux64-binary&version=${cf_cli_version}"; \ 57 | wget "${url}" -O /tmp/cf/cf.tgz; \ 58 | tar -xzf /tmp/cf/cf.tgz -C /tmp/cf; \ 59 | mv /tmp/cf/cf8 /usr/local/bin/cf; \ 60 | chmod +x /usr/local/bin/cf; \ 61 | cf version; \ 62 | rm -rf /tmp/cf 63 | 64 | # credhub 65 | RUN set -eux; \ 66 | mkdir -p /tmp/credhub; \ 67 | url="https://github.com/cloudfoundry/credhub-cli/releases/download/${credhub_cli_version}/credhub-linux-amd64-${credhub_cli_version}.tgz"; \ 68 | wget "${url}" -O /tmp/credhub/credhub.tgz; \ 69 | tar -xzf /tmp/credhub/credhub.tgz -C /tmp/credhub; \ 70 | mv /tmp/credhub/credhub /usr/local/bin; \ 71 | chmod +x /usr/local/bin/credhub; \ 72 | credhub --version; \ 73 | rm -rf /tmp/credhub 74 | 75 | # bbl 76 | RUN set -eux; \ 77 | url="https://github.com/cloudfoundry/bosh-bootloader/releases/download/v${bbl_version}/bbl-v${bbl_version}_linux_amd64"; \ 78 | wget "${url}" -O /usr/local/bin/bbl; \ 79 | chmod +x /usr/local/bin/bbl; \ 80 | bbl version 81 | 82 | # terraform 83 | RUN set -eux; \ 84 | mkdir -p /tmp/terraform; \ 85 | url="https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_linux_amd64.zip"; \ 86 | wget "${url}" -O /tmp/terraform/terraform.zip; \ 87 | unzip /tmp/terraform/terraform.zip -d /tmp/terraform; \ 88 | mv /tmp/terraform/terraform /usr/local/bin; \ 89 | chmod +x /usr/local/bin/terraform; \ 90 | terraform version; \ 91 | rm -rf /tmp/terraform 92 | 93 | # log-cache 94 | RUN set -eux; \ 95 | url="https://github.com/cloudfoundry/log-cache-cli/releases/download/v${log_cache_cli_version}/log-cache-cf-plugin_${log_cache_cli_version}_linux_amd64"; \ 96 | wget "${url}" -O /tmp/log-cache-plugin; \ 97 | cf install-plugin /tmp/log-cache-plugin -f; \ 98 | rm /tmp/log-cache-plugin 99 | 100 | # uptimer 101 | RUN set -eux; \ 102 | go install "github.com/cloudfoundry/uptimer@${uptimer_version}"; \ 103 | uptimer -v 104 | 105 | # Add trusted relint ca certificate 106 | ARG RELINT_CA_CERTIFICATE 107 | ARG PREVIOUS_RELINT_CA_CERTIFICATE 108 | RUN set -e; \ 109 | echo -n "$RELINT_CA_CERTIFICATE" > /usr/local/share/ca-certificates/relint.crt; \ 110 | echo -n "$PREVIOUS_RELINT_CA_CERTIFICATE" > /usr/local/share/ca-certificates/previous_relint.crt; \ 111 | /usr/sbin/update-ca-certificates -f 112 | -------------------------------------------------------------------------------- /open-asgs-for-bosh-instance-group/create_asgs_json.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby -w 2 | # frozen_string_literal: true 3 | 4 | require 'json' 5 | require 'open3' 6 | 7 | PRIVATE_IP_REGEX = /^10\./.freeze 8 | 9 | def write_asgs_json(instance_group_ips, destination_file) 10 | asgs = [] 11 | 12 | instance_group_ips.each do |ip| 13 | asgs << { protocol: 'tcp', destination: ip, ports: '1-65535' } 14 | end 15 | 16 | File.open(destination_file, 'w') do |f| 17 | f.write(asgs.to_json) 18 | end 19 | end 20 | 21 | def get_ips_from_bosh_output(instance_group_name) 22 | instance_ips = [] 23 | 24 | stdout, stderr, exitcode = Open3.capture3('\bosh is --json') 25 | 26 | raise "'bosh is --json' returned an error: #{stdout}\n#{stderr}" if exitcode != 0 27 | 28 | instances = JSON.parse(stdout)['Tables'].first['Rows'] 29 | instance_ips = get_instance_ips(instances, instance_group_name) 30 | 31 | raise 'No IPs detected' if instance_ips.empty? 32 | 33 | instance_ips 34 | end 35 | 36 | def get_instance_ips(instances, instance_group_name) 37 | filtered_instances = instances.select do |is| 38 | is['instance'].include? instance_group_name 39 | end 40 | 41 | filtered_instances.flat_map { |is| is['ips'].split.grep PRIVATE_IP_REGEX } 42 | end 43 | 44 | raise 'BOSH_DEPLOYMENT is required, but not set' unless ENV['BOSH_DEPLOYMENT'] 45 | 46 | instance_name, destination_file = ARGV 47 | 48 | instance_group_ips = get_ips_from_bosh_output(instance_name) 49 | write_asgs_json(instance_group_ips, destination_file) 50 | -------------------------------------------------------------------------------- /open-asgs-for-bosh-instance-group/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | # Not able to resolve our import via shellcheck, so disable warning 4 | # shellcheck disable=SC1091 5 | source cf-deployment-concourse-tasks/shared-functions 6 | 7 | function create_asg_json() { 8 | local asg_json_path 9 | asg_json_path=$1 10 | 11 | cf-deployment-concourse-tasks/open-asgs-for-bosh-instance-group/create_asgs_json.rb "${INSTANCE_GROUP_NAME}" "${asg_json_path}" 12 | } 13 | 14 | function main() { 15 | setup_bosh_env_vars 16 | 17 | cf api api."${SYSTEM_DOMAIN}" --skip-ssl-validation 18 | 19 | set +x 20 | local cf_admin_password 21 | cf_admin_password=$(get_password_from_credhub "cf_admin_password") 22 | cf auth admin "${cf_admin_password}" 23 | set -x 24 | 25 | local asgs_json 26 | asgs_json=$(mktemp) 27 | create_asg_json "${asgs_json}" 28 | 29 | cf delete-security-group "${SECURITY_GROUP_NAME}" -f 30 | cf create-security-group "${SECURITY_GROUP_NAME}" "${asgs_json}" 31 | 32 | cf bind-running-security-group "${SECURITY_GROUP_NAME}" 33 | cf bind-staging-security-group "${SECURITY_GROUP_NAME}" 34 | } 35 | 36 | main 37 | -------------------------------------------------------------------------------- /open-asgs-for-bosh-instance-group/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: bbl-state 12 | optional: true 13 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 14 | optional: true 15 | - name: cf-deployment-concourse-tasks 16 | 17 | run: 18 | path: cf-deployment-concourse-tasks/open-asgs-for-bosh-instance-group/task 19 | 20 | params: 21 | BBL_STATE_DIR: bbl-state 22 | # - Path to the directory containing the `bbl-state.json` file 23 | # - The path is relative to the `bbl-state` input 24 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 25 | # - If the specified path does not exist, it will be created 26 | # - This input and parameter will be ignored if toolsmiths-env is provided 27 | 28 | BOSH_DEPLOYMENT: cf 29 | # - Required 30 | # - Name of the deployment that has the Instance Group you want to allow 31 | # - communication to 32 | 33 | INSTANCE_GROUP_NAME: 34 | # - Required 35 | # - Name of the Instance Group from your deployment that you want to allow 36 | # - communication to 37 | 38 | SYSTEM_DOMAIN: 39 | # - Required unless toolsmiths-env optional input is provided 40 | # - CF system base domain e.g. `my-cf.com` 41 | 42 | SECURITY_GROUP_NAME: 43 | # - Required 44 | # - CF Application Security Group you wish to set 45 | -------------------------------------------------------------------------------- /run-cats-with-provided-cli: -------------------------------------------------------------------------------- 1 | run-cats -------------------------------------------------------------------------------- /run-cats/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -xeu 3 | 4 | build_dir=${PWD} 5 | 6 | if [[ -d "${build_dir}/cf-cli" ]]; then 7 | echo "Setting PATH so that input cf-cli is found by CATs..." 8 | PATH="${build_dir}/cf-cli:${PATH}" 9 | export PATH 10 | fi 11 | 12 | export CONFIG 13 | CONFIG=$(mktemp) 14 | 15 | original_config="${build_dir}/integration-config/${CONFIG_FILE_PATH}" 16 | reporter_config="${build_dir}/integration-config/${REPORTER_CONFIG_FILE_PATH}" 17 | 18 | if ${CAPTURE_LOGS}; then 19 | jq ".artifacts_directory=\"${build_dir}/cats-trace-output\"" ${original_config} > ${CONFIG} 20 | else 21 | cp ${original_config} ${CONFIG} 22 | fi 23 | 24 | if [ -f ${reporter_config} ]; then 25 | echo "Configuring Honeycomb integration..." 26 | set +x 27 | reporter_config_value="$(cat ${reporter_config})" 28 | combined_config_value="$(cat ${CONFIG} | jq --argjson reporter "$reporter_config_value" '. + $reporter')" 29 | 30 | pushd "${build_dir}/cf-acceptance-tests" 31 | combined_config_value="$(echo ${combined_config_value} | jq ".reporter_config.custom_tags.cats_git_sha=\"$(git rev-parse HEAD)\"" )" 32 | combined_config_value="$(echo ${combined_config_value} | jq ".reporter_config.custom_tags.cf_cli_version=\"$(cf -v | awk '{print $3}')\"" )" 33 | popd 34 | 35 | echo "$combined_config_value" > ${CONFIG} 36 | set -x 37 | fi 38 | 39 | cd cf-acceptance-tests 40 | 41 | export CF_DIAL_TIMEOUT=11 42 | 43 | ./bin/test \ 44 | --keep-going \ 45 | --randomize-all \ 46 | --skip-package=helpers \ 47 | --nodes="${NODES}" \ 48 | --skip="${SKIP_REGEXP}" \ 49 | --flake-attempts=${FLAKE_ATTEMPTS} \ 50 | --timeout="${TIMEOUT}" \ 51 | --no-color 52 | -------------------------------------------------------------------------------- /run-cats/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks 12 | - name: cf-acceptance-tests 13 | - name: integration-config 14 | - name: cf-cli 15 | optional: true 16 | # - Input to run your own cf-cli binary 17 | 18 | outputs: 19 | - name: cats-trace-output 20 | 21 | run: 22 | path: cf-deployment-concourse-tasks/run-cats/task 23 | 24 | params: 25 | NODES: 12 26 | # - Optional 27 | # - Number of parallel ginkgo nodes. 28 | 29 | CONFIG_FILE_PATH: integration_config.json 30 | # - Optional 31 | # - Path to the CATs integration configuration file. 32 | # - The path is relative to the `integration-config` input. 33 | 34 | REPORTER_CONFIG_FILE_PATH: reporter_config.json 35 | # - Optional 36 | # - Path to the CATs reporter configuration file. 37 | # - The path is relative to the `integration-config` input. 38 | # - If this file exists, it will automatically be combined with the config file above. 39 | # - The intent is to allow for the configuration of a CATs' custom reporter. 40 | 41 | SKIP_REGEXP: 42 | # - Optional 43 | # - A regular expression to be passed to the ginkgo's `-skip` flag. 44 | # - If set, ginkgo will skip tests that match this regular experession. 45 | 46 | CAPTURE_LOGS: false 47 | # - Optional 48 | # - If set to `true`, the task will capture logs and save them 49 | # to the output directory 50 | 51 | FLAKE_ATTEMPTS: 2 52 | # - Optional 53 | # - The number of times to retry a single failed test 54 | 55 | TIMEOUT: 2h 56 | # - Optional 57 | # - Timeout for the entire test run, as a Golang time.Duration. 58 | -------------------------------------------------------------------------------- /run-errand/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | # Not able to resolve our import via shellcheck, so disable warning 4 | # shellcheck disable=SC1091 5 | source cf-deployment-concourse-tasks/shared-functions 6 | 7 | function check_input_params() { 8 | if [ -z "$DEPLOYMENT_NAME" ]; then 9 | echo "DEPLOYMENT_NAME has not been set" 10 | exit 1 11 | fi 12 | if [ -z "$ERRAND_NAME" ]; then 13 | echo "ERRAND_NAME has not been set" 14 | exit 1 15 | fi 16 | } 17 | 18 | function main() { 19 | load_from_json_config 20 | 21 | check_input_params 22 | setup_bosh_env_vars 23 | 24 | PARAMS="" 25 | 26 | if [ "$KEEP_ALIVE" = true ]; then 27 | PARAMS="$PARAMS --keep-alive" 28 | fi 29 | 30 | if [ -n "${INSTANCE}" ]; then 31 | PARAMS="${PARAMS} --instance=${INSTANCE}" 32 | fi 33 | 34 | bosh -d "${DEPLOYMENT_NAME}" run-errand "${ERRAND_NAME}" ${PARAMS} 35 | } 36 | 37 | main 38 | -------------------------------------------------------------------------------- /run-errand/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: bbl-state 12 | optional: true 13 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 14 | optional: true 15 | - name: cf-deployment-concourse-tasks 16 | - name: pool-lock 17 | optional: true 18 | 19 | run: 20 | path: cf-deployment-concourse-tasks/run-errand/task 21 | 22 | params: 23 | BBL_STATE_DIR: bbl-state 24 | # - Path to the directory containing the `bbl-state.json` file 25 | # - The path is relative to the `bbl-state` input 26 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 27 | # - If the specified path does not exist, it will be created 28 | # - This input and parameter will be ignored if toolsmiths-env is provided 29 | 30 | DEPLOYMENT_NAME: cf 31 | # - Optional 32 | # - Deployment name for errand to be ran on 33 | # - Defaults to cf 34 | 35 | ERRAND_NAME: 36 | # - Required 37 | # - Errand name to be run 38 | 39 | INSTANCE: 40 | # - Optional 41 | # - Allows jobs to disambiguate between different copies of the same errand 42 | # - with different configurations 43 | 44 | KEEP_ALIVE: false 45 | # - Optional 46 | # - Specifies whether or not to keep instance alive after errand completion 47 | # - Defaults to false 48 | 49 | BBL_JSON_CONFIG: 50 | # - Optional 51 | # - For if you need a dynamic config for BBL 52 | -------------------------------------------------------------------------------- /set-feature-flags/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | # shellcheck disable=SC1091 4 | source cf-deployment-concourse-tasks/shared-functions 5 | 6 | function set_enabled_feature_flags() { 7 | if [ ! -z "${ENABLED_FEATURE_FLAGS}" ]; then 8 | for flag in $ENABLED_FEATURE_FLAGS; do 9 | set_feature_flag "$flag" true 10 | done 11 | fi 12 | } 13 | 14 | function set_disabled_feature_flags() { 15 | if [ ! -z "${DISABLED_FEATURE_FLAGS}" ]; then 16 | for flag in $DISABLED_FEATURE_FLAGS; do 17 | set_feature_flag "$flag" false 18 | done 19 | fi 20 | } 21 | 22 | function set_feature_flag() { 23 | if [ $2 == true ]; then 24 | cf enable-feature-flag "$1" 25 | else 26 | cf disable-feature-flag "$1" 27 | fi 28 | } 29 | 30 | function cf_login() { 31 | set +x 32 | echo "Authenticating as CF admin" 33 | cf auth admin $(get_password_from_credhub cf_admin_password) 34 | set -x 35 | } 36 | 37 | function main() { 38 | setup_bosh_env_vars 39 | 40 | if [ -z "${SYSTEM_DOMAIN}" ]; then 41 | echo "SYSTEM_DOMAIN is a required parameter" 42 | exit 1 43 | fi 44 | 45 | cf api --skip-ssl-validation "api.${SYSTEM_DOMAIN}" 46 | cf_login 47 | 48 | set_enabled_feature_flags 49 | set_disabled_feature_flags 50 | 51 | cf feature-flags 52 | } 53 | 54 | main 55 | -------------------------------------------------------------------------------- /set-feature-flags/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: cf-deployment-concourse-tasks # - This repo 12 | - name: bbl-state # - The repo containing the Director's bbl state dir 13 | optional: true 14 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 15 | optional: true 16 | 17 | run: 18 | path: cf-deployment-concourse-tasks/set-feature-flags/task 19 | 20 | params: 21 | SYSTEM_DOMAIN: 22 | # - Required unless toolsmiths-env optional input is provided 23 | # - CF system base domain e.g. `my-cf.com` 24 | 25 | BBL_STATE_DIR: bbl-state 26 | # - Optional 27 | # - Path to the directory containing the `bbl-state.json` file 28 | # - The path is relative to the `bbl-state` input 29 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 30 | # - If the specified path does not exist, it will be created 31 | # - This input and parameter will be ignored if toolsmiths-env is provided 32 | 33 | ENABLED_FEATURE_FLAGS: 34 | # - Optional 35 | # - Space seperated list of feature flag names to enable 36 | 37 | DISABLED_FEATURE_FLAGS: 38 | # - Optional 39 | # - Space seperated list of feature flag names to disable 40 | -------------------------------------------------------------------------------- /shared-functions: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | check_bbl_input_params() { 4 | set +x 5 | # Checks that we have IaaS credentials for GCP or AWS or Azure or OpenStack or vSphere 6 | if [ ! -z "${BBL_GCP_SERVICE_ACCOUNT_KEY}" ] || \ 7 | [ ! -z "${BBL_AWS_ACCESS_KEY_ID}" -a ! -z "${BBL_AWS_SECRET_ACCESS_KEY}" ] || \ 8 | [ ! -z "${BBL_AZURE_CLIENT_ID}" -a ! -z "${BBL_AZURE_CLIENT_SECRET}" -a ! -z "${BBL_AZURE_TENANT_ID}" -a ! -z "${BBL_AZURE_SUBSCRIPTION_ID}" ] || \ 9 | [ ! -z "${BBL_OPENSTACK_USERNAME}" -a ! -z "${BBL_OPENSTACK_PASSWORD}" -a ! -z "${BBL_OPENSTACK_DOMAIN}" -a ! -z "${BBL_OPENSTACK_PROJECT}" ] || \ 10 | [ ! -z "${BBL_VSPHERE_VCENTER_USER}" -a ! -z "${BBL_VSPHERE_VCENTER_PASSWORD}" -a ! -z "${BBL_VSPHERE_VCENTER_IP}" ]; then 11 | return 0 12 | else 13 | echo 'The BBL task must be configured with IaaS credentials. Please configure either `BBL_GCP_SERVICE_ACCOUNT_KEY`, or the `BBL_AWS_ACCESS_KEY_ID` `BBL_AWS_SECRET_ACCESS_KEY` pair, or `BBL_AZURE_CLIENT_ID` `BBL_AZURE_CLIENT_SECRET` `BBL_AZURE_TENANT_ID` `BBL_AZURE_SUBSCRIPTION_ID` tetrad, or `BBL_OPENSTACK_USERNAME` `BBL_OPENSTACK_PASSWORD` `BBL_OPENSTACK_DOMAIN` `BBL_OPENSTACK_PROJECT` tetrad' 14 | exit 1 15 | fi 16 | set -x 17 | } 18 | 19 | function check_input_params() { 20 | set +x 21 | if [ -z "$MANIFEST_FILE" ]; then 22 | echo "MANIFEST_FILE has not been set" 23 | exit 1 24 | fi 25 | 26 | if [ -z "$SYSTEM_DOMAIN" -a ! -d toolsmiths-env ]; then 27 | echo "SYSTEM_DOMAIN or toolsmiths-env needs to be passed in" 28 | exit 1 29 | fi 30 | 31 | if ${FAIL_ON_DOWNTIME}; then 32 | if [ -z "${DEPLOY_WITH_UPTIME_MEASUREMENTS}" ]; then 33 | echo "FAIL_ON_DOWNTIME requires that DEPLOY_WITH_UPTIME_MEASUREMENTS be true. Exiting." 34 | exit 1 35 | fi 36 | fi 37 | set -x 38 | } 39 | 40 | function load_from_json_config() { 41 | set +ux 42 | 43 | if [ -z "${BBL_JSON_CONFIG}" ]; then 44 | set -ux 45 | return 46 | fi 47 | 48 | keys=$( jq -r 'keys[]' "${BBL_JSON_CONFIG}" ) 49 | for key in $keys; do 50 | if [[ -v $key ]]; then 51 | echo "Getting $key from BBL_JSON_CONFIG" 52 | export $key=$( jq -r ".$key" "${BBL_JSON_CONFIG}" ) 53 | fi 54 | done 55 | set -ux 56 | } 57 | 58 | function commit_bbl_state_dir { 59 | local root_dir 60 | root_dir="${1}" 61 | local commit_message 62 | commit_message="${2}" 63 | 64 | pushd "${root_dir}/bbl-state/${BBL_STATE_DIR}" 65 | status="$(git status --porcelain)" 66 | if [[ -n "$status" ]]; then 67 | set_git_config 68 | git add --all . 69 | git commit -m "${commit_message}" 70 | fi 71 | popd 72 | 73 | pushd "${root_dir}" 74 | shopt -s dotglob 75 | cp -R "bbl-state/." "updated-bbl-state/" 76 | popd 77 | } 78 | 79 | function set_git_config() { 80 | git config user.name "${GIT_COMMIT_USERNAME}" 81 | git config user.email "${GIT_COMMIT_EMAIL}" 82 | } 83 | 84 | 85 | function setup_bosh_env_vars() { 86 | set +x 87 | if [ -d toolsmiths-env ]; then 88 | eval "$(bbl print-env --metadata-file toolsmiths-env/metadata)" 89 | export SYSTEM_DOMAIN="$(cat toolsmiths-env/metadata | jq -r '.cf.api_url | sub("api."; "")')" 90 | export TCP_DOMAIN="tcp.${SYSTEM_DOMAIN}" 91 | else 92 | if [ -d bbl-state ]; then 93 | pushd "bbl-state/${BBL_STATE_DIR}" 94 | eval "$(bbl print-env)" 95 | popd 96 | else 97 | echo "Must provide either toolsmiths-env or bbl-state as an input" 98 | exit 1 99 | fi 100 | fi 101 | set -x 102 | } 103 | 104 | function bosh_interpolate() { 105 | set +u 106 | local root_dir 107 | root_dir="${1}" 108 | 109 | local release_name 110 | release_name="${2}" 111 | 112 | local release_tarball_name 113 | release_tarball_name="${3}" 114 | set -u 115 | 116 | local bosh_manifest 117 | bosh_manifest="cf-deployment/${MANIFEST_FILE}" 118 | 119 | local arguments 120 | arguments="-v system_domain=${SYSTEM_DOMAIN}" 121 | 122 | for op in ${OPS_FILES} 123 | do 124 | arguments="${arguments} -o ops-files/${op}" 125 | done 126 | 127 | for vf in ${VARS_FILES} 128 | do 129 | arguments="${arguments} -l vars-files/${vf}" 130 | done 131 | 132 | if [ -n "${release_name}" ]; then 133 | if [ -n "${release_tarball_name}" ]; then 134 | tarball_path="${root_dir}/release/${release_tarball_name}" 135 | version=$(tar xzf "${tarball_path}" -O release.MF | yq -r '.version // "latest"') 136 | cat << EOF > create-provided-release.yml 137 | --- 138 | - type: replace 139 | path: /releases/name=${release_name} 140 | value: 141 | name: ${release_name} 142 | url: file://${tarball_path} 143 | version: ${version} 144 | EOF 145 | else 146 | cat << EOF > create-provided-release.yml 147 | --- 148 | - type: replace 149 | path: /releases/name=${release_name} 150 | value: 151 | name: ${release_name} 152 | version: create 153 | url: file://${root_dir}/release 154 | EOF 155 | fi 156 | 157 | arguments="${arguments} -o create-provided-release.yml" 158 | fi 159 | 160 | INTERPOLATED_MANIFEST=$(mktemp) 161 | export INTERPOLATED_MANIFEST 162 | 163 | # We are intentionally passing a series of arguments here: 164 | # shellcheck disable=SC2086 165 | local bosh_arguments 166 | bosh_arguments="" 167 | 168 | bosh \ 169 | -n \ 170 | interpolate ${arguments} \ 171 | ${bosh_arguments} \ 172 | "${bosh_manifest}" \ 173 | > "${INTERPOLATED_MANIFEST}" 174 | } 175 | 176 | write_uptimer_deploy_config() { 177 | local deployment_name 178 | deployment_name=${1} 179 | 180 | local manifest 181 | manifest=${2} 182 | 183 | # The remaining positional parameters are consumed by 184 | # jq as bosh deploy args 185 | shift 2 186 | 187 | # Give bogus values for TCP_DOMAIN, TCP_PORT, and 188 | # AVAILABLE_PORT so that we don't have to do jq magic. 189 | 190 | local tcp_domain 191 | tcp_domain=${TCP_DOMAIN:-" "} 192 | local tcp_port 193 | tcp_port=${TCP_PORT:-"-1"} 194 | local available_port 195 | available_port=${AVAILABLE_PORT:-"-1"} 196 | 197 | set +x 198 | local cf_admin_password 199 | 200 | cf_admin_password=$(get_password_from_credhub cf_admin_password) 201 | 202 | echo '{}' | jq --arg cf_api api.${SYSTEM_DOMAIN} \ 203 | --arg admin_password ${cf_admin_password} \ 204 | --arg app_domain ${SYSTEM_DOMAIN} \ 205 | --arg manifest ${manifest} \ 206 | --arg deployment_name ${deployment_name} \ 207 | --arg run_app_syslog_availability ${MEASURE_SYSLOG_AVAILABILITY} \ 208 | --arg run_tcp_availability ${MEASURE_TCP_AVAILABILITY} \ 209 | --arg tcp_domain "${tcp_domain}" \ 210 | --arg tcp_port ${tcp_port} \ 211 | --arg available_port ${available_port} \ 212 | --arg app_pushability ${APP_PUSHABILITY_THRESHOLD} \ 213 | --arg app_stats ${APP_STATS_THRESHOLD} \ 214 | --arg http_availability ${HTTP_AVAILABILITY_THRESHOLD} \ 215 | --arg tcp_availability ${TCP_AVAILABILITY_THRESHOLD} \ 216 | --arg recent_logs ${RECENT_LOGS_THRESHOLD} \ 217 | --arg streaming_logs ${STREAMING_LOGS_THRESHOLD} \ 218 | --arg use_single_app_instance ${USE_SINGLE_APP_INSTANCE} \ 219 | --arg app_syslog_availability ${APP_SYSLOG_AVAILABILITY_THRESHOLD} \ 220 | --args \ 221 | '{ 222 | "while": [{ 223 | "command":"bosh", 224 | "command_args":["--tty", "-n", "deploy", $manifest, "-d", $deployment_name, $ARGS.positional[]] 225 | }], 226 | "cf": { 227 | "api": $cf_api, 228 | "app_domain": $app_domain, 229 | "admin_user": "admin", 230 | "admin_password": $admin_password, 231 | "tcp_domain": $tcp_domain, 232 | "available_port": $available_port | tonumber, 233 | "tcp_port": $tcp_port | tonumber, 234 | "use_single_app_instance": $use_single_app_instance | ascii_downcase | contains("true") 235 | }, 236 | "allowed_failures": { 237 | "app_pushability": $app_pushability | tonumber, 238 | "app_stats": $app_stats | tonumber, 239 | "http_availability": $http_availability | tonumber, 240 | "tcp_availability": $tcp_availability | tonumber, 241 | "recent_logs": $recent_logs | tonumber, 242 | "streaming_logs": $streaming_logs | tonumber, 243 | "app_syslog_availability": $app_syslog_availability | tonumber 244 | }, 245 | "optional_tests": { 246 | "run_app_syslog_availability": $run_app_syslog_availability | ascii_downcase | contains("true"), 247 | "run_tcp_availability": $run_tcp_availability | ascii_downcase | contains("true") 248 | } 249 | }' \ 250 | -- ${@} 251 | set -x 252 | } 253 | 254 | function uptimer_bosh_deploy() { 255 | if ${MEASURE_SYSLOG_AVAILABILITY}; then 256 | if [ -z "${TCP_DOMAIN}" ] || [ -z "${AVAILABLE_PORT}" ]; then 257 | echo "Both TCP_DOMAIN and AVAILABLE_PORT are required to run syslog availability measurement." 258 | exit 1 259 | fi 260 | fi 261 | 262 | local deployment_name 263 | deployment_name=$(bosh interpolate "${INTERPOLATED_MANIFEST}" --path /name) 264 | 265 | uptimer_config=$(mktemp) 266 | write_uptimer_deploy_config "${deployment_name}" "${INTERPOLATED_MANIFEST}" ${@} > ${uptimer_config} 267 | pushd cf-deployment > /dev/null 268 | set +e 269 | local exitStatus 270 | uptimer -configFile ${uptimer_config} 271 | exitStatus=$? 272 | set -e 273 | popd 274 | 275 | if [ "${FAIL_ON_DOWNTIME}" = "false" ]; then 276 | # exitStatus 64 means that there was downtime, but the deployment was successful 277 | # exitStatus 70 means that there was a measurement setup failure, but the deployment was successful 278 | if [ $exitStatus == 64 ] || [ $exitStatus == 70 ]; then 279 | exit 0 280 | fi 281 | fi 282 | 283 | exit $exitStatus 284 | } 285 | 286 | function bosh_deploy() { 287 | local deployment_name 288 | deployment_name=$(bosh interpolate "${INTERPOLATED_MANIFEST}" --path /name) 289 | 290 | if ${DEPLOY_WITH_UPTIME_MEASUREMENTS}; then 291 | uptimer_bosh_deploy ${@} 292 | else 293 | pushd cf-deployment > /dev/null 294 | bosh \ 295 | -n \ 296 | -d "${deployment_name}" \ 297 | deploy \ 298 | "${INTERPOLATED_MANIFEST}" \ 299 | ${@} 300 | popd 301 | fi 302 | } 303 | 304 | function remove_credentials_from_credhub_in_directory() { 305 | local directory_name 306 | directory_name=$1 307 | set +x 308 | credentials_in_directory=$(credhub f -p ${directory_name} -j | jq -r .credentials[].name) 309 | 310 | echo "Deleting credentials in the ${directory_name} directory from CredHub:" 311 | for cred in ${credentials_in_directory}; do 312 | echo "Deleting ${cred} from CredHub" 313 | credhub d -n $cred > /dev/null 314 | done 315 | set -x 316 | } 317 | 318 | function remove_credentials_from_credhub() { 319 | local directory_name 320 | local deployment_name 321 | if [ -d toolsmiths-env ]; then 322 | deployment_name=$(jq -r .name toolsmiths-env/metadata) 323 | directory_name="/bosh-${deployment_name}" 324 | else 325 | local director_name 326 | director_name=$(jq -r .bosh.directorName bbl-state/${BBL_STATE_DIR}/bbl-state.json) 327 | deployment_name=$(bosh interpolate "${INTERPOLATED_MANIFEST}" --path /name) 328 | directory_name="/${director_name}/${deployment_name}" 329 | set +x 330 | credentials=$(credhub f -n /dns -j | jq -r .credentials[].name) 331 | 332 | echo "Deleting DNS credentials from CredHub:" 333 | for cred in ${credentials}; do 334 | echo "Deleting ${cred} from CredHub" 335 | credhub d -n $cred > /dev/null 336 | done 337 | set -x 338 | fi 339 | remove_credentials_from_credhub_in_directory ${directory_name} 340 | } 341 | 342 | write_gcp_service_account_key() { 343 | set +x 344 | if [ -f "${BBL_GCP_SERVICE_ACCOUNT_KEY}" ]; then 345 | cp "${BBL_GCP_SERVICE_ACCOUNT_KEY}" /tmp/google_service_account.json 346 | else 347 | echo "${BBL_GCP_SERVICE_ACCOUNT_KEY}" > /tmp/google_service_account.json 348 | fi 349 | export BBL_GCP_SERVICE_ACCOUNT_KEY="/tmp/google_service_account.json" 350 | set -x 351 | } 352 | 353 | get_password_from_credhub() { 354 | set +x 355 | local bosh_manifest_password_variable_name=$1 356 | 357 | local credential_path=$(credhub find -j -n ${bosh_manifest_password_variable_name} | jq -r .credentials[].name ) 358 | local credential_paths_len=$(echo ${credential_path} | tr ' ' '\n' | wc -l) 359 | 360 | if [ "${credential_paths_len}" -gt 1 ]; then 361 | echo "ambiguous ${bosh_manifest_password_variable_name} variable name; expected one got ${credential_paths_len}" >&2 362 | echo "${credential_path}" | tr ' ' '\n' >&2 363 | return 364 | elif [ "${credential_paths_len}" -eq 0 ]; then 365 | echo "${bosh_manifest_password_variable_name} variable not found" >&2 366 | return 367 | fi 368 | 369 | echo $(credhub find -j -n ${bosh_manifest_password_variable_name} | jq -r .credentials[].name | xargs credhub get -j -n | jq -r .value) 370 | set -x 371 | } 372 | 373 | setup_password_from_credhub() { 374 | set +x 375 | local bosh_manifest_password_variable_name=$1 376 | local environment_variable_name=$1 377 | if [ "$#" -gt 1 ]; then 378 | environment_variable_name=$2 379 | fi 380 | 381 | local credential_path=$(credhub find -j -n ${bosh_manifest_password_variable_name} | jq -r .credentials[].name ) 382 | local credential_paths_len=$(echo ${credential_path} | tr ' ' '\n' | wc -l) 383 | if [ "${credential_paths_len}" -gt 1 ]; then 384 | echo "ambiguous password variable name; expected one got ${credential_paths_len}" 385 | echo "${credential_path}" | tr ' ' '\n' 386 | exit 1 387 | fi 388 | 389 | export "${environment_variable_name}"=$(credhub find -j -n ${bosh_manifest_password_variable_name} | jq -r .credentials[].name | xargs credhub get -j -n | jq -r .value) 390 | set -x 391 | } 392 | 393 | upload_stemcells() { 394 | local arguments='' 395 | for op in ${OPS_FILES} 396 | do 397 | arguments="${arguments} -o ops-files/${op}" 398 | done 399 | 400 | bosh interpolate ${arguments} cf-deployment/${MANIFEST_FILE} > /tmp/cf.yml 401 | 402 | local stemcells_json=$(ruby -rjson -ryaml -e "puts YAML.load_file('/tmp/cf.yml').to_json" | jq .stemcells) 403 | local size=$(echo ${stemcells_json} | jq 'length') 404 | if [ ${size} -eq 0 ]; then 405 | echo "Error: Did not find any stemcells to upload." 406 | exit 1 407 | fi 408 | 409 | for i in `seq 0 $((size - 1))` 410 | do 411 | local os=$(echo ${stemcells_json} | jq -r .[$i].os) 412 | local version=$(echo ${stemcells_json} | jq -r .[$i].version) 413 | upload_stemcell ${os} ${version} 414 | done 415 | } 416 | 417 | upload_stemcell() { 418 | # Read potentially variable stemcell paramaters out of cf-deployment with bosh 419 | local os 420 | os=$1 421 | local version 422 | version=$2 423 | 424 | # Hardcode a couple of stable stemcell paramaters 425 | local stemcells_url 426 | stemcells_url="https://bosh.io/d/stemcells" 427 | 428 | # Ask bosh if it already has our OS / version stemcell combination 429 | # As of this writing, the new bosh cli doesn't have --skip-if-exists 430 | set +e 431 | local existing_stemcell 432 | existing_stemcell=$(bosh stemcells | grep "${os}" | awk '{print $2}' | tr -d "\*" | grep ^"${version}"$ ) 433 | set -e 434 | 435 | local stemcell_name 436 | 437 | local infrastructure 438 | if [ "${BOSH_LITE}" == "true" ]; then 439 | infrastructure="bosh-lite" 440 | stemcell_name="bosh-warden-boshlite" 441 | elif [ -d toolsmiths-env ]; then 442 | infrastructure="gcp" 443 | stemcell_name="bosh-google-kvm" 444 | else 445 | infrastructure="$(jq -r .iaas bbl-state/${BBL_STATE_DIR}/bbl-state.json)" 446 | 447 | if [ "$infrastructure" = "aws" ]; then 448 | stemcell_name="bosh-aws-xen-hvm" 449 | elif [ "$infrastructure" = "gcp" ]; then 450 | stemcell_name="bosh-google-kvm" 451 | elif [ "$infrastructure" = "vsphere" ]; then 452 | stemcell_name="bosh-vsphere-esxi" 453 | elif [ "$infrastructure" = "azure" ]; then 454 | stemcell_name="bosh-azure-hyperv" 455 | elif [ "$infrastructure" = "openstack" ]; then 456 | stemcell_name="bosh-openstack-kvm" 457 | fi 458 | fi 459 | 460 | stemcell_name="${stemcell_name}-${os}-go_agent" 461 | if [ "$version" = "latest" ]; then 462 | full_stemcell_url="${stemcells_url}/${stemcell_name}" 463 | else 464 | full_stemcell_url="${stemcells_url}/${stemcell_name}?v=${version}" 465 | fi 466 | 467 | # If bosh already has our stemcell, exit 0 468 | if [ "${existing_stemcell}" ]; then 469 | echo "Task bosh-upload-stemcell-from-cf-deployment:" 470 | echo "Stemcell '${stemcell_name}/${version}' already exists. Skipping..." 471 | return 472 | fi 473 | 474 | # If bosh.io doesn't have our stemcell, exit 2 475 | set +e 476 | local stemcell_exists 477 | wget -S --spider "${full_stemcell_url}" > /dev/null 2>&1 478 | stemcell_exists=$? 479 | if [ ${stemcell_exists} -ne 0 ]; then 480 | echo "Error: Could not find a '$os' stemcell for IaaS '$infrastructure' on bosh.io. Please double-check that the IaaS/OS combination is supported." 481 | exit 2 #POSIX 'No such file or directory' 482 | fi 483 | set -e 484 | 485 | # ... otherwise, begin the upload process 486 | bosh \ 487 | -n \ 488 | upload-stemcell \ 489 | "${full_stemcell_url}" 490 | } 491 | -------------------------------------------------------------------------------- /update-integration-configs/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | set -o pipefail 3 | 4 | # shellcheck disable=SC1091 5 | source cf-deployment-concourse-tasks/shared-functions 6 | 7 | function check_fast_fails() { 8 | if [ ! -f "integration-configs/$CATS_INTEGRATION_CONFIG_FILE" -a ! -f "integration-configs/$RATS_INTEGRATION_CONFIG_FILE" -a ! -f "integration-configs/$WATS_INTEGRATION_CONFIG_FILE" ]; then 9 | echo "One of the following files should be present, but none were found: cats_integration_config, rats_integration_config, wats_integration_config. Exiting." 10 | exit 1 11 | fi 12 | } 13 | 14 | 15 | function commit_integration_configs { 16 | pushd integration-configs 17 | if [[ -n $(git status --porcelain) ]]; then 18 | set_git_config 19 | 20 | if [ -f "${CATS_INTEGRATION_CONFIG_FILE}" ]; then 21 | git add "${CATS_INTEGRATION_CONFIG_FILE}" 22 | fi 23 | 24 | if [ -f "${RATS_INTEGRATION_CONFIG_FILE}" ]; then 25 | git add "${RATS_INTEGRATION_CONFIG_FILE}" 26 | fi 27 | 28 | if [ -f "${WATS_INTEGRATION_CONFIG_FILE}" ]; then 29 | git add "${WATS_INTEGRATION_CONFIG_FILE}" 30 | fi 31 | 32 | git commit -m "Update integration configs" 33 | fi 34 | popd 35 | 36 | git clone integration-configs/ updated-integration-configs 37 | } 38 | 39 | function main() { 40 | local root_dir 41 | root_dir="${1}" 42 | 43 | check_fast_fails 44 | setup_bosh_env_vars 45 | 46 | set +x 47 | local admin_password 48 | admin_password=$(get_password_from_credhub cf_admin_password) 49 | 50 | if [ -f "integration-configs/${CATS_INTEGRATION_CONFIG_FILE}" ]; then 51 | echo "updating CATs integration config file: ${CATS_INTEGRATION_CONFIG_FILE}..." 52 | 53 | local new_cats_integration_config 54 | new_cats_integration_config=$(cat "integration-configs/${CATS_INTEGRATION_CONFIG_FILE}" | jq ".admin_password=\"${admin_password}\"") 55 | 56 | local credhub_secret=$(get_password_from_credhub credhub_admin_client_secret) 57 | new_cats_integration_config=$(echo "${new_cats_integration_config}" | jq ".credhub_secret=\"${credhub_secret}\"") 58 | new_cats_integration_config=$(echo "${new_cats_integration_config}" | jq ".credhub_client=\"credhub_admin_client\"") 59 | 60 | if [ -n "${SYSTEM_DOMAIN}" ]; then 61 | new_cats_integration_config=$(echo "${new_cats_integration_config}" | jq ".api=\"api.${SYSTEM_DOMAIN}\" | .apps_domain=\"${SYSTEM_DOMAIN}\"") 62 | fi 63 | 64 | if [[ -n "${PRIVATE_DOCKER_REGISTRY_IMAGE}" && -n "${PRIVATE_DOCKER_REGISTRY_USERNAME}" && -n "${PRIVATE_DOCKER_REGISTRY_PASSWORD}" ]]; then 65 | new_cats_integration_config=$( \ 66 | echo "${new_cats_integration_config}" | \ 67 | jq ".private_docker_registry_image=\"${PRIVATE_DOCKER_REGISTRY_IMAGE}\"" | \ 68 | jq ".private_docker_registry_username=\"${PRIVATE_DOCKER_REGISTRY_USERNAME}\"" | \ 69 | jq ".private_docker_registry_password=\"${PRIVATE_DOCKER_REGISTRY_PASSWORD}\"") 70 | elif [[ -n "${PRIVATE_DOCKER_REGISTRY_IMAGE}" || -n "${PRIVATE_DOCKER_REGISTRY_USERNAME}" || -n "${PRIVATE_DOCKER_REGISTRY_PASSWORD}" ]]; then 71 | echo "All 3 private docker registry keys must be provided. (PRIVATE_DOCKER_REGISTRY_IMAGE, PRIVATE_DOCKER_REGISTRY_USERNAME, and PRIVATE_DOCKER_REGISTRY_PASSWORD)" 72 | exit 1 73 | fi 74 | 75 | echo "${new_cats_integration_config}" > "integration-configs/${CATS_INTEGRATION_CONFIG_FILE}" 76 | fi 77 | 78 | if [ -f "integration-configs/${RATS_INTEGRATION_CONFIG_FILE}" ]; then 79 | echo "updating RATs integration config file: ${RATS_INTEGRATION_CONFIG_FILE}..." 80 | 81 | local uaa_clients_routing_api_client_secret 82 | uaa_clients_routing_api_client_secret=$(get_password_from_credhub uaa_clients_routing_api_client_secret) 83 | 84 | local new_rats_integration_config 85 | new_rats_integration_config=$(cat "integration-configs/${RATS_INTEGRATION_CONFIG_FILE}") 86 | new_rats_integration_config=$(echo ${new_rats_integration_config} | jq ".admin_password=\"${admin_password}\"") 87 | new_rats_integration_config=$(echo ${new_rats_integration_config} | jq ".oauth*={\"client_secret\":\"${uaa_clients_routing_api_client_secret}\"}") 88 | 89 | echo "${new_rats_integration_config}" > "integration-configs/${RATS_INTEGRATION_CONFIG_FILE}" 90 | fi 91 | 92 | if [ -f "integration-configs/${WATS_INTEGRATION_CONFIG_FILE}" ]; then 93 | echo "updating WATs integration config file: ${WATS_INTEGRATION_CONFIG_FILE}..." 94 | 95 | local new_cats_integration_config 96 | new_wats_integration_config=$(cat "integration-configs/${WATS_INTEGRATION_CONFIG_FILE}" | jq ".admin_password=\"${admin_password}\"") 97 | 98 | echo "${new_wats_integration_config}" > "integration-configs/${WATS_INTEGRATION_CONFIG_FILE}" 99 | fi 100 | set -x 101 | 102 | } 103 | 104 | trap commit_integration_configs EXIT 105 | 106 | main "${PWD}" 107 | -------------------------------------------------------------------------------- /update-integration-configs/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: registry-image 6 | source: 7 | repository: cloudfoundry/cf-deployment-concourse-tasks 8 | tag: latest 9 | 10 | inputs: 11 | - name: bbl-state # - The repo containing the Director's bbl state dir 12 | optional: true 13 | - name: toolsmiths-env # - Directory containing Toolsmiths metadata file 14 | optional: true 15 | - name: cf-deployment-concourse-tasks # - This repo 16 | - name: integration-configs # - Integration configs to be updated 17 | 18 | outputs: 19 | - name: updated-integration-configs 20 | # The integration-configs input, with changes git committed, 21 | # ready to be `put` to a git resource. 22 | 23 | run: 24 | path: cf-deployment-concourse-tasks/update-integration-configs/task 25 | 26 | params: 27 | # Config Filepaths 28 | # - At least one of these must be specified and present 29 | # - It's fine to set both 30 | # - These paths are relative to root of the `integration-configs` input 31 | CATS_INTEGRATION_CONFIG_FILE: cats_integration_config.json 32 | RATS_INTEGRATION_CONFIG_FILE: rats_integration_config.json 33 | WATS_INTEGRATION_CONFIG_FILE: wats_integration_config.json 34 | 35 | BBL_STATE_DIR: bbl-state 36 | # - Optional 37 | # - Path to the directory containing the `bbl-state.json` file 38 | # - The path is relative to the `bbl-state` input 39 | # - If blank or '.', uses `bbl-state` input as the location for bbl state 40 | # - If the specified path does not exist, it will be created 41 | # - This input and parameter will be ignored if toolsmiths-env is provided 42 | 43 | SYSTEM_DOMAIN: 44 | # - Optional 45 | # - CF system base domain e.g. `my-cf.com` 46 | # - Should match the value passed to `bosh-deploy` 47 | 48 | GIT_COMMIT_EMAIL: "cf-release-integration@pivotal.io" 49 | GIT_COMMIT_USERNAME: "CI Bot" 50 | # - Optional 51 | # - You may choose the git committer username and email address by setting these 52 | 53 | PRIVATE_DOCKER_REGISTRY_IMAGE: 54 | PRIVATE_DOCKER_REGISTRY_USERNAME: 55 | PRIVATE_DOCKER_REGISTRY_PASSWORD: 56 | # - Optional 57 | # - You may add private docker registry information to the integration-configs 58 | # - In order to apply private docker registry config, you must provide PRIVATE_DOCKER_REGISTRY_IMAGE & PRIVATE_DOCKER_REGISTRY_USERNAME & PRIVATE_DOCKER_REGISTRY_PASSWORD 59 | --------------------------------------------------------------------------------