├── .gitignore ├── .gitmodules ├── LICENSE ├── NOTICE ├── dockerfiles ├── deployment │ └── Dockerfile ├── golang │ └── Dockerfile ├── minimal │ └── Dockerfile └── vagrant │ └── Dockerfile ├── pipelines ├── bootload-mega-ci.yml ├── bosh-bootloader.yml ├── bosh-test.yml ├── check-a-record.yml ├── etcd.yml ├── gomegamatchers.yml └── mega-ci.yml ├── scripts ├── ci │ ├── bosh-bootloader │ │ ├── aws_integration │ │ ├── aws_integration.yml │ │ ├── build-final-release │ │ │ ├── task │ │ │ └── task.yml │ │ ├── concourse_integration │ │ ├── concourse_integration.yml │ │ ├── gcp_integration │ │ ├── gcp_integration.yml │ │ ├── test │ │ ├── test.yml │ │ └── update-release-constants │ │ │ ├── task.sh │ │ │ └── task.yml │ ├── bosh-test │ │ ├── test │ │ └── test.yml │ ├── check-a-record │ │ ├── test │ │ └── test.yml │ ├── check-git-submodules │ │ ├── task │ │ └── task.yml │ ├── compile-bosh-release │ │ ├── fixtures │ │ │ └── compilation.yml │ │ ├── task-release-from-repo.sh │ │ ├── task-release-from-repo.yml │ │ ├── task.sh │ │ └── task.yml │ ├── confab │ │ ├── test │ │ └── test.yml │ ├── create-final-release │ │ ├── task │ │ └── task.yml │ ├── delete-deployments │ │ ├── task.sh │ │ └── task.yml │ ├── deploy-aws-manifests │ │ ├── awsdeployer │ │ │ ├── awsdeployer.go │ │ │ ├── awsdeployer_test.go │ │ │ └── init_test.go │ │ ├── clients │ │ │ ├── aws.go │ │ │ ├── aws_test.go │ │ │ ├── bosh.go │ │ │ ├── bosh_test.go │ │ │ └── init_test.go │ │ ├── deploy-consul-aws-manifests │ │ ├── deploy-consul-aws-manifests.yml │ │ ├── deploy-etcd-aws-manifests │ │ ├── deploy-etcd-aws-manifests.yml │ │ ├── fakes │ │ │ ├── aws.go │ │ │ ├── bosh.go │ │ │ ├── session.go │ │ │ └── subnet_checker.go │ │ ├── fixtures │ │ │ └── multi-az-ssl.yml │ │ ├── flags │ │ │ ├── flags.go │ │ │ ├── flags_test.go │ │ │ └── init_test.go │ │ ├── init_test.go │ │ ├── main.go │ │ ├── main_test.go │ │ ├── manifests │ │ │ ├── init_test.go │ │ │ ├── manifests.go │ │ │ └── manifests_test.go │ │ └── subnetchecker │ │ │ ├── init_test.go │ │ │ ├── subnetchecker.go │ │ │ └── subnetchecker_test.go │ ├── deploy-bosh │ ├── deploy-bosh-lite-manifests │ ├── deploy-bosh-lite-manifests.yml │ ├── deploy-bosh.yml │ ├── deploy-consul-cf │ │ ├── task │ │ └── task.yml │ ├── deploy-consul-diego │ │ ├── task │ │ └── task.yml │ ├── deploy-etcd-cf │ │ ├── task │ │ └── task.yml │ ├── deploy-etcd-diego │ │ ├── task │ │ └── task.yml │ ├── gomegamatchers │ │ ├── test │ │ └── test.yml │ ├── merge-master-into-develop │ │ ├── task │ │ └── task.yml │ ├── run-bosh-cleanup │ ├── run-bosh-cleanup.yml │ ├── run-bosh-command │ ├── run-bosh-command.yml │ ├── run-consats │ │ ├── fixtures │ │ │ ├── consul_compilation.yml │ │ │ ├── example.yml │ │ │ ├── expected.yml │ │ │ └── malformed.yml │ │ ├── generate_manifest.go │ │ ├── generate_manifest_test.go │ │ ├── init_test.go │ │ ├── task │ │ └── task.yml │ ├── run-eats │ │ ├── fixtures │ │ │ └── etcd_compilation.yml │ │ ├── task │ │ ├── task-turbulence │ │ ├── task-turbulence.yml │ │ └── task.yml │ ├── test │ ├── test-cf-tls-upgrade │ │ ├── task │ │ └── task.yml │ ├── test-diego │ │ ├── task │ │ └── task.yml │ ├── test-etcd-metrics-server │ │ ├── test │ │ └── test.yml │ ├── test-etcd-proxy │ │ ├── task │ │ └── task.yml │ ├── test-release │ ├── test-release.yml │ └── test.yml ├── common │ ├── atc_credentials.sh │ ├── bosh_passwords.sh │ ├── generate_certificate.sh │ └── password.sh ├── configure_final_release_bucket ├── deploy_concourse ├── setup_aws_bosh_for_concourse └── test └── templates ├── bootload-mega-ci ├── aws-for-cloudfoundry.json ├── aws-for-concourse.json └── s3-for-cloudfoundry.json ├── bosh ├── bosh-deployment-mask.yml ├── bosh-init-mask.yml └── bosh-init.yml ├── cf └── cf-stub-mask.yml ├── concourse └── concourse.yml ├── etcd └── iaas-settings.yml ├── final-release └── bucket.json └── turbulence ├── iaas-settings.yml └── turbulence-property-overrides.yml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | artifacts 3 | generated-stubs 4 | certs 5 | .DS_Store 6 | configure_* 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "scripts/ci/deploy-aws-manifests/vendor/github.com/aws/aws-sdk-go"] 2 | path = scripts/ci/deploy-aws-manifests/vendor/github.com/aws/aws-sdk-go 3 | url = https://github.com/aws/aws-sdk-go.git 4 | [submodule "scripts/ci/deploy-aws-manifests/vendor/github.com/pivotal-cf-experimental/bosh-test"] 5 | path = scripts/ci/deploy-aws-manifests/vendor/github.com/pivotal-cf-experimental/bosh-test 6 | url = https://github.com/pivotal-cf-experimental/bosh-test.git 7 | [submodule "scripts/ci/run-consats/vendor/github.com/pivotal-cf-experimental/gomegamatchers"] 8 | path = scripts/ci/run-consats/vendor/github.com/pivotal-cf-experimental/gomegamatchers 9 | url = https://github.com/pivotal-cf-experimental/gomegamatchers.git 10 | [submodule "scripts/ci/run-consats/vendor/gopkg.in/yaml.v2"] 11 | path = scripts/ci/run-consats/vendor/gopkg.in/yaml.v2 12 | url = https://gopkg.in/yaml.v2 13 | [submodule "scripts/ci/deploy-aws-manifests/vendor/gopkg.in/yaml.v2"] 14 | path = scripts/ci/deploy-aws-manifests/vendor/gopkg.in/yaml.v2 15 | url = https://gopkg.in/yaml.v2 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | mega-ci 2 | 3 | Copyright (c) CloudFoundry.org Foundation, Inc. 2016. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /dockerfiles/deployment/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM cfinfrastructure/golang 2 | MAINTAINER https://github.com/cloudfoundry/mega-ci 3 | 4 | RUN \ 5 | apt-get update && \ 6 | apt-get -qqy install --fix-missing \ 7 | awscli \ 8 | openssl \ 9 | unzip \ 10 | && \ 11 | apt-get clean 12 | 13 | # Install ruby-install 14 | RUN curl https://codeload.github.com/postmodern/ruby-install/tar.gz/v0.5.0 | tar xvz -C /tmp/ && \ 15 | cd /tmp/ruby-install-0.5.0 && \ 16 | make install 17 | 18 | # Install Ruby 19 | RUN ruby-install ruby 2.2.4 -- --disable-install-rdoc 20 | 21 | # Add ruby to PATH 22 | ENV PATH $PATH:/home/root/.gem/ruby/2.2.4/bin:/opt/rubies/ruby-2.2.4/lib/ruby/gems/2.2.4/bin:/opt/rubies/ruby-2.2.4/bin 23 | 24 | # Set permissions on ruby directory 25 | RUN chmod -R 777 /opt/rubies/ 26 | 27 | # Install bundler 28 | RUN /opt/rubies/ruby-2.2.4/bin/gem install bundler --no-rdoc --no-ri 29 | 30 | # Install bosh_cli 31 | RUN /opt/rubies/ruby-2.2.4/bin/gem install bosh_cli --no-rdoc --no-ri 32 | 33 | # Install terraform 34 | RUN wget https://releases.hashicorp.com/terraform/0.8.4/terraform_0.8.4_linux_amd64.zip && \ 35 | unzip terraform_0.8.4_linux_amd64.zip && \ 36 | rm terraform_0.8.4_linux_amd64.zip && \ 37 | mv terraform /usr/local/bin/terraform 38 | 39 | # Install jq 40 | RUN wget https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && \ 41 | mv jq-linux64 /usr/local/bin/jq && \ 42 | chmod +x /usr/local/bin/jq 43 | 44 | # Install spiff 45 | RUN wget https://github.com/cloudfoundry-incubator/spiff/releases/download/v1.0.7/spiff_linux_amd64 && \ 46 | mv spiff_linux_amd64 /usr/local/bin/spiff && \ 47 | chmod +x /usr/local/bin/spiff 48 | 49 | # Install bosh-init 50 | RUN wget https://s3.amazonaws.com/bosh-init-artifacts/bosh-init-0.0.98-linux-amd64 && \ 51 | mv bosh-init-0.0.98-linux-amd64 /usr/local/bin/bosh-init && \ 52 | chmod +x /usr/local/bin/bosh-init 53 | 54 | RUN curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&source=github" | tar -zx && \ 55 | chmod +x cf && \ 56 | mv cf /usr/local/bin/cf 57 | -------------------------------------------------------------------------------- /dockerfiles/golang/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM cfinfrastructure/minimal 2 | MAINTAINER https://github.com/cloudfoundry/mega-ci 3 | 4 | # Install go 5 | RUN wget https://storage.googleapis.com/golang/go1.7.3.linux-amd64.tar.gz && \ 6 | tar -C /usr/local -xzf go1.7.3.linux-amd64.tar.gz && \ 7 | rm -rf go1.7.3.linux-amd64.tar.gz 8 | 9 | # Create directory for GOPATH 10 | RUN mkdir -p /go/bin 11 | 12 | # set GOPATH 13 | ENV GOPATH /go 14 | 15 | # add go and GOPATH/bin to PATH 16 | ENV PATH $PATH:$GOPATH/bin:/usr/local/go/bin 17 | 18 | # use the vendor experiment 19 | ENV GO15VENDOREXPERIMENT 1 20 | 21 | # install test dependencies 22 | RUN go get github.com/onsi/ginkgo/... 23 | RUN go get github.com/onsi/gomega/... 24 | 25 | RUN chown -R testuser:testuser /go 26 | -------------------------------------------------------------------------------- /dockerfiles/minimal/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER https://github.com/cloudfoundry/mega-ci 3 | 4 | RUN \ 5 | apt-get update && \ 6 | apt-get -qqy install --fix-missing \ 7 | build-essential \ 8 | curl \ 9 | git \ 10 | libreadline6 \ 11 | libreadline6-dev \ 12 | wget \ 13 | runit \ 14 | lsof \ 15 | && \ 16 | apt-get clean 17 | 18 | # Create testuser 19 | RUN mkdir -p /home/testuser && \ 20 | groupadd -r testuser -g 433 && \ 21 | useradd -u 431 -r -g testuser -d /home/testuser -s /usr/sbin/nologin -c "Docker image test user" testuser 22 | -------------------------------------------------------------------------------- /dockerfiles/vagrant/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM cfinfrastructure/deployment 2 | MAINTAINER https://github.com/cloudfoundry/mega-ci 3 | 4 | # Install vagrant 5 | RUN wget https://releases.hashicorp.com/vagrant/1.8.1/vagrant_1.8.1_x86_64.deb -P /tmp && \ 6 | dpkg --install /tmp/vagrant_1.8.1_x86_64.deb && \ 7 | vagrant plugin install vagrant-aws && \ 8 | rm -rf /tmp/* 9 | -------------------------------------------------------------------------------- /pipelines/bootload-mega-ci.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: bootload-mega-ci 3 | jobs: 4 | - deploy-mega-ci-bosh 5 | 6 | resources: 7 | - name: mega-ci-env 8 | type: git 9 | source: 10 | branch: master 11 | private_key: {{mega-ci-env-writable-private-key}} 12 | uri: git@github.com:cloudfoundry/mega-ci-env.git 13 | - name: mega-ci 14 | type: git 15 | source: 16 | branch: master 17 | uri: https://github.com/cloudfoundry/mega-ci.git 18 | 19 | jobs: 20 | - name: deploy-mega-ci-bosh 21 | public: true 22 | plan: 23 | - aggregate: 24 | - get: mega-ci-env 25 | - get: mega-ci 26 | - task: deploy-bosh 27 | file: mega-ci/scripts/ci/deploy-bosh.yml 28 | - put: mega-ci-env 29 | params: 30 | repository: mega-ci-env-out 31 | -------------------------------------------------------------------------------- /pipelines/bosh-bootloader.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: bosh-bootloader 3 | jobs: 4 | - test-bosh-bootloader 5 | - aws-integration-tests 6 | - gcp-integration-tests 7 | - concourse-integration-tests 8 | - name: final-release 9 | jobs: 10 | - major 11 | - minor 12 | - patch 13 | - github-release 14 | - name: release-updates 15 | jobs: 16 | - build-compiled-bosh-release 17 | - update-release-constants 18 | 19 | resources: 20 | - name: bosh-bootloader-develop 21 | type: git 22 | source: 23 | branch: develop 24 | uri: https://github.com/cloudfoundry/bosh-bootloader.git 25 | - name: bosh-bootloader-develop-write 26 | type: git 27 | source: 28 | branch: develop 29 | uri: git@github.com:cloudfoundry/bosh-bootloader.git 30 | private_key: {{cf_infra_bot_user_github_private_key}} 31 | - name: bosh-bootloader-master 32 | type: git 33 | source: 34 | branch: master 35 | uri: https://github.com/cloudfoundry/bosh-bootloader.git 36 | - name: mega-ci 37 | type: git 38 | source: 39 | branch: master 40 | uri: https://github.com/cloudfoundry/mega-ci.git 41 | - name: bbl-integration-s3 42 | type: s3 43 | source: 44 | bucket: bbl-integration-tests 45 | regexp: (.*).tgz 46 | access_key_id: {{bbl_integration_tests_s3_access_key_id}} 47 | secret_access_key: {{bbl_integration_tests_s3_secret_access_key}} 48 | - name: version 49 | type: semver 50 | source: 51 | initial_version: 0.0.1 52 | driver: s3 53 | bucket: bbl-version 54 | key: bbl-version 55 | access_key_id: {{bbl_version_s3_access_key_id}} 56 | secret_access_key: {{bbl_version_s3_secret_access_key}} 57 | - name: bbl-release 58 | type: github-release 59 | source: 60 | user: cloudfoundry 61 | repository: bosh-bootloader 62 | access_token: {{cf_infra_bot_user_github_access_token}} 63 | - name: bbl-compiled-bosh-release-s3 64 | type: s3 65 | source: 66 | bucket: bbl-precompiled-bosh-releases 67 | regexp: release-bosh-(.*)-on-ubuntu-trusty-stemcell-(.*).tgz 68 | access_key_id: {{bbl_compiled_bosh_release_s3_access_key_id}} 69 | secret_access_key: {{bbl_compiled_bosh_release_s3_secret_access_key}} 70 | - name: bosh-release 71 | type: bosh-io-release 72 | source: 73 | repository: cloudfoundry/bosh 74 | - name: bosh-aws-cpi-release 75 | type: bosh-io-release 76 | source: 77 | repository: cloudfoundry-incubator/bosh-aws-cpi-release 78 | - name: bosh-google-cpi-release 79 | type: bosh-io-release 80 | source: 81 | repository: cloudfoundry-incubator/bosh-google-cpi-release 82 | - name: aws-stemcell 83 | type: bosh-io-stemcell 84 | source: 85 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 86 | - name: gcp-stemcell 87 | type: bosh-io-stemcell 88 | source: 89 | name: bosh-google-kvm-ubuntu-trusty-go_agent 90 | 91 | jobs: 92 | - name: test-bosh-bootloader 93 | public: true 94 | plan: 95 | - aggregate: 96 | - get: mega-ci 97 | - get: bosh-bootloader 98 | resource: bosh-bootloader-develop 99 | trigger: true 100 | - task: test 101 | file: mega-ci/scripts/ci/bosh-bootloader/test.yml 102 | 103 | - name: aws-integration-tests 104 | public: true 105 | plan: 106 | - aggregate: 107 | - get: bosh-bootloader 108 | resource: bosh-bootloader-develop 109 | passed: [test-bosh-bootloader] 110 | trigger: true 111 | - get: mega-ci 112 | - task: test 113 | file: mega-ci/scripts/ci/bosh-bootloader/aws_integration.yml 114 | config: 115 | params: 116 | AWS_ACCESS_KEY_ID: {{aws_access_key_id}} 117 | AWS_SECRET_ACCESS_KEY: {{aws_secret_access_key}} 118 | AWS_REGION: {{aws_region}} 119 | on_failure: 120 | put: bbl-integration-s3 121 | params: 122 | file: bbl-integration-s3/*.tgz 123 | 124 | - name: gcp-integration-tests 125 | public: true 126 | plan: 127 | - aggregate: 128 | - get: bosh-bootloader 129 | resource: bosh-bootloader-develop 130 | passed: [test-bosh-bootloader] 131 | trigger: true 132 | - get: mega-ci 133 | - task: test 134 | file: mega-ci/scripts/ci/bosh-bootloader/gcp_integration.yml 135 | config: 136 | params: 137 | GCP_SERVICE_ACCOUNT_KEY: {{gcp_service_account_key}} 138 | GCP_PROJECT_ID: {{gcp_project_id}} 139 | GCP_REGION: {{gcp_region}} 140 | GCP_ZONE: {{gcp_zone}} 141 | on_failure: 142 | put: bbl-integration-s3 143 | params: 144 | file: bbl-integration-s3/*.tgz 145 | 146 | - name: concourse-integration-tests 147 | public: true 148 | plan: 149 | - aggregate: 150 | - get: mega-ci 151 | - get: bosh-bootloader 152 | resource: bosh-bootloader-develop 153 | passed: [aws-integration-tests, gcp-integration-tests] 154 | trigger: true 155 | - task: test 156 | file: mega-ci/scripts/ci/bosh-bootloader/concourse_integration.yml 157 | config: 158 | params: 159 | AWS_ACCESS_KEY_ID: {{aws_access_key_id}} 160 | AWS_SECRET_ACCESS_KEY: {{aws_secret_access_key}} 161 | AWS_REGION: {{aws_region}} 162 | GCP_SERVICE_ACCOUNT_KEY: {{gcp_service_account_key}} 163 | GCP_PROJECT_ID: {{gcp_project_id}} 164 | GCP_REGION: {{gcp_region}} 165 | GCP_ZONE: {{gcp_zone}} 166 | on_failure: 167 | put: bbl-integration-s3 168 | params: 169 | file: bbl-integration-s3/*.tgz 170 | 171 | - name: build-compiled-bosh-release 172 | public: true 173 | plan: 174 | - aggregate: 175 | - get: mega-ci 176 | - get: bosh-release 177 | trigger: true 178 | - get: stemcell 179 | resource: aws-stemcell 180 | trigger: true 181 | - task: build-compiled-bosh-release 182 | file: mega-ci/scripts/ci/compile-bosh-release/task.yml 183 | config: 184 | params: 185 | BOSH_DIRECTOR: {{mega_bosh_director}} 186 | BOSH_USER: {{mega_bosh_user}} 187 | BOSH_PASSWORD: {{mega_bosh_password}} 188 | RELEASE_NAME: bosh 189 | - put: bbl-compiled-bosh-release-s3 190 | params: 191 | file: compiled-bosh-release/*.tgz 192 | acl: public-read 193 | 194 | - name: update-release-constants 195 | public: true 196 | plan: 197 | - aggregate: 198 | - get: mega-ci 199 | - get: bosh-bootloader 200 | resource: bosh-bootloader-develop 201 | - get: aws-stemcell 202 | passed: [build-compiled-bosh-release] 203 | - get: gcp-stemcell 204 | trigger: true 205 | - get: bbl-compiled-bosh-release-s3 206 | trigger: true 207 | passed: [build-compiled-bosh-release] 208 | - get: bosh-aws-cpi-release 209 | trigger: true 210 | - get: bosh-google-cpi-release 211 | trigger: true 212 | - get: bosh-release 213 | - task: update-release-constants 214 | file: mega-ci/scripts/ci/bosh-bootloader/update-release-constants/task.yml 215 | - put: bosh-bootloader-develop-write 216 | params: 217 | repository: bosh-bootloader-develop-write 218 | rebase: true 219 | 220 | - name: major 221 | public: true 222 | plan: 223 | - get: version 224 | - put: version 225 | params: {bump: major} 226 | 227 | - name: minor 228 | public: true 229 | plan: 230 | - get: version 231 | - put: version 232 | params: {bump: minor} 233 | 234 | - name: patch 235 | public: true 236 | plan: 237 | - get: version 238 | - put: version 239 | params: {bump: patch} 240 | 241 | - name: github-release 242 | public: true 243 | plan: 244 | - aggregate: 245 | - get: mega-ci 246 | - get: bbl-version 247 | trigger: true 248 | resource: version 249 | - get: bosh-bootloader 250 | resource: bosh-bootloader-master 251 | - task: build-binaries 252 | file: mega-ci/scripts/ci/bosh-bootloader/build-final-release/task.yml 253 | - put: bbl-release 254 | params: 255 | name: builds/name 256 | tag: builds/name 257 | commitish: builds/commitish 258 | globs: 259 | - builds/bin/bbl-* 260 | -------------------------------------------------------------------------------- /pipelines/bosh-test.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: bosh-test 3 | jobs: 4 | - test-bosh-test 5 | 6 | resources: 7 | - name: bosh-test 8 | type: git 9 | source: 10 | branch: master 11 | uri: https://github.com/pivotal-cf-experimental/bosh-test.git 12 | - name: mega-ci 13 | type: git 14 | source: 15 | branch: master 16 | uri: https://github.com/cloudfoundry/mega-ci.git 17 | 18 | jobs: 19 | - name: test-bosh-test 20 | public: true 21 | plan: 22 | - aggregate: 23 | - get: mega-ci 24 | - get: bosh-test 25 | trigger: true 26 | - task: test 27 | file: mega-ci/scripts/ci/bosh-test/test.yml 28 | -------------------------------------------------------------------------------- /pipelines/check-a-record.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: check-a-record 3 | jobs: 4 | - test-check-a-record 5 | 6 | resources: 7 | - name: check-a-record 8 | type: git 9 | source: 10 | branch: master 11 | uri: https://github.com/cloudfoundry-incubator/check-a-record.git 12 | - name: mega-ci 13 | type: git 14 | source: 15 | branch: master 16 | uri: https://github.com/cloudfoundry/mega-ci.git 17 | 18 | jobs: 19 | - name: test-check-a-record 20 | public: true 21 | plan: 22 | - aggregate: 23 | - get: mega-ci 24 | - get: check-a-record 25 | trigger: true 26 | - task: test 27 | file: mega-ci/scripts/ci/check-a-record/test.yml 28 | 29 | -------------------------------------------------------------------------------- /pipelines/gomegamatchers.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: gomegamatchers 3 | jobs: 4 | - test-gomegamatchers 5 | 6 | resources: 7 | - name: gomegamatchers 8 | type: git 9 | source: 10 | branch: master 11 | uri: https://github.com/pivotal-cf-experimental/gomegamatchers.git 12 | - name: mega-ci 13 | type: git 14 | source: 15 | branch: master 16 | uri: https://github.com/cloudfoundry/mega-ci.git 17 | 18 | jobs: 19 | - name: test-gomegamatchers 20 | public: true 21 | plan: 22 | - aggregate: 23 | - get: mega-ci 24 | - get: gomegamatchers 25 | trigger: true 26 | - task: test 27 | file: mega-ci/scripts/ci/gomegamatchers/test.yml 28 | -------------------------------------------------------------------------------- /pipelines/mega-ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | groups: 3 | - name: mega-ci 4 | jobs: 5 | - minimal-docker-image 6 | - golang-docker-image 7 | - deployment-docker-image 8 | - vagrant-docker-image 9 | - mega-ci-unit-tests 10 | 11 | resources: 12 | - name: mega-ci-master 13 | type: git 14 | source: 15 | uri: https://github.com/cloudfoundry/mega-ci 16 | branch: master 17 | 18 | - name: deployment-dockerfile 19 | type: git 20 | source: 21 | uri: https://github.com/cloudfoundry/mega-ci 22 | branch: master 23 | paths: 24 | - dockerfiles/deployment 25 | 26 | - name: golang-dockerfile 27 | type: git 28 | source: 29 | uri: https://github.com/cloudfoundry/mega-ci 30 | branch: master 31 | paths: 32 | - dockerfiles/golang 33 | 34 | - name: vagrant-dockerfile 35 | type: git 36 | source: 37 | uri: https://github.com/cloudfoundry/mega-ci 38 | branch: master 39 | paths: 40 | - dockerfiles/vagrant 41 | 42 | - name: minimal-dockerfile 43 | type: git 44 | source: 45 | uri: https://github.com/cloudfoundry/mega-ci 46 | branch: master 47 | paths: 48 | - dockerfiles/minimal 49 | 50 | - name: ubuntu-docker-image 51 | type: docker-image 52 | source: 53 | repository: ubuntu 54 | tag: "14.04" 55 | 56 | - name: minimal-docker-image 57 | type: docker-image 58 | source: 59 | repository: cfinfrastructure/minimal 60 | username: {{docker-username}} 61 | password: {{docker-password}} 62 | email: {{docker-email}} 63 | 64 | - name: golang-docker-image 65 | type: docker-image 66 | source: 67 | repository: cfinfrastructure/golang 68 | username: {{docker-username}} 69 | password: {{docker-password}} 70 | email: {{docker-email}} 71 | 72 | - name: deployment-docker-image 73 | type: docker-image 74 | source: 75 | repository: cfinfrastructure/deployment 76 | username: {{docker-username}} 77 | password: {{docker-password}} 78 | email: {{docker-email}} 79 | 80 | - name: vagrant-docker-image 81 | type: docker-image 82 | source: 83 | repository: cfinfrastructure/vagrant 84 | username: {{docker-username}} 85 | password: {{docker-password}} 86 | email: {{docker-email}} 87 | 88 | jobs: 89 | - name: minimal-docker-image 90 | public: true 91 | plan: 92 | - do: 93 | - aggregate: 94 | - get: ubuntu-docker-image 95 | trigger: true 96 | params: 97 | skip_download: true 98 | - get: mega-ci 99 | resource: minimal-dockerfile 100 | trigger: true 101 | - put: minimal-docker-image 102 | params: 103 | build: mega-ci/dockerfiles/minimal 104 | cache: false 105 | 106 | - name: golang-docker-image 107 | public: true 108 | plan: 109 | - do: 110 | - aggregate: 111 | - get: minimal-docker-image 112 | passed: [minimal-docker-image] 113 | trigger: true 114 | params: 115 | skip_download: true 116 | - get: mega-ci 117 | resource: golang-dockerfile 118 | trigger: true 119 | - put: golang-docker-image 120 | params: 121 | build: mega-ci/dockerfiles/golang 122 | cache: false 123 | 124 | - name: deployment-docker-image 125 | public: true 126 | plan: 127 | - do: 128 | - aggregate: 129 | - get: golang-docker-image 130 | passed: [golang-docker-image] 131 | trigger: true 132 | params: 133 | skip_download: true 134 | - get: mega-ci 135 | resource: deployment-dockerfile 136 | trigger: true 137 | - put: deployment-docker-image 138 | params: 139 | build: mega-ci/dockerfiles/deployment 140 | cache: false 141 | 142 | - name: vagrant-docker-image 143 | public: true 144 | plan: 145 | - do: 146 | - aggregate: 147 | - get: deployment-docker-image 148 | passed: [deployment-docker-image] 149 | trigger: true 150 | params: 151 | skip_download: true 152 | - get: mega-ci 153 | resource: vagrant-dockerfile 154 | trigger: true 155 | - put: vagrant-docker-image 156 | params: 157 | build: mega-ci/dockerfiles/vagrant 158 | cache: false 159 | 160 | - name: mega-ci-unit-tests 161 | public: true 162 | plan: 163 | - do: 164 | - get: mega-ci 165 | resource: mega-ci-master 166 | trigger: true 167 | - task: mega-ci-tests 168 | file: mega-ci/scripts/ci/test.yml 169 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/aws_integration: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | CURRENT_TIME=$(date +%s) 5 | export STATE_DIR="${ROOT}/bbl-integration-s3/${CURRENT_TIME}" 6 | 7 | function install_bosh_cli() { 8 | pushd "${ROOT}" > /dev/null 9 | wget https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-0.0.127-linux-amd64 10 | mv bosh-cli-0.0.127-linux-amd64 /usr/local/bin/bosh 11 | chmod +x /usr/local/bin/bosh 12 | popd > /dev/null 13 | } 14 | 15 | function main() { 16 | install_bosh_cli 17 | 18 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry" 19 | mkdir -p "${STATE_DIR}" 20 | 21 | pushd "${GOPATH}/src/github.com/cloudfoundry" > /dev/null 22 | ln -s "${ROOT}/bosh-bootloader" 23 | 24 | ./bosh-bootloader/scripts/aws_integration_tests 25 | popd > /dev/null 26 | } 27 | 28 | function finish() { 29 | pushd "${ROOT}/bbl-integration-s3" > /dev/null 30 | tar -cvzf "${CURRENT_TIME}.tgz" "${CURRENT_TIME}" 31 | popd > /dev/null 32 | } 33 | trap finish EXIT 34 | 35 | main 36 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/aws_integration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: bosh-bootloader 8 | - name: mega-ci 9 | outputs: 10 | - name: bbl-integration-s3 11 | 12 | run: 13 | path: mega-ci/scripts/ci/bosh-bootloader/aws_integration 14 | 15 | params: 16 | AWS_REGION: 17 | AWS_ACCESS_KEY_ID: 18 | AWS_SECRET_ACCESS_KEY: 19 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/build-final-release/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT=${PWD} 4 | BBL_VERSION=$(cat "${ROOT}/bbl-version/number") 5 | 6 | function main() { 7 | local commit_sha 8 | 9 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry/" 10 | ln -s "${ROOT}/bosh-bootloader" "${GOPATH}/src/github.com/cloudfoundry/" 11 | commit_sha=$(git -C "${ROOT}/bosh-bootloader" rev-parse HEAD) 12 | 13 | pushd "${GOPATH}/src/github.com/cloudfoundry/bosh-bootloader/bbl" > /dev/null 14 | build_bbl 15 | popd > /dev/null 16 | 17 | pushd "${ROOT}/builds" > /dev/null 18 | echo -n "v${BBL_VERSION}" > name 19 | echo -n "${commit_sha}" > commitish 20 | popd > /dev/null 21 | } 22 | 23 | function build_bbl() { 24 | echo "Building 64-bit Linux" 25 | CGO_ENABLED=0 GOARCH=amd64 GOOS=linux \ 26 | go build -a -tags netgo -installsuffix netgo \ 27 | -ldflags "-X main.Version=${BBL_VERSION}" \ 28 | -o "${ROOT}/builds/bin/bbl-v${BBL_VERSION}_linux_x86-64" . 29 | 30 | echo "Building 64-bit Darwin" 31 | GOARCH=amd64 GOOS=darwin \ 32 | go build \ 33 | -ldflags "-X main.Version=${BBL_VERSION}" \ 34 | -o "${ROOT}/builds/bin/bbl-v${BBL_VERSION}_osx" . 35 | } 36 | 37 | main 38 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/build-final-release/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: bosh-bootloader 8 | - name: bbl-version 9 | - name: mega-ci 10 | 11 | outputs: 12 | - name: builds 13 | 14 | run: 15 | path: mega-ci/scripts/ci/bosh-bootloader/build-final-release/task 16 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/concourse_integration: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | CURRENT_TIME=$(date +%s) 5 | export STATE_DIR="${ROOT}/bbl-integration-s3/${CURRENT_TIME}" 6 | 7 | function main() { 8 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry" 9 | 10 | mkdir -p "${STATE_DIR}" 11 | 12 | set +x 13 | echo "${GCP_SERVICE_ACCOUNT_KEY}" > service_account.key.json 14 | local service_account_key_path 15 | service_account_key_path=$PWD/service_account.key.json 16 | set -x 17 | 18 | pushd "${GOPATH}/src/github.com/cloudfoundry" > /dev/null 19 | ln -s "${ROOT}/bosh-bootloader" 20 | 21 | export GCP_SERVICE_ACCOUNT_KEY=$service_account_key_path 22 | ./bosh-bootloader/scripts/concourse_integration_tests 23 | popd > /dev/null 24 | } 25 | 26 | function finish() { 27 | pushd "${ROOT}/bbl-integration-s3" > /dev/null 28 | tar -cvzf "${CURRENT_TIME}.tgz" "${CURRENT_TIME}" 29 | popd > /dev/null 30 | } 31 | trap finish EXIT 32 | 33 | main 34 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/concourse_integration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: bosh-bootloader 8 | - name: mega-ci 9 | outputs: 10 | - name: bbl-integration-s3 11 | 12 | run: 13 | path: mega-ci/scripts/ci/bosh-bootloader/concourse_integration 14 | 15 | params: 16 | AWS_REGION: 17 | AWS_ACCESS_KEY_ID: 18 | AWS_SECRET_ACCESS_KEY: 19 | GCP_SERVICE_ACCOUNT_KEY: 20 | GCP_PROJECT_ID: 21 | GCP_REGION: 22 | GCP_ZONE: 23 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/gcp_integration: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | CURRENT_TIME=$(date +%s) 5 | export STATE_DIR="${ROOT}/bbl-integration-s3/${CURRENT_TIME}" 6 | 7 | function install_bosh_cli() { 8 | pushd "${ROOT}" > /dev/null 9 | wget https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-0.0.127-linux-amd64 10 | mv bosh-cli-0.0.127-linux-amd64 /usr/local/bin/bosh 11 | chmod +x /usr/local/bin/bosh 12 | popd > /dev/null 13 | } 14 | 15 | function main() { 16 | install_bosh_cli 17 | 18 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry" 19 | mkdir -p "${STATE_DIR}" 20 | 21 | set +x 22 | echo "${GCP_SERVICE_ACCOUNT_KEY}" > service_account.key.json 23 | local service_account_key_path 24 | service_account_key_path=$PWD/service_account.key.json 25 | set -x 26 | 27 | pushd "${GOPATH}/src/github.com/cloudfoundry" > /dev/null 28 | ln -s "${ROOT}/bosh-bootloader" 29 | 30 | export GCP_SERVICE_ACCOUNT_KEY="${service_account_key_path}" 31 | ./bosh-bootloader/scripts/gcp_integration_tests 32 | popd > /dev/null 33 | } 34 | 35 | function finish() { 36 | pushd "${ROOT}/bbl-integration-s3" > /dev/null 37 | tar -cvzf "${CURRENT_TIME}.tgz" "${CURRENT_TIME}" 38 | popd > /dev/null 39 | } 40 | trap finish EXIT 41 | 42 | main 43 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/gcp_integration.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: bosh-bootloader 8 | - name: mega-ci 9 | outputs: 10 | - name: bbl-integration-s3 11 | 12 | run: 13 | path: mega-ci/scripts/ci/bosh-bootloader/gcp_integration 14 | 15 | params: 16 | GCP_SERVICE_ACCOUNT_KEY: 17 | GCP_PROJECT_ID: 18 | GCP_REGION: 19 | GCP_ZONE: 20 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry" 6 | 7 | pushd "${GOPATH}/src/github.com/cloudfoundry" > /dev/null 8 | ln -s "${ROOT}/bosh-bootloader" 9 | chpst -u testuser:testuser ./bosh-bootloader/scripts/test 10 | popd > /dev/null 11 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image_resource: 5 | type: docker-image 6 | source: 7 | repository: cfinfrastructure/golang 8 | 9 | inputs: 10 | - name: bosh-bootloader 11 | - name: mega-ci 12 | 13 | run: 14 | path: mega-ci/scripts/ci/bosh-bootloader/test 15 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/update-release-constants/task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT=${PWD} 4 | 5 | function main() { 6 | pushd "${ROOT}/bosh-google-cpi-release" > /dev/null 7 | local bosh_google_cpi_url 8 | local bosh_google_cpi_sha1 9 | 10 | bosh_google_cpi_url=$(cat url) 11 | bosh_google_cpi_sha1="$(sha1sum release.tgz | cut -f1 -d" ")" 12 | popd > /dev/null 13 | 14 | pushd "${ROOT}/bosh-aws-cpi-release" > /dev/null 15 | local bosh_aws_cpi_url 16 | local bosh_aws_cpi_sha1 17 | 18 | bosh_aws_cpi_url=$(cat url) 19 | bosh_aws_cpi_sha1="$(sha1sum release.tgz | cut -f1 -d" ")" 20 | popd > /dev/null 21 | 22 | pushd "${ROOT}/gcp-stemcell" > /dev/null 23 | local gcp_stemcell_url 24 | local gcp_stemcell_sha1 25 | 26 | gcp_stemcell_url="$(cat url)" 27 | gcp_stemcell_sha1="$(sha1sum stemcell.tgz | cut -f1 -d" ")" 28 | popd > /dev/null 29 | 30 | pushd "${ROOT}/aws-stemcell" > /dev/null 31 | local aws_stemcell_url 32 | local aws_stemcell_sha1 33 | 34 | aws_stemcell_url="$(cat url)" 35 | aws_stemcell_sha1="$(sha1sum stemcell.tgz | cut -f1 -d" ")" 36 | popd > /dev/null 37 | 38 | pushd "${ROOT}/bbl-compiled-bosh-release-s3" > /dev/null 39 | local compiled_bosh_url 40 | local compiled_bosh_sha1 41 | local compiled_release_name 42 | 43 | compiled_bosh_url="$(cat url)" 44 | compiled_release_name=$(cat url | cut -f 5 -d"/") 45 | compiled_bosh_sha1="$(sha1sum ${compiled_release_name} | cut -f1 -d" ")" 46 | popd > /dev/null 47 | 48 | pushd "${ROOT}/bosh-bootloader/bbl/constants" > /dev/null 49 | cat > versions.go << EOF 50 | package constants 51 | 52 | // THIS FILE IS GENERATED AUTOMATICALLY, NO TOUCHING!!!!! 53 | 54 | const ( 55 | BOSHURL = "${compiled_bosh_url}" 56 | BOSHSHA1 = "${compiled_bosh_sha1}" 57 | BOSHAWSCPIURL = "${bosh_aws_cpi_url}" 58 | BOSHAWSCPISHA1 = "${bosh_aws_cpi_sha1}" 59 | AWSStemcellURL = "${aws_stemcell_url}" 60 | AWSStemcellSHA1 = "${aws_stemcell_sha1}" 61 | BOSHGCPCPIURL = "${bosh_google_cpi_url}" 62 | BOSHGCPCPISHA1 = "${bosh_google_cpi_sha1}" 63 | GCPStemcellURL = "${gcp_stemcell_url}" 64 | GCPStemcellSHA1 = "${gcp_stemcell_sha1}" 65 | ) 66 | EOF 67 | go fmt versions.go 68 | 69 | git config --global user.name "cf-infra-bot" 70 | git config --global user.email cf-infrastructure@pivotal.io 71 | 72 | git add versions.go 73 | git commit -m "Update constants" 74 | popd > /dev/null 75 | 76 | git clone file://${ROOT}/bosh-bootloader ${ROOT}/bosh-bootloader-develop-write 77 | } 78 | 79 | main 80 | -------------------------------------------------------------------------------- /scripts/ci/bosh-bootloader/update-release-constants/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/golang 5 | 6 | inputs: 7 | - name: bosh-bootloader 8 | - name: mega-ci 9 | - name: bosh-release 10 | - name: bosh-aws-cpi-release 11 | - name: bosh-google-cpi-release 12 | - name: bbl-compiled-bosh-release-s3 13 | - name: aws-stemcell 14 | - name: gcp-stemcell 15 | 16 | outputs: 17 | - name: bosh-bootloader-develop-write 18 | 19 | run: 20 | path: mega-ci/scripts/ci/bosh-bootloader/update-release-constants/task.sh 21 | -------------------------------------------------------------------------------- /scripts/ci/bosh-test/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | mkdir -p "${GOPATH}/src/github.com/pivotal-cf-experimental" 6 | 7 | pushd "${GOPATH}/src/github.com/pivotal-cf-experimental" > /dev/null 8 | ln -s "${ROOT}/bosh-test" 9 | ./bosh-test/scripts/test 10 | popd > /dev/null 11 | -------------------------------------------------------------------------------- /scripts/ci/bosh-test/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/golang 5 | 6 | inputs: 7 | - name: bosh-test 8 | - name: mega-ci 9 | 10 | run: 11 | path: mega-ci/scripts/ci/bosh-test/test 12 | -------------------------------------------------------------------------------- /scripts/ci/check-a-record/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | mkdir -p "${GOPATH}/src/github.com/pivotal-cf-experimental" 6 | 7 | pushd "${GOPATH}/src/github.com/pivotal-cf-experimental" > /dev/null 8 | ln -s "${ROOT}/check-a-record" 9 | ./check-a-record/scripts/ci/test 10 | popd > /dev/null 11 | -------------------------------------------------------------------------------- /scripts/ci/check-a-record/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/golang 5 | 6 | inputs: 7 | - name: check-a-record 8 | - name: mega-ci 9 | 10 | run: 11 | path: mega-ci/scripts/ci/check-a-record/test 12 | -------------------------------------------------------------------------------- /scripts/ci/check-git-submodules/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | function main() { 4 | local root_dir 5 | root_dir="${1}" 6 | 7 | local protocol 8 | protocol="${2}" 9 | 10 | local total 11 | total="$(count_with_protocol "${root_dir}/repo" "")" 12 | 13 | local with_protocol 14 | with_protocol="$(count_with_protocol "${root_dir}/repo" "${protocol}")" 15 | 16 | if [ "${total}" != "${with_protocol}" ]; then 17 | echo "found git submodules not using ${protocol}" 18 | grep "url = " "${root_dir}/repo/.gitmodules" | grep -v "url = ${protocol}://" 19 | exit 1 20 | fi 21 | } 22 | 23 | function count_with_protocol() { 24 | local repo 25 | repo="${1}" 26 | 27 | local protocol 28 | protocol="${2}" 29 | 30 | if [[ -n "${protocol}" ]]; then 31 | protocol="${protocol}://" 32 | fi 33 | 34 | printf "%s" "$(grep "url = ${protocol}" "${repo}/.gitmodules" | wc -l)" 35 | } 36 | 37 | main "${PWD}" "${PROTOCOL}" 38 | -------------------------------------------------------------------------------- /scripts/ci/check-git-submodules/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/minimal 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: repo 9 | 10 | run: 11 | path: mega-ci/scripts/ci/check-git-submodules/task 12 | 13 | params: 14 | PROTOCOL: https 15 | -------------------------------------------------------------------------------- /scripts/ci/compile-bosh-release/fixtures/compilation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | director_uuid: REPLACE_ME_DIRECTOR_UUID 3 | 4 | name: compilation_REPLACE_ME_RELEASE_NAME 5 | 6 | releases: 7 | - name: REPLACE_ME_RELEASE_NAME 8 | version: REPLACE_ME_RELEASE_VERSION 9 | 10 | update: 11 | canaries: 1 12 | max_in_flight: 1 13 | serial: false 14 | canary_watch_time: 1000-60000 15 | update_watch_time: 1000-60000 16 | 17 | stemcells: 18 | - alias: trusty 19 | os: ubuntu-trusty 20 | version: latest 21 | 22 | instance_groups: [] 23 | -------------------------------------------------------------------------------- /scripts/ci/compile-bosh-release/task-release-from-repo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | export ROOT="${PWD}" 4 | export STEMCELL_VERSION=$(cat "${ROOT}"/stemcell/version) 5 | export RELEASE_VERSION="99999+dev.$(date +%s)" 6 | 7 | function main() { 8 | upload_stemcell 9 | upload_release 10 | force_compilation 11 | } 12 | 13 | function force_compilation() { 14 | pushd /tmp > /dev/null 15 | sed \ 16 | -e "s/REPLACE_ME_DIRECTOR_UUID/$(/opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" status --uuid)/g" \ 17 | -e "s/REPLACE_ME_RELEASE_VERSION/${RELEASE_VERSION}/g" \ 18 | -e "s/REPLACE_ME_RELEASE_NAME/${RELEASE_NAME}/g" \ 19 | "${ROOT}/mega-ci/scripts/ci/compile-bosh-release/fixtures/compilation.yml" > "compilation.yml" 20 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -d "/tmp/compilation.yml" -n deploy 21 | popd > /dev/null 22 | 23 | pushd "${ROOT}/compiled-bosh-release" > /dev/null 24 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -d "/tmp/compilation.yml" export release "${RELEASE_NAME}/${RELEASE_VERSION}" "ubuntu-trusty/${STEMCELL_VERSION}" 25 | popd > /dev/null 26 | 27 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n delete deployment compilation 28 | } 29 | 30 | function upload_stemcell() { 31 | pushd "${ROOT}/stemcell" > /dev/null 32 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload stemcell stemcell.tgz --skip-if-exists 33 | popd > /dev/null 34 | } 35 | 36 | function upload_release() { 37 | pushd "${ROOT}/release-repo" > /dev/null 38 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n create release --name "${RELEASE_NAME}" --force --version "${RELEASE_VERSION}" 39 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload release 40 | popd > /dev/null 41 | } 42 | 43 | function cleanup_releases() { 44 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n cleanup 45 | } 46 | 47 | function rollup() { 48 | set +x 49 | local input 50 | input="${1}" 51 | 52 | local output 53 | 54 | IFS=$'\n' 55 | for line in ${input}; do 56 | output="${output:-""}\n${line}" 57 | done 58 | 59 | printf "%s" "${output#'\n'}" 60 | set -x 61 | } 62 | 63 | trap cleanup_releases EXIT 64 | main 65 | -------------------------------------------------------------------------------- /scripts/ci/compile-bosh-release/task-release-from-repo.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: release-repo 9 | - name: stemcell 10 | 11 | run: 12 | path: mega-ci/scripts/ci/compile-bosh-release/task-release-from-repo.sh 13 | 14 | params: 15 | BOSH_DIRECTOR: 16 | BOSH_USER: 17 | BOSH_PASSWORD: 18 | BOSH_DIRECTOR_CA_CERT: 19 | RELEASE_NAME: 20 | 21 | outputs: 22 | - name: compiled-bosh-release 23 | -------------------------------------------------------------------------------- /scripts/ci/compile-bosh-release/task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | export ROOT="${PWD}" 4 | export STEMCELL_VERSION=$(cat "${ROOT}"/stemcell/version) 5 | export RELEASE_VERSION=$(cat "${ROOT}"/bosh-release/version) 6 | 7 | function main() { 8 | upload_stemcell 9 | upload_release 10 | force_compilation 11 | check_existing_release_in_bucket 12 | } 13 | 14 | function force_compilation() { 15 | pushd /tmp > /dev/null 16 | sed \ 17 | -e "s/REPLACE_ME_DIRECTOR_UUID/$(/opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" status --uuid)/g" \ 18 | -e "s/REPLACE_ME_RELEASE_VERSION/${RELEASE_VERSION}/g" \ 19 | -e "s/REPLACE_ME_RELEASE_NAME/${RELEASE_NAME}/g" \ 20 | "${ROOT}/mega-ci/scripts/ci/compile-bosh-release/fixtures/compilation.yml" > "compilation.yml" 21 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -d "/tmp/compilation.yml" -n deploy 22 | popd > /dev/null 23 | 24 | pushd "${ROOT}/compiled-bosh-release" > /dev/null 25 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -d "/tmp/compilation.yml" export release "${RELEASE_NAME}/${RELEASE_VERSION}" "ubuntu-trusty/${STEMCELL_VERSION}" 26 | popd > /dev/null 27 | 28 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n delete deployment "compilation_${RELEASE_NAME}" 29 | } 30 | 31 | function check_existing_release_in_bucket() { 32 | pushd "${ROOT}/compiled-bosh-release" > /dev/null 33 | local filename 34 | filename=$(ls | grep tgz) 35 | 36 | set +e 37 | curl -f https://s3.amazonaws.com/bbl-precompiled-bosh-releases/${filename} > /dev/null 38 | if [ $? -eq 0 ]; then 39 | echo "${filename} already exists on s3. Quitting." 40 | exit 1 41 | fi 42 | set -e 43 | popd > /dev/null 44 | } 45 | 46 | function upload_stemcell() { 47 | pushd "${ROOT}/stemcell" > /dev/null 48 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload stemcell stemcell.tgz --skip-if-exists 49 | popd > /dev/null 50 | } 51 | 52 | function upload_release() { 53 | pushd "${ROOT}/bosh-release" > /dev/null 54 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload release release.tgz --skip-if-exists 55 | popd > /dev/null 56 | } 57 | 58 | function cleanup_releases() { 59 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n cleanup 60 | } 61 | 62 | function rollup() { 63 | set +x 64 | local input 65 | input="${1}" 66 | 67 | local output 68 | 69 | IFS=$'\n' 70 | for line in ${input}; do 71 | output="${output:-""}\n${line}" 72 | done 73 | 74 | printf "%s" "${output#'\n'}" 75 | set -x 76 | } 77 | 78 | trap cleanup_releases EXIT 79 | main 80 | -------------------------------------------------------------------------------- /scripts/ci/compile-bosh-release/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: bosh-release 9 | - name: stemcell 10 | 11 | run: 12 | path: mega-ci/scripts/ci/compile-bosh-release/task.sh 13 | 14 | params: 15 | BOSH_DIRECTOR: 16 | BOSH_USER: 17 | BOSH_PASSWORD: 18 | BOSH_DIRECTOR_CA_CERT: 19 | RELEASE_NAME: 20 | 21 | outputs: 22 | - name: compiled-bosh-release 23 | -------------------------------------------------------------------------------- /scripts/ci/confab/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry-incubator" 6 | 7 | pushd "${GOPATH}/src/github.com/cloudfoundry-incubator" > /dev/null 8 | ln -s "${ROOT}/consul-release" 9 | chpst -u testuser:testuser ./consul-release/src/confab/scripts/test 10 | popd > /dev/null 11 | -------------------------------------------------------------------------------- /scripts/ci/confab/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/golang 5 | 6 | inputs: 7 | - name: consul-release 8 | - name: mega-ci 9 | 10 | run: 11 | path: mega-ci/scripts/ci/confab/test 12 | -------------------------------------------------------------------------------- /scripts/ci/create-final-release/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | # Cannot set -u before sourcing .bashrc because of all 4 | # the unbound variables in things beyond our control. 5 | set +u 6 | source ~/.bashrc 7 | set -u 8 | 9 | function main() { 10 | local root_dir 11 | root_dir="${PWD}" 12 | 13 | local release_name 14 | release_name="${1}" 15 | 16 | local master_branch 17 | master_branch="${2}" 18 | 19 | configure_bucket "${release_name}" 20 | create_and_commit "${root_dir}" "${release_name}" "${master_branch}" 21 | copy_to_output "${root_dir}" 22 | } 23 | 24 | function configure_bucket() { 25 | local release_name 26 | release_name="${1}" 27 | 28 | set +x 29 | ./mega-ci/scripts/configure_final_release_bucket "${release_name}" ./oss-s3-buckets-stack ./release-repo/config 30 | set -x 31 | } 32 | 33 | function create_and_commit() { 34 | local root_dir 35 | root_dir="${1}" 36 | 37 | local release_name 38 | release_name="${2}" 39 | 40 | local master_branch 41 | master_branch="${3}" 42 | 43 | pushd "${root_dir}/release-repo" > /dev/null 44 | git config user.name "CF MEGA BOT" 45 | git config user.email "cf-mega@pivotal.io" 46 | 47 | git remote add -f master-repo "${root_dir}/release-repo-master" 48 | git merge "master-repo/${master_branch}" -m 'Merge with master' 49 | 50 | local exit_status 51 | for i in {1..5}; do 52 | /opt/rubies/ruby-2.2.4/bin/bosh -n create release --with-tarball --final 53 | exit_status="${PIPESTATUS[0]}" 54 | 55 | if [[ "${exit_status}" == "0" ]]; then 56 | break 57 | fi 58 | done 59 | 60 | if [[ "${exit_status}" != "0" ]]; then 61 | echo "Failed to Create ${release_name} Release" 62 | exit "${exit_status}" 63 | fi 64 | 65 | local new_release_version 66 | new_release_version="$(find releases -regex ".*${release_name}-[0-9]*.yml" | egrep -o "${release_name}-[0-9]+" | egrep -o "[0-9]+" | sort -n | tail -n 1)" 67 | 68 | git add .final_builds releases 69 | git commit -m "Final release ${new_release_version}" 70 | 71 | echo "${new_release_version}" > version_number 72 | popd > /dev/null 73 | } 74 | 75 | function copy_to_output() { 76 | local root_dir 77 | root_dir="${1}" 78 | 79 | shopt -s dotglob 80 | cp -R "${root_dir}/release-repo/"* "${root_dir}/final-release-repo" 81 | } 82 | 83 | main "${RELEASE_NAME}" "${MASTER_BRANCH:-"master"}" 84 | -------------------------------------------------------------------------------- /scripts/ci/create-final-release/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: oss-s3-buckets-stack 9 | - name: release-repo 10 | - name: release-repo-master 11 | 12 | outputs: 13 | - name: final-release-repo 14 | 15 | run: 16 | path: mega-ci/scripts/ci/create-final-release/task 17 | 18 | params: 19 | RELEASE_NAME: 20 | MASTER_BRANCH: 21 | -------------------------------------------------------------------------------- /scripts/ci/delete-deployments/task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | function main() { 4 | local raw_deployments 5 | set +x 6 | raw_deployments=$(curl -sk https://${BOSH_USER}:${BOSH_PASSWORD}@${BOSH_DIRECTOR}:25555/deployments) 7 | set -x 8 | 9 | local deployments 10 | deployments=$(echo "${raw_deployments}" | jq 'map(select(.name | contains('\"${DEPLOYMENTS_WITH_WORD}\"')))' | jq .[].name) 11 | 12 | if [ -n "${deployments}" ] 13 | then 14 | echo "${deployments}" | xargs -n 1 -P 5 /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n delete deployment --force 15 | fi 16 | 17 | echo "cleaning up orphaned disks and releases" 18 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" cleanup 19 | } 20 | 21 | main 22 | -------------------------------------------------------------------------------- /scripts/ci/delete-deployments/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | 9 | run: 10 | path: mega-ci/scripts/ci/delete-deployments/task.sh 11 | 12 | params: 13 | BOSH_PASSWORD: 14 | BOSH_DIRECTOR: 15 | BOSH_USER: 16 | DEPLOYMENTS_WITH_WORD: 17 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/awsdeployer/awsdeployer.go: -------------------------------------------------------------------------------- 1 | package awsdeployer 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | 8 | "gopkg.in/yaml.v2" 9 | 10 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 11 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/manifests" 12 | ) 13 | 14 | type SubnetChecker interface { 15 | CheckSubnets(manifestFilename string) (bool, error) 16 | } 17 | 18 | type AWSDeployer struct { 19 | bosh clients.BOSH 20 | subnetChecker SubnetChecker 21 | stdout io.Writer 22 | } 23 | 24 | func NewAWSDeployer(bosh clients.BOSH, subnetChecker SubnetChecker, stdout io.Writer) AWSDeployer { 25 | return AWSDeployer{ 26 | bosh: bosh, 27 | subnetChecker: subnetChecker, 28 | stdout: stdout, 29 | } 30 | } 31 | 32 | func (a AWSDeployer) Deploy(manifestFilename string) error { 33 | fmt.Fprintf(a.stdout, "deploying manifest: %s\n", manifestFilename) 34 | fmt.Fprintln(a.stdout, "checking subnets...") 35 | hasSubnets, err := a.subnetChecker.CheckSubnets(manifestFilename) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | if !hasSubnets { 41 | return errors.New("manifest subnets not found on AWS") 42 | } 43 | 44 | fmt.Fprintln(a.stdout, "found all manifest subnets on AWS") 45 | 46 | err = a.deployManifest(manifestFilename) 47 | if err != nil { 48 | return err 49 | } 50 | fmt.Fprintln(a.stdout, "\n\n") 51 | 52 | return nil 53 | } 54 | 55 | func (a AWSDeployer) deployManifest(manifestFilename string) error { 56 | fmt.Fprintln(a.stdout, "fetching director uuid...") 57 | manifest, err := a.replaceUUID(manifestFilename) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | buf, err := yaml.Marshal(manifest) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | fmt.Fprintln(a.stdout, "deploying...") 68 | err = a.bosh.Deploy(buf) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | fmt.Fprintln(a.stdout, "deleting deployment...") 74 | err = a.deleteDeployment(manifest) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | return nil 80 | } 81 | 82 | func (a AWSDeployer) replaceUUID(manifestFilename string) (map[string]interface{}, error) { 83 | directorUUID, err := a.bosh.UUID() 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | manifest, err := manifests.ReadManifest(manifestFilename) 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | manifest["director_uuid"] = directorUUID 94 | 95 | return manifest, nil 96 | } 97 | 98 | func (a AWSDeployer) deleteDeployment(manifest map[string]interface{}) error { 99 | deploymentName, ok := manifest["name"].(string) 100 | if !ok { 101 | return errors.New("deployment name missing from manifest") 102 | } 103 | 104 | err := a.bosh.DeleteDeployment(deploymentName) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/awsdeployer/awsdeployer_test.go: -------------------------------------------------------------------------------- 1 | package awsdeployer_test 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/awsdeployer" 10 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 11 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/fakes" 12 | "github.com/pivotal-cf-experimental/bosh-test/bosh" 13 | 14 | . "github.com/onsi/ginkgo" 15 | . "github.com/onsi/gomega" 16 | ) 17 | 18 | var _ = Describe("AWSDeployer", func() { 19 | Describe("Deploy", func() { 20 | var ( 21 | fakeBOSH *fakes.BOSH 22 | fakeSubnetChecker *fakes.SubnetChecker 23 | awsDeployer awsdeployer.AWSDeployer 24 | stdout io.Writer 25 | ) 26 | 27 | BeforeEach(func() { 28 | fakeBOSH = &fakes.BOSH{} 29 | fakeSubnetChecker = &fakes.SubnetChecker{} 30 | stdout = ioutil.Discard 31 | 32 | awsDeployer = awsdeployer.NewAWSDeployer(clients.NewBOSH(fakeBOSH, stdout), fakeSubnetChecker, stdout) 33 | fakeSubnetChecker.CheckSubnetsCall.Returns.Bool = true 34 | }) 35 | 36 | It("deploys specified bosh manifest", func() { 37 | manifestFilename := createManifestFile("director_uuid: BOSH-DIRECTOR-UUID\nname: deployment-1") 38 | 39 | deploymentError := awsDeployer.Deploy(manifestFilename) 40 | Expect(deploymentError).NotTo(HaveOccurred()) 41 | 42 | Expect(fakeBOSH.DeployCall.CallCount).To(Equal(1)) 43 | Expect(fakeBOSH.DeployCall.Receives.Manifest).To(ContainSubstring("deployment-1")) 44 | }) 45 | 46 | It("replaces the bosh director uuid before deploying each manifest", func() { 47 | manifestFilename := createBasicManifestFile() 48 | fakeBOSH.InfoCall.Returns.Info = bosh.DirectorInfo{ 49 | UUID: "retrieved-director-uuid", 50 | } 51 | deploymentError := awsDeployer.Deploy(manifestFilename) 52 | 53 | Expect(deploymentError).NotTo(HaveOccurred()) 54 | 55 | Expect(string(fakeBOSH.DeployCall.Receives.Manifest)).To(ContainSubstring("director_uuid: retrieved-director-uuid")) 56 | }) 57 | 58 | It("deletes the deployment", func() { 59 | manifestFilename := createManifestFile("director_uuid: BOSH-DIRECTOR-UUID\nname: some-deployment-name") 60 | 61 | deploymentError := awsDeployer.Deploy(manifestFilename) 62 | 63 | Expect(deploymentError).NotTo(HaveOccurred()) 64 | Expect(fakeBOSH.DeleteDeploymentCall.Receives.Name).To(Equal("some-deployment-name")) 65 | }) 66 | 67 | Context("failure cases", func() { 68 | It("returns an error when the BOSH manifest contains subnets not found on AWS", func() { 69 | const manifestWithSubnetC8 = `--- 70 | director_uuid: BOSH-DIRECTOR-UUID 71 | 72 | name: multi-az-ssl 73 | 74 | networks: 75 | - subnets: 76 | - cloud_properties: 77 | subnet: "subnet-c8b76f90" 78 | range: 10.0.20.0/24 79 | ` 80 | 81 | manifestFilename := createManifestFile(manifestWithSubnetC8) 82 | fakeSubnetChecker.CheckSubnetsCall.Returns.Bool = false 83 | 84 | deploymentError := awsDeployer.Deploy(manifestFilename) 85 | Expect(deploymentError.Error()).To(ContainSubstring("manifest subnets not found on AWS")) 86 | }) 87 | 88 | It("returns an error when CheckSubnets returns an error", func() { 89 | manifestFilename := createBasicManifestFile() 90 | fakeSubnetChecker.CheckSubnetsCall.Returns.Error = errors.New("something bad happened") 91 | 92 | deploymentError := awsDeployer.Deploy(manifestFilename) 93 | Expect(deploymentError.Error()).To(ContainSubstring("something bad happened")) 94 | }) 95 | 96 | It("returns an error when bosh deploy fails", func() { 97 | manifestFilename := createBasicManifestFile() 98 | 99 | fakeBOSH.DeployCall.Returns.Error = errors.New("bosh deployment failed") 100 | 101 | deploymentError := awsDeployer.Deploy(manifestFilename) 102 | Expect(deploymentError.Error()).To(ContainSubstring("bosh deployment failed")) 103 | }) 104 | 105 | It("returns an error when the manifest is not valid yaml", func() { 106 | manifestFilename := createManifestFile("not: valid: yaml:") 107 | deploymentError := awsDeployer.Deploy(manifestFilename) 108 | Expect(deploymentError.Error()).To(ContainSubstring("mapping values are not allowed in this context")) 109 | }) 110 | 111 | It("returns an error when bosh UUID fails", func() { 112 | manifestFilename := createBasicManifestFile() 113 | 114 | fakeBOSH.InfoCall.Returns.Error = errors.New("bosh UUID failed") 115 | 116 | deploymentError := awsDeployer.Deploy(manifestFilename) 117 | Expect(deploymentError.Error()).To(ContainSubstring("bosh UUID failed")) 118 | }) 119 | 120 | It("returns an error when the deployment name is not present in the manifest", func() { 121 | manifestFilename := createManifestFile("director_uuid: BOSH-DIRECTOR-UUID") 122 | deploymentError := awsDeployer.Deploy(manifestFilename) 123 | Expect(deploymentError.Error()).To(ContainSubstring("deployment name missing from manifest")) 124 | }) 125 | 126 | It("returns an error when deletion of the deployment fails", func() { 127 | manifestFilename := createManifestFile("director_uuid: BOSH-DIRECTOR-UUID\nname: some-deployment-name") 128 | fakeBOSH.DeleteDeploymentCall.Returns.Error = errors.New("failed to delete deployment: some-deployment-name") 129 | 130 | deploymentError := awsDeployer.Deploy(manifestFilename) 131 | Expect(deploymentError.Error()).To(ContainSubstring("failed to delete deployment: some-deployment-name")) 132 | }) 133 | }) 134 | }) 135 | }) 136 | 137 | func createBasicManifestFile() string { 138 | return createManifestFile("director_uuid: BOSH-DIRECTOR-UUID\nname: a-deployment-name") 139 | } 140 | 141 | func createManifestFile(body string) string { 142 | manifest, err := ioutil.TempFile("", "") 143 | Expect(err).NotTo(HaveOccurred()) 144 | 145 | err = ioutil.WriteFile(manifest.Name(), []byte(body), os.ModePerm) 146 | Expect(err).NotTo(HaveOccurred()) 147 | 148 | return manifest.Name() 149 | } 150 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/awsdeployer/init_test.go: -------------------------------------------------------------------------------- 1 | package awsdeployer_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestAWSDeployer(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "awsdeployer") 13 | } 14 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/clients/aws.go: -------------------------------------------------------------------------------- 1 | package clients 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/credentials" 8 | "github.com/aws/aws-sdk-go/aws/session" 9 | "github.com/aws/aws-sdk-go/service/ec2" 10 | ) 11 | 12 | type Subnet struct { 13 | SubnetID string 14 | CIDRBlock string 15 | } 16 | 17 | type AWS struct { 18 | AccessKeyID string 19 | SecretAccessKey string 20 | Region string 21 | EndpointOverride string 22 | } 23 | 24 | type Session interface { 25 | DescribeSubnets(*ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) 26 | } 27 | 28 | func NewAWS(AccessKeyID string, SecretAccessKey string, Region string, EndpointOverride string) AWS { 29 | return AWS{ 30 | AccessKeyID: AccessKeyID, 31 | SecretAccessKey: SecretAccessKey, 32 | Region: Region, 33 | EndpointOverride: EndpointOverride, 34 | } 35 | } 36 | 37 | func (a AWS) FetchSubnets(session Session, subnetIds []string) ([]Subnet, error) { 38 | var awsSubnetIds []*string 39 | for _, id := range subnetIds { 40 | awsSubnetIds = append(awsSubnetIds, aws.String(id)) 41 | } 42 | 43 | params := &ec2.DescribeSubnetsInput{ 44 | SubnetIds: awsSubnetIds, 45 | } 46 | 47 | resp, err := session.DescribeSubnets(params) 48 | if err != nil { 49 | return []Subnet{}, err 50 | } 51 | 52 | var subnets []Subnet 53 | for _, subnet := range resp.Subnets { 54 | subnets = append(subnets, Subnet{ 55 | SubnetID: *subnet.SubnetId, 56 | CIDRBlock: *subnet.CidrBlock, 57 | }) 58 | } 59 | 60 | return subnets, nil 61 | } 62 | 63 | func (a AWS) Session() (Session, error) { 64 | if a.AccessKeyID == "" { 65 | return nil, errors.New("aws access key id must be provided") 66 | } 67 | 68 | if a.SecretAccessKey == "" { 69 | return nil, errors.New("aws secret access key must be provided") 70 | } 71 | 72 | if a.Region == "" { 73 | return nil, errors.New("aws region must be provided") 74 | } 75 | 76 | awsConfig := &aws.Config{ 77 | Credentials: credentials.NewStaticCredentials(a.AccessKeyID, a.SecretAccessKey, ""), 78 | Region: aws.String(a.Region), 79 | Endpoint: aws.String(a.EndpointOverride), 80 | } 81 | 82 | return ec2.New(session.New(awsConfig)), nil 83 | } 84 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/clients/aws_test.go: -------------------------------------------------------------------------------- 1 | package clients_test 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/credentials" 8 | "github.com/aws/aws-sdk-go/service/ec2" 9 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 10 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/fakes" 11 | 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | var _ = Describe("AWS", func() { 17 | var ( 18 | awsClient clients.AWS 19 | fakeSession *fakes.Session 20 | ) 21 | BeforeEach(func() { 22 | awsClient = clients.NewAWS("some-access-key-id", "some-secret-access-key", "some-region", "some-endpoint") 23 | fakeSession = &fakes.Session{} 24 | }) 25 | 26 | Describe("Session", func() { 27 | It("returns a new ec2 session", func() { 28 | session, err := awsClient.Session() 29 | Expect(err).NotTo(HaveOccurred()) 30 | 31 | _, ok := session.(clients.Session) 32 | Expect(ok).To(BeTrue()) 33 | 34 | client, ok := session.(*ec2.EC2) 35 | Expect(ok).To(BeTrue()) 36 | 37 | Expect(client.Config.Credentials).To(Equal(credentials.NewStaticCredentials("some-access-key-id", "some-secret-access-key", ""))) 38 | Expect(client.Config.Region).To(Equal(aws.String("some-region"))) 39 | Expect(client.Config.Endpoint).To(Equal(aws.String("some-endpoint"))) 40 | }) 41 | 42 | Context("failure cases", func() { 43 | It("returns an error if no access key id is provided", func() { 44 | awsClient := clients.NewAWS("", "some-secret-access-key", "some-region", "some-endpoint") 45 | _, err := awsClient.Session() 46 | Expect(err).NotTo(BeNil()) 47 | Expect(err.Error()).To(ContainSubstring("aws access key id must be provided")) 48 | }) 49 | 50 | It("returns an error if no secret access key is provided", func() { 51 | awsClient := clients.NewAWS("some-access-key-id", "", "some-region", "some-endpoint") 52 | _, err := awsClient.Session() 53 | Expect(err).NotTo(BeNil()) 54 | Expect(err.Error()).To(ContainSubstring("aws secret access key must be provided")) 55 | }) 56 | 57 | It("returns an error if no region is provided", func() { 58 | awsClient := clients.NewAWS("some-access-key-id", "some-secret-access-key", "", "some-endpoint") 59 | _, err := awsClient.Session() 60 | Expect(err).NotTo(BeNil()) 61 | Expect(err.Error()).To(ContainSubstring("aws region must be provided")) 62 | }) 63 | }) 64 | }) 65 | 66 | Describe("FetchSubnets", func() { 67 | It("returns subnets from aws that match subnetIds passed in", func() { 68 | fakeSession.DescribeSubnetsCall.Returns.DescribeSubnetsOutput = &ec2.DescribeSubnetsOutput{ 69 | Subnets: []*ec2.Subnet{ 70 | &ec2.Subnet{ 71 | SubnetId: aws.String("subnet-1"), 72 | CidrBlock: aws.String("some-cidr-block"), 73 | }, 74 | }, 75 | } 76 | 77 | subnets, err := awsClient.FetchSubnets(fakeSession, []string{"subnet-1"}) 78 | Expect(err).NotTo(HaveOccurred()) 79 | Expect(subnets).To(Equal([]clients.Subnet{ 80 | clients.Subnet{ 81 | SubnetID: "subnet-1", 82 | CIDRBlock: "some-cidr-block", 83 | }, 84 | })) 85 | }) 86 | Context("failure cases", func() { 87 | It("returns an error if describe subnets fails", func() { 88 | fakeSession.DescribeSubnetsCall.Returns.Error = errors.New("DescribeSubnets failed") 89 | 90 | _, err := awsClient.FetchSubnets(fakeSession, []string{"subnet-1"}) 91 | Expect(err).NotTo(BeNil()) 92 | Expect(err.Error()).To(ContainSubstring("DescribeSubnets failed")) 93 | }) 94 | }) 95 | }) 96 | }) 97 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/clients/bosh.go: -------------------------------------------------------------------------------- 1 | package clients 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "time" 7 | 8 | "github.com/pivotal-cf-experimental/bosh-test/bosh" 9 | ) 10 | 11 | type BOSHClient interface { 12 | Deploy(manifest []byte) (int, error) 13 | DeleteDeployment(deploymentName string) error 14 | GetTaskOutput(taskId int) ([]bosh.TaskOutput, error) 15 | Info() (bosh.DirectorInfo, error) 16 | } 17 | 18 | type BOSH struct { 19 | boshClient BOSHClient 20 | Logger io.Writer 21 | } 22 | 23 | func NewBOSH(client BOSHClient, logger io.Writer) BOSH { 24 | return BOSH{ 25 | boshClient: client, 26 | Logger: logger, 27 | } 28 | } 29 | 30 | func (b BOSH) Deploy(manifest []byte) error { 31 | taskId, err := b.boshClient.Deploy(manifest) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | taskOutputs, err := b.boshClient.GetTaskOutput(taskId) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | fmt.Fprintf(b.Logger, "Bosh Task %d:\n", taskId) 42 | for _, taskOutput := range taskOutputs { 43 | fmt.Fprintf(b.Logger, "[%s] Stage: %s Task: %s State: %s Progress: %d\n", time.Unix(taskOutput.Time, 0).UTC().Format(time.UnixDate), taskOutput.Stage, taskOutput.Task, taskOutput.State, taskOutput.Progress) 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func (b *BOSH) UUID() (string, error) { 50 | info, err := b.boshClient.Info() 51 | if err != nil { 52 | return "", err 53 | } 54 | 55 | return info.UUID, nil 56 | } 57 | 58 | func (b *BOSH) DeleteDeployment(deploymentName string) error { 59 | if err := b.boshClient.DeleteDeployment(deploymentName); err != nil { 60 | return err 61 | } 62 | 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/clients/bosh_test.go: -------------------------------------------------------------------------------- 1 | package clients_test 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io/ioutil" 7 | 8 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 9 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/fakes" 10 | "github.com/pivotal-cf-experimental/bosh-test/bosh" 11 | 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | var _ = Describe("BOSH", func() { 17 | Describe("Deploy", func() { 18 | It("deploys a given manifest", func() { 19 | boshClient := &fakes.BOSH{} 20 | logger := bytes.NewBuffer([]byte{}) 21 | client := clients.NewBOSH(boshClient, logger) 22 | boshClient.DeployCall.Returns.TaskId = 1 23 | boshClient.GetTaskOutputCall.Returns.TaskOutputs = []bosh.TaskOutput{ 24 | { 25 | Time: 1457571106, 26 | Stage: "some-stage", 27 | Task: "some-task", 28 | State: "some-state", 29 | Progress: 0, 30 | }, 31 | } 32 | 33 | manifest := []byte("some-manifest") 34 | err := client.Deploy(manifest) 35 | Expect(err).NotTo(HaveOccurred()) 36 | Expect(boshClient.DeployCall.Receives.Manifest).To(Equal([]byte("some-manifest"))) 37 | 38 | Expect(boshClient.GetTaskOutputCall.Receives.TaskId).To(Equal(1)) 39 | Expect(logger.String()).To(ContainSubstring("Bosh Task 1:")) 40 | Expect(logger.String()).To(ContainSubstring("[Thu Mar 10 00:51:46 UTC 2016] Stage: some-stage Task: some-task State: some-state Progress: 0\n")) 41 | }) 42 | 43 | Context("failure cases", func() { 44 | It("returns an error when the deployment fails", func() { 45 | boshClient := &fakes.BOSH{} 46 | client := clients.NewBOSH(boshClient, ioutil.Discard) 47 | boshClient.DeployCall.Returns.Error = errors.New("something bad happened") 48 | 49 | manifest := []byte("some-manifest") 50 | err := client.Deploy(manifest) 51 | Expect(err).To(MatchError("something bad happened")) 52 | }) 53 | 54 | It("returns an error when get task output fails", func() { 55 | boshClient := &fakes.BOSH{} 56 | client := clients.NewBOSH(boshClient, ioutil.Discard) 57 | boshClient.GetTaskOutputCall.Returns.Error = errors.New("something bad happened") 58 | 59 | manifest := []byte("some-manifest") 60 | err := client.Deploy(manifest) 61 | Expect(err).To(MatchError("something bad happened")) 62 | }) 63 | }) 64 | }) 65 | 66 | Describe("UUID", func() { 67 | It("retrieves the UUID from the bosh director", func() { 68 | boshClient := &fakes.BOSH{} 69 | boshClient.InfoCall.Returns.Info = bosh.DirectorInfo{ 70 | UUID: "some-uuid", 71 | } 72 | client := clients.NewBOSH(boshClient, ioutil.Discard) 73 | 74 | uuid, err := client.UUID() 75 | Expect(err).NotTo(HaveOccurred()) 76 | Expect(uuid).To(Equal("some-uuid")) 77 | }) 78 | 79 | Context("failure cases", func() { 80 | It("returns an error when the deployment fails", func() { 81 | boshClient := &fakes.BOSH{} 82 | client := clients.NewBOSH(boshClient, ioutil.Discard) 83 | boshClient.InfoCall.Returns.Error = errors.New("something bad happened") 84 | 85 | _, err := client.UUID() 86 | Expect(err).To(MatchError("something bad happened")) 87 | }) 88 | }) 89 | }) 90 | 91 | Describe("DeleteDeployment", func() { 92 | It("delete the deployment with the given name", func() { 93 | boshClient := &fakes.BOSH{} 94 | client := clients.NewBOSH(boshClient, ioutil.Discard) 95 | 96 | err := client.DeleteDeployment("some-deployment") 97 | Expect(err).NotTo(HaveOccurred()) 98 | Expect(boshClient.DeleteDeploymentCall.Receives.Name).To(Equal("some-deployment")) 99 | }) 100 | 101 | Context("failure cases", func() { 102 | It("returns an error when the deployment fails", func() { 103 | boshClient := &fakes.BOSH{} 104 | client := clients.NewBOSH(boshClient, ioutil.Discard) 105 | boshClient.DeleteDeploymentCall.Returns.Error = errors.New("something bad happened") 106 | 107 | err := client.DeleteDeployment("some-deployment-name") 108 | Expect(err).To(MatchError("something bad happened")) 109 | }) 110 | }) 111 | }) 112 | }) 113 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/clients/init_test.go: -------------------------------------------------------------------------------- 1 | package clients_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestClients(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "clients") 13 | } 14 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/deploy-consul-aws-manifests: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | consul_release_version="99999+dev.$(date +%s)" 5 | 6 | function upload_consul_release() { 7 | pushd "${ROOT}/release" > /dev/null 8 | bosh -n create release --force --version ${consul_release_version} 9 | bosh -n -t ${BOSH_DIRECTOR} upload release --rebase 10 | popd > /dev/null 11 | } 12 | 13 | function deploy_consul() { 14 | set +ux 15 | source ~/.bashrc 16 | set -ux 17 | 18 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry" 19 | pushd "${GOPATH}/src/github.com/cloudfoundry" > /dev/null 20 | ln -s "${ROOT}/mega-ci" 21 | pushd ./mega-ci/scripts/ci/deploy-aws-manifests > /dev/null 22 | set +x 23 | go run main.go \ 24 | --manifest-path "${ROOT}/release/${MANIFEST_PATH}" \ 25 | --director "${BOSH_DIRECTOR}" \ 26 | --user "${BOSH_USER}" \ 27 | --password "${BOSH_PASSWORD}" \ 28 | --aws-access-key-id "${AWS_ACCESS_KEY_ID}" \ 29 | --aws-secret-access-key "${AWS_SECRET_ACCESS_KEY}" \ 30 | --aws-region "${AWS_DEFAULT_REGION}" 31 | set -x 32 | popd > /dev/null 33 | popd > /dev/null 34 | } 35 | 36 | function main() { 37 | upload_consul_release 38 | deploy_consul 39 | } 40 | 41 | function teardown() { 42 | set +e 43 | bosh -n -t ${BOSH_DIRECTOR} delete release consul ${consul_release_version} 44 | set -e 45 | } 46 | 47 | trap teardown EXIT 48 | 49 | main 50 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/deploy-consul-aws-manifests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: release 9 | 10 | params: 11 | BOSH_DIRECTOR: 12 | BOSH_USER: 13 | BOSH_PASSWORD: 14 | AWS_ACCESS_KEY_ID: 15 | AWS_SECRET_ACCESS_KEY: 16 | AWS_DEFAULT_REGION: 17 | MANIFEST_PATH: 18 | 19 | run: 20 | path: mega-ci/scripts/ci/deploy-aws-manifests/deploy-consul-aws-manifests 21 | 22 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/deploy-etcd-aws-manifests: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | release_version="99999+dev.$(date +%s)" 5 | 6 | function upload_release() { 7 | local release_path="$1" 8 | 9 | pushd "${ROOT}/${release_path}" > /dev/null 10 | /opt/rubies/ruby-2.2.4/bin/bosh -n create release --force --version ${release_version} 11 | /opt/rubies/ruby-2.2.4/bin/bosh -n -t ${BOSH_DIRECTOR} upload release --rebase 12 | popd > /dev/null 13 | } 14 | 15 | function deploy_etcd() { 16 | set +ux 17 | source ~/.bashrc 18 | set -ux 19 | 20 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry" 21 | pushd "${GOPATH}/src/github.com/cloudfoundry" > /dev/null 22 | ln -s "${ROOT}/mega-ci" 23 | pushd ./mega-ci/scripts/ci/deploy-aws-manifests > /dev/null 24 | set +x 25 | go run main.go \ 26 | --manifest-path "${ROOT}/release/${MANIFEST_PATH}" \ 27 | --director "${BOSH_DIRECTOR}" \ 28 | --user "${BOSH_USER}" \ 29 | --password "${BOSH_PASSWORD}" \ 30 | --aws-access-key-id "${AWS_ACCESS_KEY_ID}" \ 31 | --aws-secret-access-key "${AWS_SECRET_ACCESS_KEY}" \ 32 | --aws-region "${AWS_DEFAULT_REGION}" 33 | set -x 34 | popd > /dev/null 35 | popd > /dev/null 36 | } 37 | 38 | function main() { 39 | local etcd_release_path="release" 40 | local consul_release_path="consul-release" 41 | 42 | upload_release "${consul_release_path}" 43 | upload_release "${etcd_release_path}" 44 | deploy_etcd 45 | } 46 | 47 | function teardown() { 48 | set +e 49 | /opt/rubies/ruby-2.2.4/bin/bosh -n -t ${BOSH_DIRECTOR} delete release consul ${release_version} 50 | /opt/rubies/ruby-2.2.4/bin/bosh -n -t ${BOSH_DIRECTOR} delete release etcd ${release_version} 51 | set -e 52 | } 53 | 54 | trap teardown EXIT 55 | 56 | main 57 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/deploy-etcd-aws-manifests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: release 9 | - name: consul-release 10 | 11 | params: 12 | BOSH_DIRECTOR: 13 | BOSH_USER: 14 | BOSH_PASSWORD: 15 | AWS_ACCESS_KEY_ID: 16 | AWS_SECRET_ACCESS_KEY: 17 | AWS_DEFAULT_REGION: 18 | MANIFEST_PATH: 19 | 20 | run: 21 | path: mega-ci/scripts/ci/deploy-aws-manifests/deploy-etcd-aws-manifests 22 | 23 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/fakes/aws.go: -------------------------------------------------------------------------------- 1 | package fakes 2 | 3 | import "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 4 | 5 | type AWS struct { 6 | SessionCall struct { 7 | Returns struct { 8 | Session clients.Session 9 | Error error 10 | } 11 | } 12 | FetchSubnetsCall struct { 13 | Receives struct { 14 | SubnetIds []string 15 | } 16 | Returns struct { 17 | Subnets []clients.Subnet 18 | Error error 19 | } 20 | } 21 | } 22 | 23 | func (a *AWS) Session() (clients.Session, error) { 24 | return a.SessionCall.Returns.Session, a.SessionCall.Returns.Error 25 | } 26 | 27 | func (a *AWS) FetchSubnets(session clients.Session, subnetIds []string) ([]clients.Subnet, error) { 28 | a.FetchSubnetsCall.Receives.SubnetIds = subnetIds 29 | return a.FetchSubnetsCall.Returns.Subnets, a.FetchSubnetsCall.Returns.Error 30 | } 31 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/fakes/bosh.go: -------------------------------------------------------------------------------- 1 | package fakes 2 | 3 | import ( 4 | "github.com/pivotal-cf-experimental/bosh-test/bosh" 5 | ) 6 | 7 | type BOSH struct { 8 | DeployCall struct { 9 | CallCount int 10 | Receives struct { 11 | Manifest []byte 12 | } 13 | Returns struct { 14 | TaskId int 15 | Error error 16 | } 17 | } 18 | 19 | InfoCall struct { 20 | Returns struct { 21 | Info bosh.DirectorInfo 22 | Error error 23 | } 24 | } 25 | 26 | DeleteDeploymentCall struct { 27 | Receives struct { 28 | Name string 29 | } 30 | 31 | Returns struct { 32 | Error error 33 | } 34 | } 35 | 36 | GetTaskOutputCall struct { 37 | Receives struct { 38 | TaskId int 39 | } 40 | Returns struct { 41 | TaskOutputs []bosh.TaskOutput 42 | Error error 43 | } 44 | } 45 | } 46 | 47 | func (b *BOSH) Deploy(manifest []byte) (int, error) { 48 | b.DeployCall.CallCount++ 49 | b.DeployCall.Receives.Manifest = manifest 50 | return b.DeployCall.Returns.TaskId, b.DeployCall.Returns.Error 51 | } 52 | 53 | func (b *BOSH) Info() (bosh.DirectorInfo, error) { 54 | return b.InfoCall.Returns.Info, b.InfoCall.Returns.Error 55 | } 56 | 57 | func (b *BOSH) DeleteDeployment(name string) error { 58 | b.DeleteDeploymentCall.Receives.Name = name 59 | return b.DeleteDeploymentCall.Returns.Error 60 | } 61 | 62 | func (b *BOSH) GetTaskOutput(taskId int) ([]bosh.TaskOutput, error) { 63 | b.GetTaskOutputCall.Receives.TaskId = taskId 64 | return b.GetTaskOutputCall.Returns.TaskOutputs, b.GetTaskOutputCall.Returns.Error 65 | } 66 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/fakes/session.go: -------------------------------------------------------------------------------- 1 | package fakes 2 | 3 | import "github.com/aws/aws-sdk-go/service/ec2" 4 | 5 | type Session struct { 6 | DescribeSubnetsCall struct { 7 | Returns struct { 8 | DescribeSubnetsOutput *ec2.DescribeSubnetsOutput 9 | Error error 10 | } 11 | } 12 | } 13 | 14 | func (s *Session) DescribeSubnets(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) { 15 | return s.DescribeSubnetsCall.Returns.DescribeSubnetsOutput, s.DescribeSubnetsCall.Returns.Error 16 | } 17 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/fakes/subnet_checker.go: -------------------------------------------------------------------------------- 1 | package fakes 2 | 3 | type SubnetChecker struct { 4 | CheckSubnetsCall struct { 5 | Returns struct { 6 | Bool bool 7 | Error error 8 | } 9 | } 10 | } 11 | 12 | func (s *SubnetChecker) CheckSubnets(manifestFilename string) (bool, error) { 13 | return s.CheckSubnetsCall.Returns.Bool, s.CheckSubnetsCall.Returns.Error 14 | } 15 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/fixtures/multi-az-ssl.yml: -------------------------------------------------------------------------------- 1 | director_uuid: BOSH-DIRECTOR-UUID 2 | 3 | name: multi-az-ssl 4 | 5 | releases: 6 | - name: consul 7 | version: latest 8 | 9 | compilation: 10 | cloud_properties: 11 | instance_type: m3.medium 12 | availability_zone: us-east-1a 13 | ephemeral_disk: 14 | size: 1024 15 | type: gp2 16 | network: consul1 17 | reuse_compilation_vms: true 18 | workers: 3 19 | 20 | jobs: 21 | - instances: 3 22 | name: consul_z1 23 | networks: 24 | - name: consul1 25 | static_ips: 26 | - 10.0.20.4 27 | - 10.0.20.5 28 | - 10.0.20.6 29 | resource_pool: consul_z1 30 | templates: 31 | - name: consul_agent 32 | release: consul 33 | persistent_disk: 1024 34 | properties: 35 | consul: 36 | agent: 37 | mode: server 38 | - instances: 3 39 | name: consul_z2 40 | networks: 41 | - name: consul2 42 | static_ips: 43 | - 10.0.21.4 44 | - 10.0.21.5 45 | - 10.0.21.6 46 | resource_pool: consul_z2 47 | templates: 48 | - name: consul_agent 49 | release: consul 50 | persistent_disk: 1024 51 | properties: 52 | consul: 53 | agent: 54 | mode: server 55 | 56 | networks: 57 | - name: consul1 58 | - name: consul2 59 | 60 | resource_pools: 61 | - cloud_properties: 62 | instance_type: m3.medium 63 | availability_zone: us-east-1a 64 | ephemeral_disk: 65 | size: 1024 66 | type: gp2 67 | name: consul_z1 68 | network: consul1 69 | stemcell: 70 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 71 | version: latest 72 | - cloud_properties: 73 | instance_type: m3.medium 74 | availability_zone: us-east-1c 75 | ephemeral_disk: 76 | size: 1024 77 | type: gp2 78 | name: consul_z2 79 | network: consul2 80 | stemcell: 81 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 82 | version: latest 83 | 84 | update: 85 | canaries: 1 86 | canary_watch_time: 1000-180000 87 | max_in_flight: 1 88 | serial: true 89 | update_watch_time: 1000-180000 90 | 91 | properties: 92 | consul: 93 | agent: 94 | servers: 95 | lan: 96 | - 10.0.20.4 97 | - 10.0.20.5 98 | - 10.0.20.6 99 | - 10.0.21.4 100 | - 10.0.21.5 101 | - 10.0.21.6 102 | ca_cert: |+ 103 | -----BEGIN CERTIFICATE----- 104 | MIIFAzCCAu2gAwIBAgIBATALBgkqhkiG9w0BAQswEzERMA8GA1UEAxMIY29uc3Vs 105 | Q0EwHhcNMTUwODA4MDAxMTI4WhcNMjUwODA4MDAxMTM0WjATMREwDwYDVQQDEwhj 106 | b25zdWxDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMzjDTn+WDVK 107 | AJ6H+8T2krtyzoMBTwEzGcG9UojDfvgdkbht1jLmbyPRJ06k6qc4q08ppdqDDCTg 108 | OKnfib1vB839+gqMQjor+66rMMgazwwzlfcm2gCKYNbkuHe8iUYQpyBjJNMF6O6r 109 | tHcxW8ETqO8tbXODBEKOxKzjTylCGl41+tRt4fG8mreCs8go7IHmughkQaxck5gc 110 | kNy0LVIYiM5UlpTGQJHJzCkT2jCaqV665VHDlFY/X5UXNOEGDjx7UA2NIE9zCVD3 111 | g2HBNxB+kPyvpXGWGz6X+fqJkHXe5xhMMKv9Omf3xZNd+7o+nWD0EDz2sHdnH60C 112 | lQE+eM9ZhtQmII3MF7AskiKF3x2W+u87W0woJf5xgTVT+7PZRyb25r+g8unCJdRv 113 | XZnO4g6rM5yAZtyh3DqepiRFrwvuOWUsbGdGZmv1jL07xL1uNIvJtwXlwIKwKeab 114 | OK0PMrrehxcmMJqp+Osoy+YwlSSG54u/VKFsI28W1kL68GEkV3oevlb0hIRIPajT 115 | YCIl266Zdo4sO7lmEJysDBi5p0zBz+erOmNhpxm1UsUssZrYaHnVLiodnc2jx4hB 116 | 0HhiU78Dt7B+jyuc9D3C91wellcgOgLfluQgvPS+n0/EopfLGG4RDDsASe+hqUWH 117 | ExxZBac1DPqLhywmb2zRJijXwiiBD/CJAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIA 118 | BjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBQf89zduge2sxN5FVLeQzYH 119 | rpw6JzAfBgNVHSMEGDAWgBQf89zduge2sxN5FVLeQzYHrpw6JzALBgkqhkiG9w0B 120 | AQsDggIBAIdodeDVVF+HZXH5mmhuxmdg0RCU6pnvYODPKhwZiIT9lEHmFlexaF85 121 | UYvh5z3DL1beisH1yfrN/7z+535nPyIVJUHPHziaKCPed5au4LiHWeRBd/O0kj4W 122 | P+GIBRJvAvShWQuQT30hlzh3ZAdTRVNNrKY4UMRclZbTv2dBT7IdlN/FsqHp1GbK 123 | bGHQH5/AneF/vttjRiaMJZFrHq3Aw0wntzRCAIIfVZo5r93KGiGUFWXQKdMyoOYD 124 | R04DQPzu/otY0hqUPuzMqPU5xYJzLopOtfKpQtnhpB/yNbyjmAh3795zWb51Gead 125 | cO6BfaCpzR0vA0SkfTBMhqmvc0AUvoGOqh3tNibFkVBo17KTEWphyIl2+61s/a7N 126 | 22xgQZIrwyMWXL6Mr4ZHloryJ1Nu5iFUHfT8ATYtqNotGATEV+oi03xACSzqAboE 127 | rmGwMRNJAtWbeTqhS6bFqQAkAmvpZBO+h3LkoAixcCIoNXWJA9+pDptIRU48Av2u 128 | /DkAOjhNh2MJkQs8owhAvxKsT1BwsPlc71tGzFLEbOJkGwt+RtzqjWc6Nq+l+uup 129 | sD0sQMSg/b/b60OtJ/Qm5+u42kDaQi3v9+92S/bs9fTzA25c/mr8KA6dvG4LAsJU 130 | /QlnEFyH789135x864ym1G2fvaPTqcgIc7jfw3tMTs5TtP/VSTn0 131 | -----END CERTIFICATE----- 132 | agent_cert: |+ 133 | -----BEGIN CERTIFICATE----- 134 | MIIEITCCAgugAwIBAgIQTDMCJHBiQrY8hlM78doHTzALBgkqhkiG9w0BAQswEzER 135 | MA8GA1UEAxMIY29uc3VsQ0EwHhcNMTUwODA4MDAxMTM3WhcNMTcwODA4MDAxMTM4 136 | WjAXMRUwEwYDVQQDEwxjb25zdWwgYWdlbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IB 137 | DwAwggEKAoIBAQC9ybg/ARaW53ItlhLQlEobtzqjCgMchq6G8T8+Y6nV8lTFB9+E 138 | AJyQbNz0OATZO09Ezexijict4YL9Ux0oekQ8n9EjoNFcF0VBqa9Iy137cevY97hh 139 | Em8a6w5aepyaoh9YcWXGnp4uJ+xf4tzQUQCkb0QuedqSobJhPOrCCRyfL4KuHX0j 140 | caJD7MDDX3brlxYLJAKTdaV9xWpLVE/9MMrTQnaZoiJOqoyDMc3scs4gkZ0ohM0R 141 | yyFHPp3AC9Y0bZNcpitDYNzRFFvrPzNZTUrM0ar8kwI+Xt7Pbpc47v8xxVAQswXM 142 | wPg+1yd9Jr7N3belToFZCvIJeI4vDF97MAM7AgMBAAGjcTBvMA4GA1UdDwEB/wQE 143 | AwIAuDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwHQYDVR0OBBYEFC/B 144 | 5cFoXH/8xxXZ+Z4aJrqKNT0fMB8GA1UdIwQYMBaAFB/z3N26B7azE3kVUt5DNgeu 145 | nDonMAsGCSqGSIb3DQEBCwOCAgEAM5mUgC45Mz17CGxP64cm/CVuHLMEsBThufYH 146 | FlVk/NOi2fRTxEYzD1kx7lEk1T8D+cktpcLhU7AdHt8ckYfbpkOCfU27Pnql006h 147 | tHGGut5K8mfLk/8/qwx99c1J9IE1lgWxPpzVylI8QSG/NkDis3ZUqlv4R0YqSVIs 148 | a+NLjWbVHwRRPtUtlOh8aPlWbV9JlM7HWGJwA1K+lz3cG22hxw/lhS+gNkBdV4Ce 149 | waTTjKU6c5/Y+7TU00wBy0TFfg0M5ZxnzzJaxSG7bOvy/OYPeIDGsSu4P8BWkJzb 150 | wdfyYn+PzIEH49QQJpaExVy0SmMTwh9pVvQImBkexN0e+3PnynDq/GWu+vdBBAHr 151 | 8IPTVyxwZI1xwLV6LQsX2sMvxYnQTVxxv3/spNbzhmF2Pf0/XKVOawzqOUqSAXc/ 152 | 0uSF9j5EhFFF9azqfdffzqju3q3/4npJn8XKgN6Ve071Li46h+0A+Y+FRdd0zMMj 153 | qJd8kgL/rTWPINp0eI47RHW6xMIiZikeeT3BTlO4eSLOc/bUG+oAkfaFNTXBneDX 154 | K0+CThR8fyn4ukW3jgAmwx2xmzn2uQvWKxPc3IpofDxNes2DP2J+/1p4jyoveOzp 155 | 6lwqqBsBz3E/z7sAqPOho4W1iI3o3BmzWhzAfEYuvHlSaZ8Ju74hDuh2pMnMCITB 156 | ZjkHgOE= 157 | -----END CERTIFICATE----- 158 | agent_key: |+ 159 | -----BEGIN RSA PRIVATE KEY----- 160 | MIIEpAIBAAKCAQEAvcm4PwEWludyLZYS0JRKG7c6owoDHIauhvE/PmOp1fJUxQff 161 | hACckGzc9DgE2TtPRM3sYo4nLeGC/VMdKHpEPJ/RI6DRXBdFQamvSMtd+3Hr2Pe4 162 | YRJvGusOWnqcmqIfWHFlxp6eLifsX+Lc0FEApG9ELnnakqGyYTzqwgkcny+Crh19 163 | I3GiQ+zAw19265cWCyQCk3WlfcVqS1RP/TDK00J2maIiTqqMgzHN7HLOIJGdKITN 164 | EcshRz6dwAvWNG2TXKYrQ2Dc0RRb6z8zWU1KzNGq/JMCPl7ez26XOO7/McVQELMF 165 | zMD4PtcnfSa+zd23pU6BWQryCXiOLwxfezADOwIDAQABAoIBAGQNkjp04mnPvlvW 166 | ADlB1Afkgt8tChLh1eSCDmtfs0t9I79ztRwsDl39ZkbBuZykLdgCD5qtgm4nsxzF 167 | 0ltiidLEFkeIIpdAFVzWjff8bqF+n22UuvlB9JzmSnzcou7KhWfk8FjTimSe23h5 168 | axvPeF0342P1DYu7/wRQTxrYHuS0BvCmDPZ07wzFJbOqx6gHM1oAoN3ICTswlKxU 169 | a8L0nDkeTtjDwBvoStdpY7+3xUu6zvLxCuLn9bAQjJQVd5Qs5599VNIY055GSJxs 170 | 0T/p05+ViZWJt4ezUka+7uaKc1ApIA9uLwPiDYlzYNQPgvNpJklb0jw8y/NNaFYl 171 | GZrS0GECgYEA+1fSkOQvQPR5ZkKHJ24mLHIyN5hopySrEm8mUSPyBgvAMIN8aIFl 172 | c1UsecVAbT7C57lo8Kg51GyTcRcOFegce8K2g60n5VZRqwIKZ5QFKGcnbbmHlrk6 173 | U6PCL1sVydKMK86lXWHigRMkPX9THewn5fimTVpDtSiWKXosSE0pL0kCgYEAwU3t 174 | ZwE6MFXdpb+o2ZVO7AxqRAjt+DvG/Jyzi2mUm2Q/IPEf9GYrojMK843SmFa+hW/R 175 | +WYfjyeXxX/pjiNaajcC9i0/IjHhghtjUMgPTnk3tWg+6JFoGqYOMxmD1jL8JQJ6 176 | Q4dMiAJF0FBtA3OWt0rsnHCYnZKGk6+49IS66mMCgYEAr3/eHp/GQwOxtO6lMqod 177 | 65D9cYZxJKxc621DAwsnVoNj+jwFZW9cqFCD0q8h0mYtb+It3USJxMLfnOqtQiyl 178 | nuER0hXZMrC4K8EsBLD9fP2yMVKH032YtYg18h0WtKrYh0oue2r63ofAGVTLc6xP 179 | G6woawCpIFirqWCOhRmjtUkCgYB41wdGshy3JKZnvDWIt27b3zL7Dv38hRnyxt7y 180 | kvEEZxmTtUi9lrVGM1RwRsNU2j1F205O80ldS2n0W+8cHPujlHo0fLqP3NDVbdub 181 | H0V6UArES8OvzV5f8ljEFvo0iDDZzf+ygT0VdR5BiFVtp++a66qYbUbqkjOw1VPw 182 | /5x9cwKBgQC69SK7iEZZs3v/wgS/wycxXeJ1KV17wvez3D8XjodO37wh+VrTU5Vi 183 | FswLS6coxP5optNulBjEogYA4FV6RW1KpJtzOK9pYbYquYZ7s7jJob99FAG/S4w+ 184 | 32Mj4ovtRtbWPQdq5SNnSWOrp92FSXSZAWTIpGAZC2jaNg6ofV/XNw== 185 | -----END RSA PRIVATE KEY----- 186 | server_cert: |+ 187 | -----BEGIN CERTIFICATE----- 188 | MIIELDCCAhagAwIBAgIRAN0EJexxqEEyVatWrjzc7zAwCwYJKoZIhvcNAQELMBMx 189 | ETAPBgNVBAMTCGNvbnN1bENBMB4XDTE1MDgwODAwMTEzNloXDTE3MDgwODAwMTEz 190 | NlowITEfMB0GA1UEAxMWc2VydmVyLmRjMS5jZi5pbnRlcm5hbDCCASIwDQYJKoZI 191 | hvcNAQEBBQADggEPADCCAQoCggEBAMnlRbL6VBmDhSqCzVfez1yJsntK1YD1pdbP 192 | LBgdfqxKEJv3p6w1FJH/TFpbKQU2HGvGgeCooUcO2fo0pkW8nDp9xASG3eMeL7pG 193 | aTSYrlCGD9jmGn7ENLOke6J+S3gjlf3Y029RchPNfUH6PnFQeSnYPmhSgTbeMVBm 194 | 02aN4KfOuGUiC3ivjaY3RrhcvybWGNhp8PK+hUYWnN8e6lyPDTo6kSgtapiSfCa3 195 | 32pZITRnQDs/1Y0es5Gjou/7Kh40cKf4pA38xlu8TIxpfsxzlnMbW4mNvP6cdoFg 196 | HUhT/uZhYIX0jEqEUSOM/zjj08HFAYYJTTQ7qovY/6z8WXeQR8kCAwEAAaNxMG8w 197 | DgYDVR0PAQH/BAQDAgC4MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAd 198 | BgNVHQ4EFgQUp59BTY2TSXRJ0GNzVmkUE14j5hgwHwYDVR0jBBgwFoAUH/Pc3boH 199 | trMTeRVS3kM2B66cOicwCwYJKoZIhvcNAQELA4ICAQBC37sEtmLj/qS7Q3RIezTc 200 | hs3EZJweWNN5vjEWAYVf9g4pq/f39hmMF9rqXugBxm8WgQRCGxDLzMr5IEX1fOcI 201 | Ksz98rUfWtS8WxuqFOihnZQmSj9KQmHvOCtgs/zIMW55AEPcdpVtZvl2Y2z9+j68 202 | K6Z2cfgZFC8OPkk4Kfvdj3o+wWfMlnaGrwSansmuRNupXGexwQ01L1qClXxDJc/c 203 | zGetvKs67HsOdSeuza2Y7D/OJUWdcyaGM7cqfs2MPoLiyAydQRGtXjCsH4mUjzKZ 204 | wZ9Rq72y4jmK/Stc40TpBDnav486PZtlap/F6rUhgKKJ3UPH3fBXkIDXtRhztcQa 205 | y670ZyTgFWn5mUYLB3Z66YMYqAIIJBX2cbwObbrLH1eI3Qtuh/rcCSms9XXz3jHu 206 | R6+saTPl3UWw2hrHc8MAuDSTa/Qsej8TMxigju9bZxxtIY0+sUbz436Yg1I43HzR 207 | /wIj682hruww4hwT7lQJUw8Zp0QQA2hfhfr4kPJGIz9+96cGT1LWiEwSuptcwYPw 208 | WV4VuQJccQN4yQYR1Qp1jepEBs7NuIX46VgSU55BKSsQf1zUqP6A3G9D0nnMtACO 209 | WpO0sgj3uHHCsNTC0DwR7TM5dByZWzJK0lQJCG6HzHkLcvovq/9uhn8dqOlpzeti 210 | amuBD7LTPTc3rnu4GL7aZA== 211 | -----END CERTIFICATE----- 212 | server_key: |+ 213 | -----BEGIN RSA PRIVATE KEY----- 214 | MIIEogIBAAKCAQEAyeVFsvpUGYOFKoLNV97PXImye0rVgPWl1s8sGB1+rEoQm/en 215 | rDUUkf9MWlspBTYca8aB4KihRw7Z+jSmRbycOn3EBIbd4x4vukZpNJiuUIYP2OYa 216 | fsQ0s6R7on5LeCOV/djTb1FyE819Qfo+cVB5Kdg+aFKBNt4xUGbTZo3gp864ZSIL 217 | eK+NpjdGuFy/JtYY2Gnw8r6FRhac3x7qXI8NOjqRKC1qmJJ8JrffalkhNGdAOz/V 218 | jR6zkaOi7/sqHjRwp/ikDfzGW7xMjGl+zHOWcxtbiY28/px2gWAdSFP+5mFghfSM 219 | SoRRI4z/OOPTwcUBhglNNDuqi9j/rPxZd5BHyQIDAQABAoIBADcgwopfbi1VSzxT 220 | YQyYS1jCRJkD0w6WhMUF5s1i6F7VDnn4ArG0ALzo0AlwEOBaaSJIntggU4FancyD 221 | Kc+q86HJC4gM53OHn5KmfT2eXyKfqJ4aHqv0mUtgI144TBUu6lrZJMqlm5eqqYQl 222 | d31rctopMMk9lgjMXPzORvUe0nQfMuD24y0vv+l8/O8Ib5mIzI8bBor7d+TNbF+W 223 | 1vXqlZ/NEpqrI/yyacJDFQC0yhDAbEC5SMiYumf7Vc9jJtBKDk2YCIvHpSjKdaIG 224 | MUa8JG6A8cpWhSdpP1GO6LnMFloC7LdioGaXLEJeUkWnbhGQ/VuDfqogBM/4lLIt 225 | IjDVEI0CgYEA639Ya/pgUe8kVl0mR/wuDI+EENWeJiLbumLpDhTFdiGXSl9ZIYez 226 | GAyR23rJOU2zGoIt4IRxxVq6QLPFq0ZTKoq1kGqyAqAc0GenKgU+Uq6Mc3E0A9MG 227 | z/+tBCoc9ZtC4+u/q9VdMAu/W4xJ7mtwVFItX115FsYUYw6p3hXsz/cCgYEA23kG 228 | YI/KxpAFW1V04urY8YUsdvPiDtJbA0JiQeWoNRUjIoJOUVY/2P+ZdVcCaWcXEi3l 229 | mjHPsXoNR5ZzHxbP/5tN8m/jl8tAD17is1qfPR7oUBfjZOHY7xFamWMK0Lo2hi7H 230 | xT7BJXLOwqV3hfYcJ+x51R/H3wjpyF5g1vAtNj8CgYA9ZV/qFsaR1eUFVxep4Mco 231 | oyntMaQfkSrz9uGHuEaau7szupQEN9qrRGuqauKXO/ibyqCnTiBTMYopYDUCqDz6 232 | dFtNoWNzZ8bbVoqwW9mZuMQJPNQwww2doKy8zzXpmmbgARBhfijjY8yp03Na40vP 233 | z/TgTgBJva6G/MWwjsrElQKBgCi6guZ0iMrkezoB19ksf+oCLsg8Zh0eCGnIbfeQ 234 | qPCA5a5HxETv3pVkiZPu+7GXwf5Lqio9SC/FWKWKU/7W+u6SYZq2DORkgZTYpPVn 235 | wdlT3QTQChD0oI9tBwUkDiPCCtBH6ia+iJVsgtY4Yr/ndj4qckmMxkirnMbkTNBW 236 | be19AoGAelJR7jzk6TLw6ZGbGXsAQSAcP5IUM7GHAPo26ywIOTe8p40hmKrVnxfY 237 | EroZtdjv1A9NVpCHJM7a/3nl0aALEsn3JKYEuG4HCY/oJ7rPpJItfrJ3MUTn/uqD 238 | BKcJHlf9kYxFaLL17iOq9rY0+LURr63BkRa2uyn489luxcHnTjk= 239 | -----END RSA PRIVATE KEY----- 240 | encrypt_keys: 241 | - Atzo3VBv+YVDzQAzlQRPRA== 242 | require_ssl: true 243 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/flags/flags.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | ) 7 | 8 | type Configuration struct { 9 | ManifestPath string 10 | BoshDirector string 11 | BoshUser string 12 | BoshPassword string 13 | AWSAccessKeyID string 14 | AWSSecretAccessKey string 15 | AWSRegion string 16 | AWSEndpointOverride string 17 | } 18 | 19 | func ParseFlags(arguments []string) (Configuration, error) { 20 | flags := flag.NewFlagSet("boshflags", flag.ContinueOnError) 21 | flags.SetOutput(ioutil.Discard) 22 | 23 | configuration := Configuration{} 24 | flags.StringVar(&configuration.ManifestPath, "manifest-path", "", "path to manifests directory or file") 25 | flags.StringVar(&configuration.BoshDirector, "director", "", "bosh director") 26 | flags.StringVar(&configuration.BoshUser, "user", "", "bosh user") 27 | flags.StringVar(&configuration.BoshPassword, "password", "", "bosh password") 28 | flags.StringVar(&configuration.AWSAccessKeyID, "aws-access-key-id", "", "aws access key id") 29 | flags.StringVar(&configuration.AWSSecretAccessKey, "aws-secret-access-key", "", "aws secret access key") 30 | flags.StringVar(&configuration.AWSRegion, "aws-region", "", "aws region") 31 | flags.StringVar(&configuration.AWSEndpointOverride, "aws-endpoint-override", "", "aws endpoint override") 32 | 33 | err := flags.Parse(arguments) 34 | if err != nil { 35 | return Configuration{}, err 36 | } 37 | 38 | return configuration, nil 39 | } 40 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/flags/flags_test.go: -------------------------------------------------------------------------------- 1 | package flags_test 2 | 3 | import ( 4 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/flags" 5 | . "github.com/onsi/ginkgo" 6 | . "github.com/onsi/gomega" 7 | ) 8 | 9 | var _ = Describe("flags", func() { 10 | It("extracts configuration data from the command line flags", func() { 11 | configuration, err := flags.ParseFlags([]string{ 12 | "--manifest-path", "some-manifest-path", 13 | "--director", "some-director", 14 | "--user", "some-user", 15 | "--password", "some-password", 16 | "--aws-access-key-id", "some-aws-access-key-id", 17 | "--aws-secret-access-key", "some-aws-secret-access-key", 18 | "--aws-region", "some-aws-region", 19 | "--aws-endpoint-override", "some-aws-endpoint-override", 20 | }) 21 | 22 | Expect(err).NotTo(HaveOccurred()) 23 | Expect(configuration.ManifestPath).To(Equal("some-manifest-path")) 24 | Expect(configuration.BoshDirector).To(Equal("some-director")) 25 | Expect(configuration.BoshUser).To(Equal("some-user")) 26 | Expect(configuration.BoshPassword).To(Equal("some-password")) 27 | Expect(configuration.AWSAccessKeyID).To(Equal("some-aws-access-key-id")) 28 | Expect(configuration.AWSSecretAccessKey).To(Equal("some-aws-secret-access-key")) 29 | Expect(configuration.AWSRegion).To(Equal("some-aws-region")) 30 | Expect(configuration.AWSEndpointOverride).To(Equal("some-aws-endpoint-override")) 31 | }) 32 | 33 | Describe("failure cases", func() { 34 | It("returns an error when flag parsing fails", func() { 35 | _, err := flags.ParseFlags([]string{"--wrong-flag", "some-string"}) 36 | Expect(err.Error()).To(ContainSubstring("flag provided but not defined")) 37 | }) 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/flags/init_test.go: -------------------------------------------------------------------------------- 1 | package flags_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestFlags(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "flags") 13 | } 14 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/init_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | "github.com/onsi/gomega/gexec" 7 | 8 | "testing" 9 | ) 10 | 11 | func TestMain(t *testing.T) { 12 | RegisterFailHandler(Fail) 13 | RunSpecs(t, "main") 14 | } 15 | 16 | var ( 17 | pathToMain string 18 | ) 19 | 20 | var _ = BeforeSuite(func() { 21 | var err error 22 | 23 | pathToMain, err = gexec.Build("github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests") 24 | Expect(err).NotTo(HaveOccurred()) 25 | }) 26 | 27 | var _ = AfterSuite(func() { 28 | gexec.CleanupBuildArtifacts() 29 | }) 30 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/awsdeployer" 8 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 9 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/flags" 10 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/subnetchecker" 11 | "github.com/pivotal-cf-experimental/bosh-test/bosh" 12 | ) 13 | 14 | func main() { 15 | configuration, err := flags.ParseFlags(os.Args[1:]) 16 | if err != nil { 17 | fmt.Fprintf(os.Stderr, "\n\n%s\n", err) 18 | os.Exit(1) 19 | } 20 | 21 | boshConfig := bosh.Config{ 22 | URL: configuration.BoshDirector, 23 | Password: configuration.BoshPassword, 24 | Username: configuration.BoshUser, 25 | AllowInsecureSSL: true, 26 | } 27 | 28 | aws := clients.NewAWS(configuration.AWSAccessKeyID, configuration.AWSSecretAccessKey, 29 | configuration.AWSRegion, configuration.AWSEndpointOverride) 30 | bosh := clients.NewBOSH(bosh.NewClient(boshConfig), os.Stdout) 31 | subnetChecker := subnetchecker.NewSubnetChecker(aws) 32 | 33 | awsDeployer := awsdeployer.NewAWSDeployer(bosh, subnetChecker, os.Stdout) 34 | 35 | err = awsDeployer.Deploy(configuration.ManifestPath) 36 | if err != nil { 37 | fmt.Fprintf(os.Stderr, "\n\n%s\n", err) 38 | os.Exit(1) 39 | } 40 | 41 | os.Exit(0) 42 | } 43 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/main_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "os/exec" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | "github.com/onsi/gomega/gexec" 12 | ) 13 | 14 | var _ = Describe("main", func() { 15 | It("deploys manifest file specified", func() { 16 | var awsWasCalled bool 17 | awsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | awsWasCalled = true 19 | })) 20 | 21 | var boshcalls struct { 22 | DeleteCall int 23 | DeployCall int 24 | InfoCall int 25 | } 26 | boshServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 27 | switch r.URL.Path { 28 | case "/deployments/multi-az-ssl": 29 | boshcalls.DeleteCall++ 30 | w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host)) 31 | w.WriteHeader(http.StatusFound) 32 | case "/deployments": 33 | if r.Method == "POST" { 34 | boshcalls.DeployCall++ 35 | } 36 | 37 | w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host)) 38 | w.WriteHeader(http.StatusFound) 39 | case "/tasks/1": 40 | w.Write([]byte(`{"id": 1, "state": "done"}`)) 41 | case "/tasks/1/output": 42 | w.Write([]byte(`{"time": 0, "stage": "some-stage", "task": "some-task", "state": "some-state", "progress": 0}`)) 43 | case "/info": 44 | boshcalls.InfoCall++ 45 | w.Write([]byte(`{"uuid":"some-director-uuid", "cpi":"some-cpi"}`)) 46 | default: 47 | return 48 | } 49 | })) 50 | 51 | args := []string{ 52 | "--manifest-path", "fixtures/multi-az-ssl.yml", 53 | "--director", boshServer.URL, 54 | "--user", "some-user", 55 | "--password", "some-password", 56 | "--aws-access-key-id", "some-aws-access-key-id", 57 | "--aws-secret-access-key", "some-aws-secret-access-key", 58 | "--aws-region", "some-aws-region", 59 | "--aws-endpoint-override", awsServer.URL, 60 | } 61 | 62 | session, err := gexec.Start(exec.Command(pathToMain, args...), GinkgoWriter, GinkgoWriter) 63 | Expect(err).NotTo(HaveOccurred()) 64 | Eventually(session).Should(gexec.Exit(0)) 65 | 66 | Expect(awsWasCalled).To(BeTrue()) 67 | 68 | Expect(boshcalls.DeleteCall).To(Equal(1)) 69 | Expect(boshcalls.DeployCall).To(Equal(1)) 70 | Expect(boshcalls.InfoCall).To(Equal(1)) 71 | }) 72 | 73 | It("prints an error when the deployment fails", func() { 74 | awsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 75 | })) 76 | 77 | boshServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 78 | w.WriteHeader(http.StatusBadGateway) 79 | })) 80 | 81 | args := []string{ 82 | "--manifest-path", "fixtures/multi-az-ssl.yml", 83 | "--director", boshServer.URL, 84 | "--user", "some-user", 85 | "--password", "some-password", 86 | "--aws-access-key-id", "some-aws-access-key-id", 87 | "--aws-secret-access-key", "some-aws-secret-access-key", 88 | "--aws-region", "some-aws-region", 89 | "--aws-endpoint-override", awsServer.URL, 90 | } 91 | 92 | session, err := gexec.Start(exec.Command(pathToMain, args...), GinkgoWriter, GinkgoWriter) 93 | Expect(err).NotTo(HaveOccurred()) 94 | Eventually(session).Should(gexec.Exit(1)) 95 | 96 | Expect(session.Err.Contents()).To(ContainSubstring("unexpected response 502 Bad Gateway")) 97 | }) 98 | 99 | It("prints an error when an unknown flag is provided", func() { 100 | session, err := gexec.Start(exec.Command(pathToMain, "--some-unknown-flag"), GinkgoWriter, GinkgoWriter) 101 | 102 | Expect(err).NotTo(HaveOccurred()) 103 | Eventually(session).Should(gexec.Exit(1)) 104 | Expect(session.Err.Contents()).To(ContainSubstring("flag provided but not defined: -some-unknown-flag")) 105 | }) 106 | }) 107 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/manifests/init_test.go: -------------------------------------------------------------------------------- 1 | package manifests_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestManifests(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "manifests") 13 | } 14 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/manifests/manifests.go: -------------------------------------------------------------------------------- 1 | package manifests 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | "gopkg.in/yaml.v2" 7 | ) 8 | 9 | type Manifest struct { 10 | Networks []Network 11 | } 12 | 13 | type Network struct { 14 | Subnets []struct { 15 | CloudProperties struct { 16 | Subnet string 17 | } `yaml:"cloud_properties"` 18 | Range string 19 | } 20 | } 21 | 22 | func ReadManifest(manifestFilename string) (map[string]interface{}, error) { 23 | contents, err := ioutil.ReadFile(manifestFilename) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | var document map[string]interface{} 29 | err = yaml.Unmarshal(contents, &document) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return document, nil 35 | } 36 | 37 | func ReadNetworksFromManifest(manifestFilename string) ([]Network, error) { 38 | contents, err := ioutil.ReadFile(manifestFilename) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | var manifest Manifest 44 | err = yaml.Unmarshal(contents, &manifest) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | return manifest.Networks, nil 50 | } 51 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/manifests/manifests_test.go: -------------------------------------------------------------------------------- 1 | package manifests_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/manifests" 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | const manifestWithSubnets = `--- 14 | director_uuid: BOSH-DIRECTOR-UUID 15 | 16 | name: multi-az-ssl 17 | 18 | networks: 19 | - subnets: 20 | - cloud_properties: 21 | subnet: "subnet-1" 22 | range: 10.0.20.0/24 23 | - subnets: 24 | - cloud_properties: 25 | subnet: "subnet-2" 26 | range: 10.1.20.0/24 27 | ` 28 | 29 | var _ = Describe("manifests", func() { 30 | var ( 31 | manifestsDirectory string 32 | err error 33 | ) 34 | 35 | BeforeEach(func() { 36 | manifestsDirectory, err = ioutil.TempDir("", "") 37 | Expect(err).NotTo(HaveOccurred()) 38 | }) 39 | 40 | Describe("ReadManifest", func() { 41 | It("returns the manifest as a map", func() { 42 | writeManifest(manifestsDirectory, "manifest.yml") 43 | 44 | manifestMap, err := manifests.ReadManifest(filepath.Join(manifestsDirectory, "manifest.yml")) 45 | Expect(err).NotTo(HaveOccurred()) 46 | Expect(manifestMap["director_uuid"]).To(Equal("BOSH-DIRECTOR-UUID")) 47 | }) 48 | 49 | Context("failure cases", func() { 50 | It("returns an error when given invalid yaml", func() { 51 | manifestFile := filepath.Join(manifestsDirectory, "invalid_manifest.yml") 52 | err := ioutil.WriteFile(manifestFile, []byte("not: valid: yaml:"), os.ModePerm) 53 | Expect(err).NotTo(HaveOccurred()) 54 | 55 | _, err = manifests.ReadManifest(manifestFile) 56 | Expect(err.Error()).To(ContainSubstring("mapping values are not allowed in this context")) 57 | }) 58 | 59 | It("returns an error when the file doesn't exist", func() { 60 | _, err = manifests.ReadManifest("/nonexistent/file") 61 | Expect(err.Error()).To(ContainSubstring("no such file or directory")) 62 | }) 63 | }) 64 | }) 65 | 66 | Describe("ReadNetworksFromManifest", func() { 67 | It("reads the network information from the given manifest", func() { 68 | writeManifestWithBody(manifestsDirectory, "manifest-with-subnets.yml", manifestWithSubnets) 69 | 70 | networks, err := manifests.ReadNetworksFromManifest(filepath.Join(manifestsDirectory, "manifest-with-subnets.yml")) 71 | Expect(err).NotTo(HaveOccurred()) 72 | 73 | Expect(networks[0].Subnets[0].CloudProperties.Subnet).To(Equal("subnet-1")) 74 | Expect(networks[0].Subnets[0].Range).To(Equal("10.0.20.0/24")) 75 | 76 | Expect(networks[1].Subnets[0].CloudProperties.Subnet).To(Equal("subnet-2")) 77 | Expect(networks[1].Subnets[0].Range).To(Equal("10.1.20.0/24")) 78 | }) 79 | 80 | Context("failure cases", func() { 81 | It("returns an error when given invalid yaml", func() { 82 | manifestFile := filepath.Join(manifestsDirectory, "invalid_manifest.yml") 83 | err := ioutil.WriteFile(manifestFile, []byte("not: valid: yaml:"), os.ModePerm) 84 | Expect(err).NotTo(HaveOccurred()) 85 | 86 | _, err = manifests.ReadNetworksFromManifest(manifestFile) 87 | Expect(err.Error()).To(ContainSubstring("mapping values are not allowed in this context")) 88 | }) 89 | 90 | It("returns an error when the file doesn't exist", func() { 91 | _, err = manifests.ReadNetworksFromManifest("/nonexistent/file") 92 | Expect(err.Error()).To(ContainSubstring("no such file or directory")) 93 | }) 94 | }) 95 | }) 96 | 97 | }) 98 | 99 | func writeManifest(directory string, filename string) { 100 | manifest := []byte("director_uuid: BOSH-DIRECTOR-UUID") 101 | err := ioutil.WriteFile(filepath.Join(directory, filename), manifest, os.ModePerm) 102 | Expect(err).NotTo(HaveOccurred()) 103 | } 104 | 105 | func writeManifestWithBody(directory string, filename string, body string) { 106 | err := ioutil.WriteFile(filepath.Join(directory, filename), []byte(body), os.ModePerm) 107 | Expect(err).NotTo(HaveOccurred()) 108 | } 109 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/subnetchecker/init_test.go: -------------------------------------------------------------------------------- 1 | package subnetchecker_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestSubnetChecker(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "subnetchecker") 13 | } 14 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/subnetchecker/subnetchecker.go: -------------------------------------------------------------------------------- 1 | package subnetchecker 2 | 3 | import ( 4 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 5 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/manifests" 6 | ) 7 | 8 | type SubnetChecker struct { 9 | awsClient AWSClient 10 | } 11 | 12 | type AWSClient interface { 13 | FetchSubnets(session clients.Session, subnetIds []string) ([]clients.Subnet, error) 14 | Session() (clients.Session, error) 15 | } 16 | 17 | func (s SubnetChecker) CheckSubnets(manifestFilename string) (bool, error) { 18 | networks, err := manifests.ReadNetworksFromManifest(manifestFilename) 19 | if err != nil { 20 | return false, err 21 | } 22 | 23 | manifestSubnetMap := mapSubnets(subnetsFromManifestNetworks(networks)) 24 | 25 | session, err := s.awsClient.Session() 26 | if err != nil { 27 | return false, err 28 | } 29 | 30 | awsSubnets, err := s.awsClient.FetchSubnets(session, subnetIdsFromSubnets(manifestSubnetMap)) 31 | if err != nil { 32 | return false, err 33 | } 34 | awsSubnetMap := mapSubnets(awsSubnets) 35 | 36 | for id, manifestSubnet := range manifestSubnetMap { 37 | awsSubnet, ok := awsSubnetMap[id] 38 | if !ok { 39 | return false, nil 40 | } 41 | if awsSubnet.CIDRBlock != manifestSubnet.CIDRBlock { 42 | return false, nil 43 | } 44 | } 45 | 46 | return true, nil 47 | } 48 | 49 | func NewSubnetChecker(awsClient AWSClient) SubnetChecker { 50 | return SubnetChecker{ 51 | awsClient: awsClient, 52 | } 53 | } 54 | 55 | func subnetsFromManifestNetworks(networks []manifests.Network) []clients.Subnet { 56 | var returnedSubnets []clients.Subnet 57 | for _, network := range networks { 58 | for _, subnet := range network.Subnets { 59 | returnedSubnets = append( 60 | returnedSubnets, 61 | clients.Subnet{ 62 | SubnetID: subnet.CloudProperties.Subnet, 63 | CIDRBlock: subnet.Range, 64 | }, 65 | ) 66 | } 67 | } 68 | return returnedSubnets 69 | } 70 | 71 | func subnetIdsFromSubnets(subnets map[string]clients.Subnet) []string { 72 | var subnetIds []string 73 | for subnetId, _ := range subnets { 74 | subnetIds = append(subnetIds, subnetId) 75 | } 76 | return subnetIds 77 | } 78 | 79 | func mapSubnets(subnets []clients.Subnet) map[string]clients.Subnet { 80 | subnetMap := make(map[string]clients.Subnet) 81 | for _, subnet := range subnets { 82 | subnetMap[subnet.SubnetID] = subnet 83 | } 84 | return subnetMap 85 | } 86 | -------------------------------------------------------------------------------- /scripts/ci/deploy-aws-manifests/subnetchecker/subnetchecker_test.go: -------------------------------------------------------------------------------- 1 | package subnetchecker_test 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/clients" 10 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/fakes" 11 | "github.com/cloudfoundry/mega-ci/scripts/ci/deploy-aws-manifests/subnetchecker" 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | var _ = Describe("SubnetChecker", func() { 17 | var ( 18 | manifestsDirectory string 19 | fakeAWS *fakes.AWS 20 | subnetChecker subnetchecker.SubnetChecker 21 | ) 22 | 23 | BeforeEach(func() { 24 | fakeAWS = new(fakes.AWS) 25 | 26 | var err error 27 | manifestsDirectory, err = ioutil.TempDir("", "") 28 | Expect(err).NotTo(HaveOccurred()) 29 | subnetChecker = subnetchecker.NewSubnetChecker(fakeAWS) 30 | }) 31 | 32 | It("returns true if all subnets in manifest exist on AWS", func() { 33 | const manifestWithSubnets1And2 = `--- 34 | director_uuid: BOSH-DIRECTOR-UUID 35 | 36 | name: multi-az-ssl 37 | 38 | networks: 39 | - subnets: 40 | - cloud_properties: 41 | subnet: "subnet-1" 42 | range: 10.0.20.0/24 43 | - subnets: 44 | - cloud_properties: 45 | subnet: "subnet-2" 46 | range: 10.1.20.0/24 47 | ` 48 | var awsSubnetsContainingSubnets1And2 = []clients.Subnet{ 49 | { 50 | SubnetID: "subnet-1", 51 | CIDRBlock: "10.0.20.0/24", 52 | }, 53 | { 54 | SubnetID: "subnet-2", 55 | CIDRBlock: "10.1.20.0/24", 56 | }, 57 | } 58 | 59 | writeManifestWithBody(manifestsDirectory, "manifest.yml", manifestWithSubnets1And2) 60 | fakeAWS.FetchSubnetsCall.Returns.Subnets = awsSubnetsContainingSubnets1And2 61 | 62 | hasSubnets, err := subnetChecker.CheckSubnets(filepath.Join(manifestsDirectory, "manifest.yml")) 63 | 64 | Expect(err).NotTo(HaveOccurred()) 65 | Expect(fakeAWS.FetchSubnetsCall.Receives.SubnetIds).To(ConsistOf([]string{"subnet-1", "subnet-2"})) 66 | Expect(hasSubnets).To(BeTrue()) 67 | }) 68 | 69 | It("returns false if some subnet in the manifest does not exist on AWS", func() { 70 | const manifestWithSubnets1And2 = `--- 71 | director_uuid: BOSH-DIRECTOR-UUID 72 | 73 | name: multi-az-ssl 74 | 75 | networks: 76 | - subnets: 77 | - cloud_properties: 78 | subnet: "subnet-1" 79 | range: 10.0.20.0/24 80 | - subnets: 81 | - cloud_properties: 82 | subnet: "subnet-2" 83 | range: 10.1.20.0/24 84 | ` 85 | var awsSubnetsMissingSubnet2 = []clients.Subnet{ 86 | { 87 | SubnetID: "subnet-1", 88 | CIDRBlock: "10.0.20.0/24", 89 | }, 90 | } 91 | 92 | writeManifestWithBody(manifestsDirectory, "manifest.yml", manifestWithSubnets1And2) 93 | fakeAWS.FetchSubnetsCall.Returns.Subnets = awsSubnetsMissingSubnet2 94 | 95 | hasSubnets, err := subnetChecker.CheckSubnets(filepath.Join(manifestsDirectory, "manifest.yml")) 96 | 97 | Expect(err).NotTo(HaveOccurred()) 98 | Expect(hasSubnets).To(BeFalse()) 99 | Expect(fakeAWS.FetchSubnetsCall.Receives.SubnetIds).To(ConsistOf([]string{"subnet-1", "subnet-2"})) 100 | }) 101 | 102 | It("returns false if some subnet range in the manifest does not match the subnet range in AWS for the same id", func() { 103 | const manifestWithSubnets1And2 = `--- 104 | director_uuid: BOSH-DIRECTOR-UUID 105 | 106 | name: multi-az-ssl 107 | 108 | networks: 109 | - subnets: 110 | - cloud_properties: 111 | subnet: "subnet-1" 112 | range: 10.0.20.0/24 113 | - subnets: 114 | - cloud_properties: 115 | subnet: "subnet-2" 116 | range: 10.1.20.0/24 117 | ` 118 | var awsSubnetsMissingSubnet2 = []clients.Subnet{ 119 | { 120 | SubnetID: "subnet-1", 121 | CIDRBlock: "10.0.20.0/24", 122 | }, 123 | { 124 | SubnetID: "subnet-2", 125 | CIDRBlock: "10.3.20.0/24", 126 | }, 127 | } 128 | 129 | writeManifestWithBody(manifestsDirectory, "manifest.yml", manifestWithSubnets1And2) 130 | fakeAWS.FetchSubnetsCall.Returns.Subnets = awsSubnetsMissingSubnet2 131 | 132 | hasSubnets, err := subnetChecker.CheckSubnets(filepath.Join(manifestsDirectory, "manifest.yml")) 133 | 134 | Expect(err).NotTo(HaveOccurred()) 135 | Expect(hasSubnets).To(BeFalse()) 136 | Expect(fakeAWS.FetchSubnetsCall.Receives.SubnetIds).To(ConsistOf([]string{"subnet-1", "subnet-2"})) 137 | }) 138 | 139 | Context("failure cases", func() { 140 | 141 | It("returns an error when the manifest is not valid yaml", func() { 142 | writeManifestWithBody(manifestsDirectory, "invalid_manifest.yml", "not: valid: yaml:") 143 | _, err := subnetChecker.CheckSubnets(filepath.Join(manifestsDirectory, "invalid_manifest.yml")) 144 | Expect(err.Error()).To(ContainSubstring("mapping values are not allowed in this context")) 145 | }) 146 | 147 | It("returns an error when aws client cannot get a session", func() { 148 | writeManifest(manifestsDirectory, "manifest.yml") 149 | fakeAWS.SessionCall.Returns.Error = errors.New("no aws session") 150 | 151 | _, err := subnetChecker.CheckSubnets(filepath.Join(manifestsDirectory, "manifest.yml")) 152 | Expect(err).NotTo(BeNil()) 153 | Expect(err.Error()).To(ContainSubstring("no aws session")) 154 | }) 155 | 156 | It("returns an error when it FetchSubnets fails", func() { 157 | const manifestWithSubnets1And2 = `--- 158 | director_uuid: BOSH-DIRECTOR-UUID 159 | 160 | name: multi-az-ssl 161 | 162 | networks: 163 | - subnets: 164 | - cloud_properties: 165 | subnet: "subnet-1" 166 | range: 10.0.20.0/24 167 | - subnets: 168 | - cloud_properties: 169 | subnet: "subnet-2" 170 | range: 10.1.20.0/24 171 | ` 172 | writeManifestWithBody(manifestsDirectory, "manifest.yml", manifestWithSubnets1And2) 173 | fakeAWS.FetchSubnetsCall.Returns.Error = errors.New("something bad happened") 174 | 175 | _, err := subnetChecker.CheckSubnets(filepath.Join(manifestsDirectory, "manifest.yml")) 176 | Expect(err).NotTo(BeNil()) 177 | Expect(err.Error()).To(ContainSubstring("something bad happened")) 178 | }) 179 | }) 180 | }) 181 | 182 | func writeManifest(directory string, filename string) { 183 | writeManifestWithBody(directory, filename, "director_uuid: BOSH-DIRECTOR-UUID\nname: a-deployment-name") 184 | } 185 | 186 | func writeManifestWithBody(directory string, filename string, body string) { 187 | err := ioutil.WriteFile(filepath.Join(directory, filename), []byte(body), os.ModePerm) 188 | Expect(err).NotTo(HaveOccurred()) 189 | } 190 | -------------------------------------------------------------------------------- /scripts/ci/deploy-bosh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ./mega-ci/scripts/setup_aws_bosh_for_concourse "mega-ci-env" 4 | 5 | git config --global user.name "CF MEGA BOT" 6 | git config --global user.email "cf-mega@pivotal.io" 7 | 8 | pushd mega-ci-env > /dev/null 9 | git checkout master 10 | git add artifacts/deployments/bosh-state.json 11 | git add artifacts/deployments/bosh.yml 12 | set +e 13 | git commit -m "Update bosh-state.json and bosh.yml" 14 | set -e 15 | popd > /dev/null 16 | 17 | shopt -s dotglob 18 | cp -R mega-ci-env/* mega-ci-env-out 19 | -------------------------------------------------------------------------------- /scripts/ci/deploy-bosh-lite-manifests: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | function setup_env() { 4 | set +x 5 | export BUILD_DIR="${PWD}" 6 | export BOSH_LITE_PRIVATE_KEY="${BUILD_DIR}/bosh-lite.pem" 7 | export BOSH_USER="admin" 8 | export BOSH_PASSWORD="admin" 9 | 10 | echo "${BOSH_LITE_PRIVATE_KEY_CONTENTS}" > "${BOSH_LITE_PRIVATE_KEY}" 11 | set -x 12 | } 13 | 14 | function deploy_boshlite() { 15 | pushd "${BUILD_DIR}/bosh-lite" > /dev/null 16 | vagrant up --provider=aws 17 | export BOSH_DIRECTOR=$(vagrant ssh-config 2>/dev/null | grep HostName | awk '{print $2}') 18 | 19 | target_director 20 | popd > /dev/null 21 | 22 | update_cloud_config 23 | } 24 | 25 | function update_cloud_config() { 26 | set +e 27 | pushd "${BUILD_DIR}/release" > /dev/null 28 | /opt/rubies/ruby-2.2.4/bin/bosh update cloud-config "${CLOUD_CONFIG}" 29 | popd > /dev/null 30 | set -e 31 | } 32 | 33 | function target_director() { 34 | set +e 35 | local rc 36 | rc=1 37 | 38 | while [ "${rc}" -ne "0" ]; do 39 | sleep 10 40 | curl -k -s "https://${BOSH_DIRECTOR}:25555/info" > /dev/null 41 | rc=$? 42 | done 43 | set -e 44 | 45 | /opt/rubies/ruby-2.2.4/bin/bosh target "${BOSH_DIRECTOR}" 46 | } 47 | 48 | function upload_stemcell() { 49 | /opt/rubies/ruby-2.2.4/bin/bosh upload stemcell "${BUILD_DIR}/bosh-lite-stemcell/stemcell.tgz" 50 | } 51 | 52 | function upload_dependencies() { 53 | for dependency in ${DEPENDENCIES}; do 54 | /opt/rubies/ruby-2.2.4/bin/bosh -n upload release https://bosh.io/d/github.com/${dependency} 55 | done 56 | } 57 | 58 | function create_and_upload_release() { 59 | pushd "${BUILD_DIR}/release" > /dev/null 60 | /opt/rubies/ruby-2.2.4/bin/bosh -n create release --force 61 | /opt/rubies/ruby-2.2.4/bin/bosh -n upload release 62 | popd > /dev/null 63 | } 64 | 65 | function deploy_manifests() { 66 | pushd "${BUILD_DIR}/release" > /dev/null 67 | for manifest in ${MANIFESTS}; do 68 | sed -i -e "s/BOSH-DIRECTOR-UUID/$(/opt/rubies/ruby-2.2.4/bin/bosh status --uuid)/g" "${manifest}" 69 | /opt/rubies/ruby-2.2.4/bin/bosh -d "${manifest}" -n deploy 70 | /opt/rubies/ruby-2.2.4/bin/bosh -n delete deployment $(/opt/rubies/ruby-2.2.4/bin/bosh deployments | grep bosh-warden-boshlite | awk '{print $2}') 71 | done 72 | popd > /dev/null 73 | } 74 | 75 | function destroy_boshlite() { 76 | pushd "${BUILD_DIR}/bosh-lite" > /dev/null 77 | vagrant destroy -f 78 | popd > /dev/null 79 | } 80 | 81 | function main() { 82 | setup_env 83 | deploy_boshlite 84 | upload_stemcell 85 | upload_dependencies 86 | create_and_upload_release 87 | deploy_manifests 88 | } 89 | 90 | trap destroy_boshlite EXIT 91 | 92 | main 93 | -------------------------------------------------------------------------------- /scripts/ci/deploy-bosh-lite-manifests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/vagrant 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: bosh-lite 9 | - name: release 10 | - name: bosh-lite-stemcell 11 | 12 | params: 13 | BOSH_AWS_ACCESS_KEY_ID: 14 | BOSH_AWS_SECRET_ACCESS_KEY: 15 | BOSH_LITE_SECURITY_GROUP: 16 | BOSH_LITE_SUBNET_ID: 17 | BOSH_LITE_NAME: 18 | BOSH_LITE_KEYPAIR: 19 | BOSH_LITE_PRIVATE_KEY_CONTENTS: 20 | CLOUD_CONFIG: 21 | MANIFESTS: 22 | DEPENDENCIES: 23 | 24 | run: 25 | path: mega-ci/scripts/ci/deploy-bosh-lite-manifests 26 | 27 | -------------------------------------------------------------------------------- /scripts/ci/deploy-bosh.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: mega-ci-env 9 | 10 | outputs: 11 | - name: mega-ci-env-out 12 | 13 | run: 14 | path: mega-ci/scripts/ci/deploy-bosh 15 | 16 | params: 17 | -------------------------------------------------------------------------------- /scripts/ci/deploy-consul-cf/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | preflight_check() { 6 | set +x 7 | test -n "${BOSH_DIRECTOR}" 8 | test -n "${BOSH_USER}" 9 | test -n "${BOSH_PASSWORD}" 10 | set -x 11 | } 12 | 13 | deploy() { 14 | bosh \ 15 | -n \ 16 | -t "${1}" \ 17 | -d "${2}" \ 18 | deploy 19 | } 20 | 21 | generate_releases_stub() { 22 | local build_dir 23 | build_dir="${1}" 24 | 25 | cat < /dev/null 64 | bosh -t "${BOSH_DIRECTOR}" upload stemcell stemcell.tgz --skip-if-exists 65 | popd > /dev/null 66 | } 67 | 68 | main() { 69 | local root_dir 70 | root_dir=$PWD 71 | 72 | preflight_check 73 | 74 | mkdir stubs 75 | 76 | upload_stemcell 77 | 78 | pushd stubs > /dev/null 79 | generate_releases_stub ${root_dir} > releases.yml 80 | generate_stemcell_stub > stemcells.yml 81 | generate_job_templates_stub > job_templates.yml 82 | popd > /dev/null 83 | 84 | "${root_dir}/cf-release/scripts/generate_deployment_manifest" \ 85 | "aws" \ 86 | "${root_dir}/stubs/releases.yml" \ 87 | "${root_dir}/stubs/stemcells.yml" \ 88 | "${root_dir}/stubs/job_templates.yml" \ 89 | "${root_dir}/consul-cf-env/stubs/director-uuid.yml" \ 90 | "${root_dir}/consul-cf-env/stubs/cf/diego.yml" \ 91 | "${root_dir}/consul-cf-env/stubs/cf/properties.yml" \ 92 | "${root_dir}/consul-cf-env/stubs/cf/stub.yml" \ 93 | > "${root_dir}/cf.yml" 94 | 95 | deploy \ 96 | "${BOSH_DIRECTOR}" \ 97 | "${root_dir}/cf.yml" 98 | } 99 | 100 | if [ "$(basename "${0}")" = "task" ]; then 101 | main 102 | fi 103 | -------------------------------------------------------------------------------- /scripts/ci/deploy-consul-cf/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: cf-release 8 | - name: consul-cf-env 9 | - name: consul-release 10 | - name: mega-ci 11 | - name: stemcell 12 | 13 | run: 14 | path: mega-ci/scripts/ci/deploy-consul-cf/task 15 | 16 | params: 17 | BOSH_DIRECTOR: 18 | BOSH_USER: 19 | BOSH_PASSWORD: 20 | -------------------------------------------------------------------------------- /scripts/ci/deploy-consul-diego/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | function deploy_diego() { 6 | bosh -t $BOSH_DIRECTOR download manifest consul-cf-deployment cf.yml 7 | 8 | pushd diego-release > /dev/null 9 | ./scripts/generate-deployment-manifest \ 10 | -c $ROOT/cf.yml \ 11 | -i $ROOT/consul-cf-env/stubs/diego/iaas-settings.yml \ 12 | -p $ROOT/consul-cf-env/stubs/diego/property-overrides.yml \ 13 | -n $ROOT/consul-cf-env/stubs/diego/instance-count-overrides.yml \ 14 | -v $ROOT/consul-cf-env/stubs/diego/release-versions.yml \ 15 | -g \ 16 | > $ROOT/diego.yml 17 | popd > /dev/null 18 | 19 | bosh -n \ 20 | -d diego.yml \ 21 | -t ${BOSH_DIRECTOR} \ 22 | deploy 23 | } 24 | 25 | function upload_release() { 26 | local release 27 | release=${1} 28 | bosh -t ${BOSH_DIRECTOR} upload release https://bosh.io/d/github.com/${release} 29 | } 30 | 31 | function upload_stemcell() { 32 | pushd "${ROOT}/stemcell" > /dev/null 33 | bosh -t "${BOSH_DIRECTOR}" upload stemcell stemcell.tgz --skip-if-exists 34 | popd > /dev/null 35 | } 36 | 37 | function main() { 38 | upload_stemcell 39 | 40 | upload_release "cloudfoundry/cflinuxfs2-rootfs-release" 41 | upload_release "cloudfoundry/diego-release" 42 | upload_release "cloudfoundry/garden-runc-release" 43 | upload_release "cloudfoundry-incubator/etcd-release" 44 | 45 | deploy_diego 46 | } 47 | 48 | main 49 | -------------------------------------------------------------------------------- /scripts/ci/deploy-consul-diego/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: consul-cf-env 8 | - name: diego-release 9 | - name: mega-ci 10 | - name: stemcell 11 | 12 | run: 13 | path: mega-ci/scripts/ci/deploy-consul-diego/task 14 | 15 | params: 16 | BOSH_DIRECTOR: 17 | BOSH_USER: 18 | BOSH_PASSWORD: 19 | -------------------------------------------------------------------------------- /scripts/ci/deploy-etcd-cf/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | preflight_check() { 6 | set +x 7 | test -n "${BOSH_ENVIRONMENT}" 8 | test -n "${BOSH_CLIENT}" 9 | test -n "${BOSH_CLIENT_SECRET}" 10 | test -n "${BOSH_CA_CERT}" 11 | set -x 12 | } 13 | 14 | function install_bosh_cli() { 15 | pushd "${ROOT}" > /dev/null 16 | wget https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-0.0.147-linux-amd64 17 | mv bosh-cli-0.0.147-linux-amd64 /usr/local/bin/boshv2 18 | chmod +x /usr/local/bin/boshv2 19 | popd > /dev/null 20 | } 21 | 22 | deploy() { 23 | boshv2 \ 24 | -n \ 25 | -d etcd-cf-deployment \ 26 | interpolate "${@}" > final-cf.yml 27 | 28 | 29 | set +x 30 | export BOSH_USER=$BOSH_CLIENT 31 | export BOSH_PASSWORD=$BOSH_CLIENT_SECRET 32 | set -x 33 | 34 | /opt/rubies/ruby-2.2.4/bin/bosh -n -t ${BOSH_ENVIRONMENT} -d final-cf.yml deploy 35 | } 36 | 37 | generate_releases_stub() { 38 | local build_dir 39 | build_dir="${1}" 40 | 41 | cat < /dev/null 94 | boshv2 upload-stemcell stemcell.tgz 95 | popd > /dev/null 96 | } 97 | 98 | main() { 99 | local root_dir 100 | root_dir=$PWD 101 | 102 | preflight_check 103 | 104 | mkdir stubs 105 | 106 | install_bosh_cli 107 | upload_stemcell 108 | 109 | pushd stubs > /dev/null 110 | generate_releases_stub ${root_dir} > releases.yml 111 | generate_stemcell_stub > stemcells.yml 112 | generate_job_templates_stub > job_templates.yml 113 | popd > /dev/null 114 | 115 | "${root_dir}/cf-release/scripts/generate_deployment_manifest" \ 116 | "aws" \ 117 | "${root_dir}/stubs/releases.yml" \ 118 | "${root_dir}/stubs/stemcells.yml" \ 119 | "${root_dir}/stubs/job_templates.yml" \ 120 | "${root_dir}/etcd-cf-env/stubs/director-uuid.yml" \ 121 | "${root_dir}/etcd-cf-env/stubs/cf/diego.yml" \ 122 | "${root_dir}/etcd-cf-env/stubs/cf/properties.yml" \ 123 | "${root_dir}/etcd-cf-env/stubs/cf/stub.yml" \ 124 | > "${root_dir}/cf.yml" 125 | 126 | deploy \ 127 | "${root_dir}/cf.yml" \ 128 | -o "${root_dir}/etcd-cf-env/stubs/cf/non_tls_ops_file.yml" 129 | } 130 | 131 | if [ "$(basename "${0}")" = "task" ]; then 132 | main 133 | fi 134 | -------------------------------------------------------------------------------- /scripts/ci/deploy-etcd-cf/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: cf-release 8 | - name: etcd-cf-env 9 | - name: etcd-release 10 | - name: mega-ci 11 | - name: stemcell 12 | 13 | run: 14 | path: mega-ci/scripts/ci/deploy-etcd-cf/task 15 | 16 | params: 17 | BOSH_ENVIRONMENT: 18 | BOSH_CLIENT: 19 | BOSH_CLIENT_SECRET: 20 | BOSH_CA_CERT: 21 | -------------------------------------------------------------------------------- /scripts/ci/deploy-etcd-diego/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | function deploy_diego() { 6 | /opt/rubies/ruby-2.2.4/bin/bosh -t $BOSH_DIRECTOR download manifest etcd-cf-deployment cf.yml 7 | 8 | pushd diego-release > /dev/null 9 | ./scripts/generate-deployment-manifest \ 10 | -c $ROOT/cf.yml \ 11 | -i $ROOT/etcd-cf-env/stubs/diego/iaas-settings.yml \ 12 | -p $ROOT/etcd-cf-env/stubs/diego/property-overrides.yml \ 13 | -s $ROOT/etcd-cf-env/stubs/diego/diego-sql.yml \ 14 | -n $ROOT/etcd-cf-env/stubs/diego/instance-count-overrides.yml \ 15 | -v $ROOT/etcd-cf-env/stubs/diego/release-versions.yml \ 16 | -g \ 17 | > $ROOT/diego.yml 18 | popd > /dev/null 19 | 20 | /opt/rubies/ruby-2.2.4/bin/bosh -n \ 21 | -d diego.yml \ 22 | -t ${BOSH_DIRECTOR} \ 23 | deploy 24 | } 25 | 26 | function upload_release() { 27 | local release 28 | release=${1} 29 | /opt/rubies/ruby-2.2.4/bin/bosh -t ${BOSH_DIRECTOR} upload release https://bosh.io/d/github.com/${release} 30 | } 31 | 32 | function upload_stemcell() { 33 | pushd "${ROOT}/stemcell" > /dev/null 34 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload stemcell stemcell.tgz --skip-if-exists 35 | popd > /dev/null 36 | } 37 | 38 | function main() { 39 | upload_stemcell 40 | 41 | upload_release "cloudfoundry/cflinuxfs2-rootfs-release" 42 | upload_release "cloudfoundry/diego-release" 43 | upload_release "cloudfoundry/garden-runc-release" 44 | 45 | deploy_diego 46 | } 47 | 48 | main 49 | -------------------------------------------------------------------------------- /scripts/ci/deploy-etcd-diego/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: etcd-cf-env 8 | - name: diego-release 9 | - name: mega-ci 10 | - name: stemcell 11 | 12 | run: 13 | path: mega-ci/scripts/ci/deploy-etcd-diego/task 14 | 15 | params: 16 | BOSH_DIRECTOR: 17 | BOSH_USER: 18 | BOSH_PASSWORD: 19 | -------------------------------------------------------------------------------- /scripts/ci/gomegamatchers/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | mkdir -p "${GOPATH}/src/github.com/pivotal-cf-experimental" 6 | 7 | pushd "${GOPATH}/src/github.com/pivotal-cf-experimental" > /dev/null 8 | ln -s "${ROOT}/gomegamatchers" 9 | ./gomegamatchers/scripts/test 10 | popd > /dev/null 11 | -------------------------------------------------------------------------------- /scripts/ci/gomegamatchers/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/golang 5 | 6 | inputs: 7 | - name: gomegamatchers 8 | - name: mega-ci 9 | 10 | run: 11 | path: mega-ci/scripts/ci/gomegamatchers/test 12 | -------------------------------------------------------------------------------- /scripts/ci/merge-master-into-develop/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | MERGED_REPO="${PWD}/${MERGED_REPO:?"MERGED_REPO required"}" 4 | MASTER_BRANCH="${MASTER_BRANCH:-master}" 5 | 6 | # Cannot set -u before sourcing .bashrc because of all 7 | # the unbound variables in things beyond our control. 8 | set +u 9 | source ~/.bashrc 10 | set -u 11 | 12 | pushd release-repo > /dev/null 13 | git config user.name "CF MEGA BOT" 14 | git config user.email "cf-mega@pivotal.io" 15 | 16 | git remote add -f master-repo ../release-repo-master 17 | git merge --no-edit "master-repo/${MASTER_BRANCH}" 18 | 19 | git status 20 | git show --color | cat 21 | popd > /dev/null 22 | 23 | shopt -s dotglob 24 | cp -R release-repo/* $MERGED_REPO 25 | -------------------------------------------------------------------------------- /scripts/ci/merge-master-into-develop/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/minimal 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: release-repo 9 | - name: release-repo-master 10 | 11 | outputs: 12 | - name: final-release-repo 13 | 14 | run: 15 | path: mega-ci/scripts/ci/merge-master-into-develop/task 16 | 17 | params: 18 | MASTER_BRANCH: 19 | MERGED_REPO: final-release-repo 20 | -------------------------------------------------------------------------------- /scripts/ci/run-bosh-cleanup: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | set +u 4 | source ~/.bashrc 5 | set -u 6 | 7 | if [ -n "$LOGS_DIR" ]; then 8 | mkdir $LOGS_DIR 9 | fi 10 | 11 | bosh -t $BOSH_DIRECTOR cleanup --all 12 | -------------------------------------------------------------------------------- /scripts/ci/run-bosh-cleanup.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | 9 | run: 10 | path: mega-ci/scripts/ci/run-bosh-cleanup 11 | 12 | params: 13 | BOSH_DIRECTOR: 14 | BOSH_USER: 15 | BOSH_PASSWORD: 16 | LOGS_DIR: 17 | -------------------------------------------------------------------------------- /scripts/ci/run-bosh-command: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | set +u 4 | source ~/.bashrc 5 | set -u 6 | 7 | if [ -n "$LOGS_DIR" ]; then 8 | mkdir $LOGS_DIR 9 | fi 10 | 11 | bosh -t $BOSH_DIRECTOR download manifest $DEPLOYMENT_NAME manifest.yml 12 | 13 | bosh -n --color -t $BOSH_DIRECTOR -d manifest.yml $COMMAND 14 | -------------------------------------------------------------------------------- /scripts/ci/run-bosh-command.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | 9 | run: 10 | path: mega-ci/scripts/ci/run-bosh-command 11 | 12 | params: 13 | BOSH_DIRECTOR: 14 | BOSH_USER: 15 | BOSH_PASSWORD: 16 | COMMAND: 17 | DEPLOYMENT_NAME: 18 | LOGS_DIR: 19 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/fixtures/consul_compilation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | director_uuid: REPLACE_ME_DIRECTOR_UUID 3 | 4 | name: compilation 5 | 6 | releases: 7 | - name: consul 8 | version: "CONSUL_RELEASE_VERSION" 9 | - name: turbulence 10 | version: "TURBULENCE_RELEASE_VERSION" 11 | - name: bosh-aws-cpi 12 | version: "CPI_RELEASE_VERSION" 13 | 14 | update: 15 | canaries: 1 16 | max_in_flight: 1 17 | serial: false 18 | canary_watch_time: 1000-60000 19 | update_watch_time: 1000-60000 20 | 21 | stemcells: 22 | - alias: trusty 23 | os: ubuntu-trusty 24 | version: "STEMCELL_VERSION" 25 | 26 | instance_groups: [] 27 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/fixtures/example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: consats 3 | 4 | director_uuid: BOSH_DIRECTOR_UUID 5 | 6 | releases: 7 | - name: consul 8 | version: CONSUL_RELEASE_VERSION 9 | 10 | stemcells: 11 | - alias: default 12 | os: ubuntu-trusty 13 | version: STEMCELL_VERSION 14 | 15 | instance_groups: 16 | - name: acceptance-tests 17 | instances: 1 18 | lifecycle: errand 19 | azs: [BOSH_ERRAND_CLOUD_CONFIG_NETWORK_AZ] 20 | vm_type: m3.medium 21 | vm_extensions: [10GB_ephemeral_disk] 22 | stemcell: default 23 | networks: 24 | - name: BOSH_ERRAND_CLOUD_CONFIG_NETWORK_NAME 25 | static_ips: [BOSH_ERRAND_CLOUD_CONFIG_NETWORK_STATIC_IP] 26 | jobs: 27 | - name: acceptance-tests 28 | release: consul 29 | 30 | properties: 31 | consul: 32 | acceptance_tests: 33 | aws: 34 | access_key_id: AWS_ACCESS_KEY_ID 35 | secret_access_key: AWS_SECRET_ACCESS_KEY 36 | region: AWS_REGION 37 | default_key_name: AWS_DEFAULT_KEY_NAME 38 | default_security_groups: 39 | - AWS_SECURITY_GROUP_NAME 40 | subnets: 41 | - id: AWS_SUBNET_ID 42 | range: AWS_SUBNET_RANGE 43 | az: AWS_SUBNET_AZ 44 | security_group: AWS_SUBNET_SECURITY_GROUP 45 | cloud_config_subnets: 46 | - id: AWS_CLOUD_CONFIG_SUBNET_ID 47 | range: AWS_CLOUD_CONFIG_SUBNET_RANGE 48 | az: AWS_CLOUD_CONFIG_SUBNET_AZ 49 | security_group: AWS_CLOUD_CONFIG_SUBNET_SECURITY_GROUP 50 | bosh: 51 | target: BOSH_DIRECTOR 52 | username: BOSH_USER 53 | password: BOSH_PASSWORD 54 | director_ca_cert: BOSH_DIRECTOR_CA_CERT 55 | errand: 56 | network: 57 | name: BOSH_ERRAND_CLOUD_CONFIG_NETWORK_NAME 58 | static_ip: BOSH_ERRAND_CLOUD_CONFIG_NETWORK_STATIC_IP 59 | az: BOSH_ERRAND_CLOUD_CONFIG_NETWORK_AZ 60 | default_persistent_disk_type: BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_PERSISTENT_DISK_TYPE 61 | default_vm_type: BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_VM_TYPE 62 | registry: 63 | username: REGISTRY_USERNAME 64 | password: REGISTRY_PASSWORD 65 | host: REGISTRY_HOST 66 | port: 25777 67 | parallel_nodes: 1 68 | consul_release_version: CONSUL_RELEASE_VERSION 69 | latest_consul_release_version: LATEST_CONSUL_RELEASE_VERSION 70 | 71 | update: 72 | serial: true 73 | canaries: 1 74 | max_in_flight: 1 75 | canary_watch_time: 1000-180000 76 | update_watch_time: 1000-180000 77 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/fixtures/expected.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: consats 3 | 4 | director_uuid: some-bosh-director-uuid 5 | 6 | releases: 7 | - name: consul 8 | version: some-consul-release-version 9 | 10 | stemcells: 11 | - alias: default 12 | os: ubuntu-trusty 13 | version: some-stemcell-version 14 | 15 | instance_groups: 16 | - name: acceptance-tests 17 | instances: 1 18 | lifecycle: errand 19 | azs: [some-errand-az] 20 | vm_type: m3.medium 21 | vm_extensions: [10GB_ephemeral_disk] 22 | stemcell: default 23 | networks: 24 | - name: some-errand-network-name 25 | static_ips: [some-errand-network-static-ip] 26 | jobs: 27 | - name: acceptance-tests 28 | release: consul 29 | 30 | properties: 31 | consul: 32 | acceptance_tests: 33 | aws: 34 | access_key_id: some-aws-access-key-id 35 | secret_access_key: some-aws-secret-access-key 36 | region: some-aws-region 37 | default_key_name: some-aws-default-key-name 38 | default_security_groups: 39 | - some-aws-security-group-name 40 | subnets: 41 | - id: some-subnet-1 42 | range: 10.0.4.0/24 43 | az: some-az-1 44 | security_group: some-security-group-1 45 | - id: some-subnet-2 46 | range: 10.0.5.0/24 47 | az: some-az-2 48 | security_group: some-security-group-2 49 | cloud_config_subnets: 50 | - id: some-cloud-config-subnet-1 51 | range: 10.0.6.0/24 52 | az: some-cloud-config-az-1 53 | security_group: some-cloud-config-security-group-1 54 | - id: some-cloud-config-subnet-2 55 | range: 10.0.7.0/24 56 | az: some-cloud-config-az-2 57 | security_group: some-cloud-config-security-group-2 58 | bosh: 59 | target: some-bosh-target 60 | username: some-bosh-username 61 | password: some-bosh-password 62 | director_ca_cert: some-bosh-director-ca-cert 63 | errand: 64 | network: 65 | name: some-errand-network-name 66 | static_ip: some-errand-network-static-ip 67 | az: some-errand-az 68 | default_persistent_disk_type: some-persistent-disk-type 69 | default_vm_type: some-vm-type 70 | registry: 71 | username: some-registry-username 72 | password: some-registry-password 73 | host: some-registry-host 74 | port: 25777 75 | parallel_nodes: 10 76 | consul_release_version: some-consul-release-version 77 | latest_consul_release_version: some-latest-consul-release-version 78 | enable_turbulence_tests: true 79 | windows_clients: true 80 | 81 | update: 82 | serial: true 83 | canaries: 1 84 | max_in_flight: 1 85 | canary_watch_time: 1000-180000 86 | update_watch_time: 1000-180000 87 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/fixtures/malformed.yml: -------------------------------------------------------------------------------- 1 | --- 2 | this is not valid YAML 3 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/generate_manifest.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "strconv" 9 | 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | type Network struct { 14 | Name string `yaml:"name"` 15 | StaticIPs []string `yaml:"static_ips"` 16 | } 17 | 18 | type Manifest struct { 19 | Name interface{} `yaml:"name"` 20 | DirectorUUID string `yaml:"director_uuid"` 21 | Releases []struct { 22 | Name string `yaml:"name"` 23 | Version string `yaml:"version"` 24 | } `yaml:"releases"` 25 | Stemcells []struct { 26 | Alias string `yaml:"alias"` 27 | OS string `yaml:"os"` 28 | Version string `yaml:"version"` 29 | } `yaml:"stemcells"` 30 | InstanceGroups []struct { 31 | Instances int `yaml:"instances"` 32 | Name string `yaml:"name"` 33 | Lifecycle string `yaml:"lifecycle"` 34 | VMExtensions []string `yaml:"vm_extensions"` 35 | VMType string `yaml:"vm_type"` 36 | Stemcell string `yaml:"stemcell"` 37 | AZs []string `yaml:"azs"` 38 | Networks []Network `yaml:"networks"` 39 | Jobs []struct { 40 | Name string `yaml:"name'` 41 | Release string `yaml:"release'` 42 | } `yaml:"jobs"` 43 | } `yaml:"instance_groups"` 44 | Properties struct { 45 | Consul struct { 46 | AcceptanceTests struct { 47 | AWS struct { 48 | AccessKeyID string `yaml:"access_key_id"` 49 | SecretAccessKey string `yaml:"secret_access_key"` 50 | Region string `yaml:"region"` 51 | DefaultKeyName interface{} `yaml:"default_key_name"` 52 | DefaultSecurityGroups []string `yaml:"default_security_groups"` 53 | Subnets []struct { 54 | ID string `yaml:"id"` 55 | Range string `yaml:"range"` 56 | AZ string `yaml:"az"` 57 | SecurityGroup string `yaml:"security_group"` 58 | } `yaml:"subnets"` 59 | CloudConfigSubnets []struct { 60 | ID string `yaml:"id"` 61 | Range string `yaml:"range"` 62 | AZ string `yaml:"az"` 63 | SecurityGroup string `yaml:"security_group"` 64 | } `yaml:"cloud_config_subnets"` 65 | } `yaml:"aws"` 66 | BOSH struct { 67 | Target string `yaml:"target"` 68 | Username string `yaml:"username"` 69 | Password string `yaml:"password"` 70 | DirectorCACert string `yaml:"director_ca_cert"` 71 | Errand struct { 72 | Network struct { 73 | Name string `yaml:"name"` 74 | StaticIP string `yaml:"static_ip"` 75 | AZ string `yaml:"az"` 76 | } `yaml:"network"` 77 | DefaultPersistentDiskType string `yaml:"default_persistent_disk_type"` 78 | DefaultVMType string `yaml:"default_vm_type"` 79 | } `yaml:"errand"` 80 | } `yaml:"bosh"` 81 | Registry struct { 82 | Username string `yaml:"username"` 83 | Password string `yaml:"password"` 84 | Host interface{} `yaml:"host"` 85 | Port interface{} `yaml:"port"` 86 | } `yaml:"registry"` 87 | ParallelNodes int `yaml:"parallel_nodes"` 88 | ConsulReleaseVersion string `yaml:"consul_release_version"` 89 | LatestConsulReleaseVersion string `yaml:"latest_consul_release_version"` 90 | EnableTurbulenceTests bool `yaml:"enable_turbulence_tests"` 91 | WindowsClients bool `yaml:"windows_clients"` 92 | } `yaml:"acceptance_tests"` 93 | } `yaml:"consul"` 94 | } `yaml:"properties"` 95 | Update interface{} `yaml:"update"` 96 | } 97 | 98 | func main() { 99 | output, err := Generate(os.Args[1]) 100 | if err != nil { 101 | fmt.Fprintln(os.Stderr, err) 102 | os.Exit(1) 103 | } 104 | 105 | fmt.Fprintln(os.Stdout, string(output)) 106 | } 107 | 108 | func Generate(exampleManifestFilePath string) ([]byte, error) { 109 | contents, err := ioutil.ReadFile(exampleManifestFilePath) 110 | if err != nil { 111 | return nil, err 112 | } 113 | 114 | var manifest Manifest 115 | err = yaml.Unmarshal(contents, &manifest) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | manifest.DirectorUUID = os.Getenv("BOSH_DIRECTOR_UUID") 121 | manifest.Releases[0].Version = os.Getenv("CONSUL_RELEASE_VERSION") 122 | manifest.Stemcells[0].Version = os.Getenv("STEMCELL_VERSION") 123 | manifest.InstanceGroups[0].AZs = []string{os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_NETWORK_AZ")} 124 | manifest.InstanceGroups[0].Networks = []Network{ 125 | { 126 | Name: os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_NETWORK_NAME"), 127 | StaticIPs: []string{os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_NETWORK_STATIC_IP")}, 128 | }, 129 | } 130 | manifest.Properties.Consul.AcceptanceTests.AWS.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID") 131 | manifest.Properties.Consul.AcceptanceTests.AWS.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY") 132 | manifest.Properties.Consul.AcceptanceTests.AWS.Region = os.Getenv("AWS_REGION") 133 | manifest.Properties.Consul.AcceptanceTests.AWS.DefaultSecurityGroups = []string{os.Getenv("AWS_SECURITY_GROUP_NAME")} 134 | manifest.Properties.Consul.AcceptanceTests.AWS.DefaultKeyName = os.Getenv("AWS_DEFAULT_KEY_NAME") 135 | 136 | manifest.Properties.Consul.AcceptanceTests.BOSH.Target = os.Getenv("BOSH_DIRECTOR") 137 | manifest.Properties.Consul.AcceptanceTests.BOSH.Username = os.Getenv("BOSH_USER") 138 | manifest.Properties.Consul.AcceptanceTests.BOSH.Password = os.Getenv("BOSH_PASSWORD") 139 | manifest.Properties.Consul.AcceptanceTests.BOSH.DirectorCACert = os.Getenv("BOSH_DIRECTOR_CA_CERT") 140 | manifest.Properties.Consul.AcceptanceTests.BOSH.Errand.Network.AZ = os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_NETWORK_AZ") 141 | manifest.Properties.Consul.AcceptanceTests.BOSH.Errand.Network.Name = os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_NETWORK_NAME") 142 | manifest.Properties.Consul.AcceptanceTests.BOSH.Errand.Network.StaticIP = os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_NETWORK_STATIC_IP") 143 | manifest.Properties.Consul.AcceptanceTests.BOSH.Errand.DefaultVMType = os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_VM_TYPE") 144 | manifest.Properties.Consul.AcceptanceTests.BOSH.Errand.DefaultPersistentDiskType = os.Getenv("BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_PERSISTENT_DISK_TYPE") 145 | 146 | manifest.Properties.Consul.AcceptanceTests.Registry.Host = os.Getenv("REGISTRY_HOST") 147 | manifest.Properties.Consul.AcceptanceTests.Registry.Username = os.Getenv("REGISTRY_USERNAME") 148 | manifest.Properties.Consul.AcceptanceTests.Registry.Password = os.Getenv("REGISTRY_PASSWORD") 149 | 150 | manifest.Properties.Consul.AcceptanceTests.ConsulReleaseVersion = os.Getenv("CONSUL_RELEASE_VERSION") 151 | manifest.Properties.Consul.AcceptanceTests.LatestConsulReleaseVersion = os.Getenv("LATEST_CONSUL_RELEASE_VERSION") 152 | manifest.Properties.Consul.AcceptanceTests.EnableTurbulenceTests = (os.Getenv("ENABLE_TURBULENCE_TESTS") == "true") 153 | manifest.Properties.Consul.AcceptanceTests.WindowsClients = (os.Getenv("WINDOWS_CLIENTS") == "true") 154 | 155 | parallelNodes, err := strconv.Atoi(os.Getenv("PARALLEL_NODES")) 156 | if err != nil { 157 | return nil, err 158 | } 159 | manifest.Properties.Consul.AcceptanceTests.ParallelNodes = parallelNodes 160 | 161 | if err := json.Unmarshal([]byte(os.Getenv("AWS_SUBNETS")), &manifest.Properties.Consul.AcceptanceTests.AWS.Subnets); err != nil { 162 | return nil, err 163 | } 164 | 165 | if err := json.Unmarshal([]byte(os.Getenv("AWS_CLOUD_CONFIG_SUBNETS")), &manifest.Properties.Consul.AcceptanceTests.AWS.CloudConfigSubnets); err != nil { 166 | return nil, err 167 | } 168 | 169 | contents, err = yaml.Marshal(manifest) 170 | if err != nil { 171 | return nil, err 172 | } 173 | 174 | return contents, nil 175 | } 176 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/generate_manifest_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/pivotal-cf-experimental/gomegamatchers" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | var _ = Describe("Generate", func() { 14 | var variables map[string]string 15 | 16 | BeforeEach(func() { 17 | variables = map[string]string{ 18 | "AWS_SUBNETS": `[{"id":"some-subnet-1","range":"10.0.4.0/24","az":"some-az-1","securityGroup":"some-security-group-1"},{"id":"some-subnet-2","range":"10.0.5.0/24","az":"some-az-2","securityGroup":"some-security-group-2"}]`, 19 | "AWS_CLOUD_CONFIG_SUBNETS": `[{"id":"some-cloud-config-subnet-1","range":"10.0.6.0/24","az":"some-cloud-config-az-1","securityGroup":"some-cloud-config-security-group-1"},{"id":"some-cloud-config-subnet-2","range":"10.0.7.0/24","az":"some-cloud-config-az-2","securityGroup":"some-cloud-config-security-group-2"}]`, 20 | "AWS_ACCESS_KEY_ID": "some-aws-access-key-id", 21 | "AWS_SECRET_ACCESS_KEY": "some-aws-secret-access-key", 22 | "AWS_REGION": "some-aws-region", 23 | "AWS_SECURITY_GROUP_NAME": "some-aws-security-group-name", 24 | "AWS_DEFAULT_KEY_NAME": "some-aws-default-key-name", 25 | "BOSH_ERRAND_CLOUD_CONFIG_NETWORK_NAME": "some-errand-network-name", 26 | "BOSH_ERRAND_CLOUD_CONFIG_NETWORK_STATIC_IP": "some-errand-network-static-ip", 27 | "BOSH_ERRAND_CLOUD_CONFIG_NETWORK_AZ": "some-errand-az", 28 | "BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_VM_TYPE": "some-vm-type", 29 | "BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_PERSISTENT_DISK_TYPE": "some-persistent-disk-type", 30 | "BOSH_DIRECTOR_UUID": "some-bosh-director-uuid", 31 | "BOSH_DIRECTOR": "some-bosh-target", 32 | "BOSH_USER": "some-bosh-username", 33 | "BOSH_PASSWORD": "some-bosh-password", 34 | "BOSH_DIRECTOR_CA_CERT": "some-bosh-director-ca-cert", 35 | "REGISTRY_USERNAME": "some-registry-username", 36 | "REGISTRY_PASSWORD": "some-registry-password", 37 | "REGISTRY_HOST": "some-registry-host", 38 | "PARALLEL_NODES": "10", 39 | "CONSUL_RELEASE_VERSION": "some-consul-release-version", 40 | "STEMCELL_VERSION": "some-stemcell-version", 41 | "LATEST_CONSUL_RELEASE_VERSION": "some-latest-consul-release-version", 42 | "ENABLE_TURBULENCE_TESTS": "true", 43 | "WINDOWS_CLIENTS": "true", 44 | } 45 | 46 | for name, value := range variables { 47 | variables[name] = os.Getenv(name) 48 | os.Setenv(name, value) 49 | } 50 | }) 51 | 52 | AfterEach(func() { 53 | for name, value := range variables { 54 | os.Setenv(name, value) 55 | } 56 | }) 57 | 58 | It("generates a manifest", func() { 59 | expectedManifest, err := ioutil.ReadFile("fixtures/expected.yml") 60 | Expect(err).NotTo(HaveOccurred()) 61 | 62 | manifest, err := Generate("fixtures/example.yml") 63 | Expect(err).NotTo(HaveOccurred()) 64 | 65 | Expect(manifest).To(gomegamatchers.MatchYAML(expectedManifest)) 66 | }) 67 | 68 | Context("failure cases", func() { 69 | It("returns an error when the parallel nodes is not an int", func() { 70 | os.Setenv("PARALLEL_NODES", "not an int") 71 | _, err := Generate("fixtures/example.yml") 72 | Expect(err).To(MatchError(ContainSubstring(`parsing "not an int": invalid syntax`))) 73 | }) 74 | 75 | It("returns an error when the example manifest does not exist", func() { 76 | _, err := Generate("fixtures/doesnotexist.yml") 77 | Expect(err).To(MatchError(ContainSubstring("no such file or directory"))) 78 | }) 79 | 80 | It("returns an error when the example manifest is malformed", func() { 81 | _, err := Generate("fixtures/malformed.yml") 82 | Expect(err).To(MatchError(ContainSubstring("cannot unmarshal !!str `this is...`"))) 83 | }) 84 | 85 | It("returns an error when the AWS_SUBNETS are not valid json", func() { 86 | os.Setenv("AWS_SUBNETS", "%%%%%") 87 | _, err := Generate("fixtures/example.yml") 88 | Expect(err).To(MatchError(ContainSubstring("invalid character '%' looking for beginning of value"))) 89 | }) 90 | 91 | It("returns an error when the AWS_CLOUD_CONFIG_SUBNETS are not valid json", func() { 92 | os.Setenv("AWS_CLOUD_CONFIG_SUBNETS", "%%%%%") 93 | _, err := Generate("fixtures/example.yml") 94 | Expect(err).To(MatchError(ContainSubstring("invalid character '%' looking for beginning of value"))) 95 | }) 96 | }) 97 | }) 98 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/init_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestRunConsats(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "run-consats") 13 | } 14 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | export ROOT="${PWD}" 4 | export CONSUL_RELEASE_VERSION="99999+dev.$(date +%s)" 5 | export STEMCELL_VERSION="$(cat ${ROOT}/stemcell/version)" 6 | export TURBULENCE_RELEASE_VERSION="$(cat ${ROOT}/turbulence-release/version)" 7 | export BOSH_AWS_CPI_RELEASE_VERSION="$(cat ${ROOT}/bosh-aws-cpi-release/version)" 8 | export LATEST_CONSUL_RELEASE_VERSION="$(cat ${ROOT}/latest-consul-release/version)" 9 | 10 | function main() { 11 | bosh target "${BOSH_DIRECTOR}" 12 | export BOSH_DIRECTOR_UUID="$(bosh -t "${BOSH_DIRECTOR}" status --uuid)" 13 | 14 | upload_stemcell 15 | if $WINDOWS_CLIENTS; then 16 | upload_windows_stemcell 17 | fi 18 | 19 | upload_releases 20 | generate_manifest 21 | force_compilation 22 | 23 | bosh -t "${BOSH_DIRECTOR}" deployment "${ROOT}/consats.yml" 24 | bosh -t "${BOSH_DIRECTOR}" -n deploy 25 | bosh -t "${BOSH_DIRECTOR}" run errand acceptance-tests 26 | 27 | cleanup_deployment "consul" 28 | cleanup_deployment "turbulence-consul" 29 | 30 | bosh -t "${BOSH_DIRECTOR}" -n cleanup 31 | } 32 | 33 | function upload_stemcell() { 34 | pushd "${ROOT}/stemcell" > /dev/null 35 | bosh -t "${BOSH_DIRECTOR}" upload stemcell stemcell.tgz --skip-if-exists 36 | popd > /dev/null 37 | } 38 | 39 | function upload_windows_stemcell() { 40 | pushd "${ROOT}/windows-stemcell" > /dev/null 41 | bosh -t "${BOSH_DIRECTOR}" upload stemcell light-bosh-stemcell-*-aws-xen-hvm-windows2012R2-go_agent.tgz --skip-if-exists 42 | popd > /dev/null 43 | } 44 | 45 | function upload_releases() { 46 | pushd "${ROOT}/turbulence-release" > /dev/null 47 | bosh -t "${BOSH_DIRECTOR}" upload release release.tgz --skip-if-exists 48 | popd > /dev/null 49 | 50 | pushd "${ROOT}/bosh-aws-cpi-release" > /dev/null 51 | bosh -t "${BOSH_DIRECTOR}" upload release release.tgz --skip-if-exists 52 | popd > /dev/null 53 | 54 | pushd "${ROOT}/consul-release" > /dev/null 55 | bosh -t "${BOSH_DIRECTOR}" -n create release --force --version "${CONSUL_RELEASE_VERSION}" 56 | bosh -t "${BOSH_DIRECTOR}" upload release 57 | popd > /dev/null 58 | 59 | pushd "${ROOT}/latest-consul-release" > /dev/null 60 | bosh -t "${BOSH_DIRECTOR}" upload release release.tgz --skip-if-exists 61 | popd > /dev/null 62 | 63 | } 64 | 65 | function force_compilation() { 66 | pushd /tmp > /dev/null 67 | sed -e "s/REPLACE_ME_DIRECTOR_UUID/${BOSH_DIRECTOR_UUID}/g" \ 68 | -e "s/CONSUL_RELEASE_VERSION/${CONSUL_RELEASE_VERSION}/g" \ 69 | -e "s/TURBULENCE_RELEASE_VERSION/${TURBULENCE_RELEASE_VERSION}/g" \ 70 | -e "s/CPI_RELEASE_VERSION/${BOSH_AWS_CPI_RELEASE_VERSION}/g" \ 71 | -e "s/STEMCELL_VERSION/${STEMCELL_VERSION}/g" \ 72 | "${ROOT}/mega-ci/scripts/ci/run-consats/fixtures/consul_compilation.yml" > "consul_compilation.yml" 73 | 74 | bosh -t "${BOSH_DIRECTOR}" -d "consul_compilation.yml" -n deploy 75 | bosh -t "${BOSH_DIRECTOR}" -d "consul_compilation.yml" export release "consul/${CONSUL_RELEASE_VERSION}" "ubuntu-trusty/${STEMCELL_VERSION}" 76 | bosh -t "${BOSH_DIRECTOR}" -d "consul_compilation.yml" export release "turbulence/${TURBULENCE_RELEASE_VERSION}" "ubuntu-trusty/${STEMCELL_VERSION}" 77 | bosh -t "${BOSH_DIRECTOR}" -d "consul_compilation.yml" export release "bosh-aws-cpi/${BOSH_AWS_CPI_RELEASE_VERSION}" "ubuntu-trusty/${STEMCELL_VERSION}" 78 | bosh -t "${BOSH_DIRECTOR}" -d "consul_compilation.yml" -n delete deployment compilation 79 | popd > /dev/null 80 | } 81 | 82 | function generate_manifest() { 83 | mkdir -p "${ROOT}/consul-release/aws" 84 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry" 85 | 86 | pushd "${GOPATH}/src/github.com/cloudfoundry" > /dev/null 87 | ln -s "${ROOT}/mega-ci" 88 | 89 | pushd "${GOPATH}/src/github.com/cloudfoundry/mega-ci" > /dev/null 90 | go run "./scripts/ci/run-consats/generate_manifest.go" \ 91 | "${ROOT}/consul-release/manifests/aws/consats.yml" \ 92 | > "${ROOT}/consats.yml" 93 | popd > /dev/null 94 | popd > /dev/null 95 | } 96 | 97 | function cleanup_deployment() { 98 | local deployment 99 | deployment="${1}-[A-Za-z0-9]\{8\}-[A-Za-z0-9]\{4\}-[A-Za-z0-9]\{4\}-[A-Za-z0-9]\{4\}-[A-Za-z0-9]\{12\}" 100 | 101 | for i in $(bosh -t "${BOSH_DIRECTOR}" deployments | grep -o "${deployment}" | uniq); do 102 | bosh -t "${BOSH_DIRECTOR}" -n delete deployment $i 103 | done 104 | 105 | test -z "$(bosh -t "${BOSH_DIRECTOR}" deployments | grep "${deployment}")" 106 | } 107 | 108 | function teardown() { 109 | set +e 110 | bosh -t "${BOSH_DIRECTOR}" -n delete deployment consats 111 | bosh -t "${BOSH_DIRECTOR}" -n delete release consul 112 | set -e 113 | } 114 | 115 | trap teardown EXIT 116 | 117 | main 118 | -------------------------------------------------------------------------------- /scripts/ci/run-consats/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: consul-release 9 | - name: stemcell 10 | - name: windows-stemcell 11 | - name: turbulence-release 12 | - name: bosh-aws-cpi-release 13 | - name: latest-consul-release 14 | 15 | run: 16 | path: mega-ci/scripts/ci/run-consats/task 17 | 18 | params: 19 | AWS_ACCESS_KEY_ID: 20 | AWS_REGION: 21 | AWS_SECRET_ACCESS_KEY: 22 | AWS_SECURITY_GROUP_NAME: 23 | AWS_CLOUD_CONFIG_SUBNETS: 24 | AWS_SUBNETS: 25 | AWS_DEFAULT_KEYPAIR_NAME: 26 | BOSH_DIRECTOR_CA_CERT: 27 | BOSH_PASSWORD: 28 | BOSH_DIRECTOR: 29 | BOSH_USER: 30 | BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_PERSISTENT_DISK_TYPE: 31 | BOSH_ERRAND_CLOUD_CONFIG_DEFAULT_VM_TYPE: 32 | BOSH_ERRAND_CLOUD_CONFIG_NETWORK_NAME: 33 | BOSH_ERRAND_CLOUD_CONFIG_NETWORK_STATIC_IP: 34 | BOSH_ERRAND_CLOUD_CONFIG_NETWORK_AZ: 35 | REGISTRY_PASSWORD: 36 | REGISTRY_USERNAME: 37 | REGISTRY_HOST: 38 | PARALLEL_NODES: 39 | ENABLE_TURBULENCE_TESTS: 40 | WINDOWS_CLIENTS: 41 | -------------------------------------------------------------------------------- /scripts/ci/run-eats/fixtures/etcd_compilation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | director_uuid: REPLACE_ME_DIRECTOR_UUID 3 | 4 | name: etcd-compilation 5 | 6 | releases: 7 | - name: etcd 8 | version: REPLACE_ME_ETCD_VERSION 9 | 10 | update: 11 | canaries: 1 12 | max_in_flight: 1 13 | serial: false 14 | canary_watch_time: 1000-60000 15 | update_watch_time: 1000-60000 16 | 17 | stemcells: 18 | - alias: trusty 19 | os: ubuntu-trusty 20 | version: latest 21 | 22 | instance_groups: [] 23 | -------------------------------------------------------------------------------- /scripts/ci/run-eats/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | export ROOT="${PWD}" 4 | export ETCD_RELEASE_VERSION="$(ls compiled-etcd-release/*.tgz | sed 's/compiled-etcd-release\/release-etcd-\(.*\)-on-ubuntu.*/\1/')" 5 | 6 | function main() { 7 | mkdir -p $GOPATH/src/github.com/cloudfoundry-incubator 8 | pushd $GOPATH/src/github.com/cloudfoundry-incubator > /dev/null 9 | ln -s $ROOT/etcd-release 10 | popd > /dev/null 11 | 12 | upload_releases 13 | write_configuration 14 | install_bosh_cli 15 | unset_boshv1 16 | 17 | pushd $GOPATH/src/github.com/cloudfoundry-incubator/etcd-release > /dev/null 18 | EATS_CONFIG=$PWD/../eats_config.json ./src/acceptance-tests/scripts/test -nodes "${PARALLEL_NODES}" --skipPackage=turbulence 19 | popd > /dev/null 20 | } 21 | 22 | function install_bosh_cli() { 23 | pushd "${ROOT}" > /dev/null 24 | wget https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-0.0.147-linux-amd64 25 | mv bosh-cli-0.0.147-linux-amd64 /usr/local/bin/bosh 26 | chmod +x /usr/local/bin/bosh 27 | popd > /dev/null 28 | } 29 | 30 | function unset_boshv1() { 31 | unset BOSH_USER 32 | unset BOSH_PASSWORD 33 | } 34 | 35 | function set_boshv1() { 36 | export BOSH_USER="$BOSH_CLIENT" 37 | export BOSH_PASSWORD="$BOSH_CLIENT_SECRET" 38 | } 39 | 40 | function write_configuration() { 41 | cat > eats_config.json < /dev/null 64 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_ENVIRONMENT}" -n upload release release-*.tgz 65 | popd > /dev/null 66 | 67 | pushd "${ROOT}/consul-release" > /dev/null 68 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_ENVIRONMENT}" upload release release-*.tgz --skip-if-exists 69 | popd > /dev/null 70 | } 71 | 72 | function cleanup_releases() { 73 | set_boshv1 74 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_ENVIRONMENT}" -n cleanup 75 | } 76 | 77 | function rollup() { 78 | set +x 79 | local input 80 | input="${1}" 81 | 82 | local output 83 | 84 | IFS=$'\n' 85 | for line in ${input}; do 86 | output="${output:-""}\n${line}" 87 | done 88 | 89 | printf "%s" "${output#'\n'}" 90 | set -x 91 | } 92 | 93 | trap cleanup_releases EXIT 94 | main 95 | -------------------------------------------------------------------------------- /scripts/ci/run-eats/task-turbulence: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | export ROOT="${PWD}" 4 | export ETCD_RELEASE_VERSION="$(ls compiled-etcd-release/*.tgz | sed 's/compiled-etcd-release\/release-etcd-\(.*\)-on-ubuntu.*/\1/')" 5 | 6 | function main() { 7 | mkdir -p $GOPATH/src/github.com/cloudfoundry-incubator 8 | pushd $GOPATH/src/github.com/cloudfoundry-incubator > /dev/null 9 | ln -s $ROOT/etcd-release 10 | popd > /dev/null 11 | 12 | upload_releases 13 | write_configuration 14 | 15 | pushd $GOPATH/src/github.com/cloudfoundry-incubator/etcd-release > /dev/null 16 | EATS_CONFIG=$PWD/../eats_config.json ./src/acceptance-tests/scripts/test -nodes "${PARALLEL_NODES}" turbulence 17 | popd > /dev/null 18 | } 19 | 20 | function write_configuration() { 21 | cat > eats_config.json < /dev/null 50 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n upload release release-*.tgz 51 | popd > /dev/null 52 | 53 | pushd "${ROOT}/consul-release" > /dev/null 54 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload release release-*.tgz --skip-if-exists 55 | popd > /dev/null 56 | 57 | pushd "${ROOT}/turbulence-release" > /dev/null 58 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload release release.tgz --skip-if-exists 59 | popd > /dev/null 60 | 61 | pushd "${ROOT}/bosh-aws-cpi-release" > /dev/null 62 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" upload release release.tgz --skip-if-exists 63 | popd > /dev/null 64 | } 65 | 66 | function cleanup_releases() { 67 | /opt/rubies/ruby-2.2.4/bin/bosh -t "${BOSH_DIRECTOR}" -n cleanup 68 | } 69 | 70 | function rollup() { 71 | set +x 72 | local input 73 | input="${1}" 74 | 75 | local output 76 | 77 | IFS=$'\n' 78 | for line in ${input}; do 79 | output="${output:-""}\n${line}" 80 | done 81 | 82 | printf "%s" "${output#'\n'}" 83 | set -x 84 | } 85 | 86 | trap cleanup_releases EXIT 87 | main 88 | -------------------------------------------------------------------------------- /scripts/ci/run-eats/task-turbulence.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: compiled-etcd-release 9 | - name: etcd-release 10 | - name: consul-release 11 | - name: turbulence-release 12 | - name: bosh-aws-cpi-release 13 | 14 | run: 15 | path: mega-ci/scripts/ci/run-eats/task-turbulence 16 | 17 | params: 18 | BOSH_DIRECTOR: 19 | BOSH_USER: 20 | BOSH_PASSWORD: 21 | BOSH_DIRECTOR_CA_CERT: 22 | AWS_SUBNET: 23 | AWS_ACCESS_KEY_ID: 24 | AWS_SECRET_ACCESS_KEY: 25 | AWS_DEFAULT_KEY_NAME: 26 | AWS_DEFAULT_SECURITY_GROUPS: 27 | AWS_REGION: 28 | PARALLEL_NODES: 29 | REGISTRY_USERNAME: 30 | REGISTRY_PASSWORD: 31 | REGISTRY_PORT: 32 | REGISTRY_HOST: 33 | -------------------------------------------------------------------------------- /scripts/ci/run-eats/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: etcd-release 9 | - name: compiled-etcd-release 10 | - name: consul-release 11 | 12 | run: 13 | path: mega-ci/scripts/ci/run-eats/task 14 | 15 | params: 16 | BOSH_DIRECTOR: 17 | BOSH_PASSWORD: 18 | BOSH_USER: 19 | BOSH_ENVIRONMENT: 20 | BOSH_CLIENT: 21 | BOSH_CLIENT_SECRET: 22 | BOSH_CA_CERT: 23 | AWS_SUBNET: 24 | AWS_ACCESS_KEY_ID: 25 | AWS_SECRET_ACCESS_KEY: 26 | AWS_DEFAULT_KEY_NAME: 27 | AWS_DEFAULT_SECURITY_GROUPS: 28 | AWS_REGION: 29 | REGISTRY_HOST: 30 | REGISTRY_PORT: 31 | REGISTRY_USERNAME: 32 | REGISTRY_PASSWORD: 33 | PARALLEL_NODES: 34 | -------------------------------------------------------------------------------- /scripts/ci/test: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eux 4 | 5 | root_dir=$(cd "$(dirname $0)/../.." && pwd) 6 | 7 | scripts_dir="${root_dir}/scripts" 8 | 9 | # install ginkgo 10 | go get -v github.com/onsi/ginkgo/ginkgo 11 | go get -v github.com/onsi/gomega 12 | 13 | "${scripts_dir}/test" 14 | -------------------------------------------------------------------------------- /scripts/ci/test-cf-tls-upgrade/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | function create_integration_config() { 4 | cat > eats_config.json < /dev/null 26 | ln -s $ROOT/etcd-release 27 | popd > /dev/null 28 | 29 | pushd $GOPATH/src/github.com/cloudfoundry-incubator/etcd-release > /dev/null 30 | create_integration_config 31 | EATS_CONFIG="${PWD}/eats_config.json" ginkgo \ 32 | -r \ 33 | -randomizeAllSpecs \ 34 | -randomizeSuites \ 35 | -failFast \ 36 | -succinct \ 37 | -slowSpecThreshold 300 \ 38 | ./src/cf-tls-upgrade/ 39 | popd > /dev/null 40 | } 41 | 42 | main "${PWD}" 43 | -------------------------------------------------------------------------------- /scripts/ci/test-cf-tls-upgrade/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | - name: etcd-release 9 | 10 | run: 11 | path: mega-ci/scripts/ci/test-cf-tls-upgrade/task 12 | 13 | params: 14 | CF_DOMAIN: 15 | CF_USER: 16 | CF_PASSWORD: 17 | BOSH_DIRECTOR: 18 | BOSH_USER: 19 | BOSH_PASSWORD: 20 | DEPLOYMENT_NAME: 21 | -------------------------------------------------------------------------------- /scripts/ci/test-diego/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | function main() { 4 | 5 | cf api api.$CF_DOMAIN --skip-ssl-validation 6 | 7 | set +x 8 | cf auth $CF_USER $CF_PASSWORD 9 | set -x 10 | 11 | cf enable-feature-flag diego_docker 12 | 13 | /opt/rubies/ruby-2.2.4/bin/bosh -t $BOSH_DIRECTOR download manifest $DEPLOYMENT_NAME manifest.yml 14 | 15 | /opt/rubies/ruby-2.2.4/bin/bosh -n --color -t $BOSH_DIRECTOR -d manifest.yml run errand acceptance_tests 16 | } 17 | 18 | main 19 | -------------------------------------------------------------------------------- /scripts/ci/test-diego/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | 9 | run: 10 | path: mega-ci/scripts/ci/test-diego/task 11 | 12 | params: 13 | CF_DOMAIN: 14 | CF_USER: 15 | CF_PASSWORD: 16 | BOSH_DIRECTOR: 17 | BOSH_USER: 18 | BOSH_PASSWORD: 19 | DEPLOYMENT_NAME: 20 | -------------------------------------------------------------------------------- /scripts/ci/test-etcd-metrics-server/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | function main() { 4 | local root 5 | root="${1}" 6 | 7 | curl -L https://github.com/coreos/etcd/releases/download/v2.1.1/etcd-v2.1.1-linux-amd64.tar.gz | tar zxf - -C /tmp 8 | curl -L https://github.com/nats-io/gnatsd/releases/download/v0.6.0/gnatsd-v0.6.0-linux-amd64.tar.gz | tar zxf - -C /tmp 9 | 10 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry-incubator" 11 | pushd "${GOPATH}/src/github.com/cloudfoundry-incubator" > /dev/null 12 | ln -s "${root}/etcd-release/src/etcd-metrics-server" 13 | pushd ./etcd-metrics-server > /dev/null 14 | export GOPATH="${PWD}/Godeps/_workspace:${GOPATH}" 15 | export PATH="${PWD}/Godeps/_workspace/bin:${PATH}" 16 | 17 | mkdir -p "${PWD}/Godeps/_workspace/bin" 18 | cp /tmp/etcd-v2.1.1-linux-amd64/etcd "${PWD}/Godeps/_workspace/bin/" 19 | cp /tmp/gnatsd "${PWD}/Godeps/_workspace/bin/" 20 | 21 | ginkgo -r -race -randomizeAllSpecs -randomizeSuites 22 | popd > /dev/null 23 | 24 | popd > /dev/null 25 | } 26 | 27 | main "${PWD}" 28 | -------------------------------------------------------------------------------- /scripts/ci/test-etcd-metrics-server/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/golang 5 | 6 | inputs: 7 | - name: etcd-release 8 | - name: mega-ci 9 | 10 | run: 11 | path: mega-ci/scripts/ci/test-etcd-metrics-server/test 12 | -------------------------------------------------------------------------------- /scripts/ci/test-etcd-proxy/task: -------------------------------------------------------------------------------- 1 | #!/bin/bash -exu 2 | 3 | ROOT="${PWD}" 4 | 5 | mkdir -p "${GOPATH}/src/github.com/cloudfoundry-incubator" 6 | 7 | pushd "${GOPATH}/src/github.com/cloudfoundry-incubator" > /dev/null 8 | ln -s "${ROOT}/etcd-release" 9 | ginkgo -r etcd-release/src/etcd-proxy 10 | popd > /dev/null 11 | -------------------------------------------------------------------------------- /scripts/ci/test-etcd-proxy/task.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/golang 5 | 6 | inputs: 7 | - name: etcd-release 8 | - name: mega-ci 9 | 10 | run: 11 | path: mega-ci/scripts/ci/test-etcd-proxy/task 12 | -------------------------------------------------------------------------------- /scripts/ci/test-release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -x 3 | 4 | source ~/.bashrc 5 | 6 | cd release 7 | 8 | RELEASE_OUT="../create-release.out" 9 | for i in {1..5}; do 10 | echo "Syncing blobs, attempt $i" 11 | bosh -n --parallel 10 sync blobs && break 12 | done 13 | 14 | for i in {1..5}; do 15 | echo "Creating release, attempt $i" 16 | bosh -n create release --with-tarball | tee -a $RELEASE_OUT 17 | EXIT_STATUS=${PIPESTATUS[0]} 18 | if [ "$EXIT_STATUS" = "0" ]; then 19 | break 20 | fi 21 | done 22 | 23 | if [ ! "$EXIT_STATUS" = "0" ]; then 24 | echo "Failed to Create Release" 25 | exit $EXIT_STATUS 26 | fi 27 | 28 | MANIFEST_YML=`grep -a "Release manifest" $RELEASE_OUT | cut -d " " -f3` 29 | if [ "$MANIFEST_YML" = "" ]; then 30 | echo "No Release Manifest Found" 31 | exit 1 32 | fi 33 | 34 | TARBALL=`grep -a "Release tarball" $RELEASE_OUT | cut -d " " -f4` 35 | if [ "$TARBALL" = "" ]; then 36 | echo "No Release Tarball Found" 37 | exit 1 38 | fi 39 | 40 | rm -rf ./.blobs 41 | 42 | mkdir -p output 43 | 44 | mv $MANIFEST_YML ./output/manifest.yml 45 | mv $TARBALL ./output/$RELEASE_NAME-${COMMIT_SHA}.tgz 46 | mv $RELEASE_OUT ./output/ 47 | 48 | -------------------------------------------------------------------------------- /scripts/ci/test-release.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: release 8 | - name: mega-ci 9 | 10 | run: 11 | path: mega-ci/scripts/ci/test-release 12 | 13 | params: 14 | RELEASE_DIR: 15 | -------------------------------------------------------------------------------- /scripts/ci/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | platform: linux 3 | 4 | image: docker:///cfinfrastructure/deployment 5 | 6 | inputs: 7 | - name: mega-ci 8 | 9 | run: 10 | path: mega-ci/scripts/ci/test 11 | -------------------------------------------------------------------------------- /scripts/common/atc_credentials.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function atc_credentials() { 4 | file=$1 5 | 6 | cat > ${file} < ${file} </dev/null || { echo "aws is required"; exit 1; } 12 | command -v jq >/dev/null || { echo "jq is required"; exit 1; } 13 | } 14 | 15 | function load_credentials() { 16 | set +x 17 | source "${credentials_dir}/aws_environment" 18 | 19 | if [[ -z "${AWS_DEFAULT_REGION}" ]]; then 20 | echo "AWS_DEFAULT_REGION is not set" 21 | exit 1 22 | fi 23 | 24 | if [[ -z "${AWS_ACCESS_KEY_ID}" ]]; then 25 | echo "AWS_DEFAULT_REGION is not set" 26 | exit 1 27 | fi 28 | 29 | if [[ -z "${AWS_SECRET_ACCESS_KEY}" ]]; then 30 | echo "AWS_DEFAULT_REGION is not set" 31 | exit 1 32 | fi 33 | 34 | set -x 35 | } 36 | 37 | function apply_cloudformation_template() { 38 | local release_name 39 | release_name="${1}" 40 | 41 | local script_dir 42 | script_dir="${2}" 43 | 44 | if aws cloudformation describe-stacks --stack-name "${release_name}-buckets" > /dev/null; then 45 | apply_stack_operation update "${release_name}" "${script_dir}" || true 46 | else 47 | apply_stack_operation create "${release_name}" "${script_dir}" 48 | fi 49 | 50 | wait_on_cloudformation "${release_name}" 51 | check_final_cloudformation_state "${release_name}" 52 | } 53 | 54 | function apply_stack_operation() { 55 | local operation 56 | operation="${1}" 57 | 58 | local release_name 59 | release_name="${2}" 60 | 61 | local script_dir 62 | script_dir="${3}" 63 | 64 | aws cloudformation "${operation}-stack" \ 65 | --stack-name "${release_name}-buckets" \ 66 | --parameters "ParameterKey=BucketName,ParameterValue=${release_name}-release-blobs" \ 67 | --template-body "file://${script_dir}/../templates/final-release/bucket.json" \ 68 | --capabilities CAPABILITY_IAM > /dev/null 69 | } 70 | 71 | function cloudformation_stack_status() { 72 | local release_name 73 | release_name="${1}" 74 | 75 | local status 76 | status="$(aws cloudformation describe-stacks --stack-name "${release_name}-buckets" | jq -r .Stacks[].StackStatus)" 77 | 78 | printf "%s" "${status}" 79 | } 80 | 81 | function wait_on_cloudformation() { 82 | local release_name 83 | release_name="${1}" 84 | 85 | local half_an_hour 86 | half_an_hour="$((30 * 60))" 87 | 88 | local deadline_in_seconds 89 | deadline_in_seconds=$(($(date +%s) + ${half_an_hour})) 90 | 91 | while cloudformation_stack_status "${release_name}" | grep IN_PROGRESS ; do 92 | echo "CloudFormation stack '${release_name}-buckets' still in progress..." 93 | 94 | local remaining_time_in_seconds 95 | remaining_time_in_seconds="$((deadline_in_seconds - $(date +%s)))" 96 | 97 | if [ "${remaining_time_in_seconds}" -gt 0 ]; then 98 | echo " Waiting ${remaining_time_in_seconds} more seconds." 99 | sleep 15 100 | else 101 | echo " Waited ${half_an_hour} seconds, aborting." 102 | exit 1 103 | fi 104 | done 105 | } 106 | 107 | function check_final_cloudformation_state() { 108 | local release_name 109 | release_name="${1}" 110 | 111 | local status 112 | status="$(cloudformation_stack_status "${release_name}")" 113 | 114 | if echo "${status}" | grep ROLLBACK ; then 115 | echo "CloudFormation failed to apply template: ${status}" 116 | exit 1 117 | fi 118 | 119 | if ! echo "${status}" | grep COMPLETE ; then 120 | echo "CloudFormation failed to apply template: ${status}" 121 | exit 1 122 | fi 123 | } 124 | 125 | function write_private_yaml() { 126 | local release_name 127 | release_name="${1}" 128 | 129 | local credentials_dir 130 | credentials_dir="${2}" 131 | 132 | local release_config_dir 133 | release_config_dir="${3}" 134 | 135 | local bucket_output 136 | bucket_output="${credentials_dir}/bucket.json" 137 | 138 | aws cloudformation describe-stacks --stack-name "${release_name}-buckets" > "${bucket_output}" 139 | 140 | local access_key_id 141 | local secret_access_key 142 | 143 | set +x 144 | access_key_id="$(jq -e -r ".Stacks[0].Outputs[0].OutputValue" "${bucket_output}")" 145 | secret_access_key="$(jq -e -r ".Stacks[0].Outputs[1].OutputValue" "${bucket_output}")" 146 | set -x 147 | 148 | cat > "${release_config_dir}/private.yml" </dev/null || { echo "aws is required"; exit 1; } 16 | command -v bosh >/dev/null || { echo "bosh is required"; exit 1; } 17 | command -v jq >/dev/null || { echo "jq is required"; exit 1; } 18 | command -v spiff >/dev/null || { echo "spiff is required"; exit 1; } 19 | 20 | set +x 21 | source ${deployment_dir}/aws_environment 22 | 23 | if [ -z ${AWS_DEFAULT_REGION} ]; then 24 | echo 'AWS_DEFAULT_REGION is not set' 25 | exit 1 26 | fi 27 | 28 | if [ -z ${AWS_ACCESS_KEY_ID} ]; then 29 | echo 'AWS_DEFAULT_REGION is not set' 30 | exit 1 31 | fi 32 | 33 | if [ -z ${AWS_SECRET_ACCESS_KEY} ]; then 34 | echo 'AWS_DEFAULT_REGION is not set' 35 | exit 1 36 | fi 37 | 38 | source ${deployment_dir}/bosh_environment 39 | 40 | if [ -z ${BOSH_USER} ]; then 41 | echo 'BOSH_USER is not set' 42 | exit 1 43 | fi 44 | 45 | if [ -z ${BOSH_PASSWORD} ]; then 46 | echo 'BOSH_PASSWORD is not set' 47 | exit 1 48 | fi 49 | 50 | if [ -z ${BOSH_DIRECTOR} ]; then 51 | echo 'BOSH_DIRECTOR is not set' 52 | exit 1 53 | fi 54 | set -x 55 | 56 | mkdir -p ${deployment_dir}/artifacts/deployments 57 | mkdir -p generated-stubs 58 | 59 | for script_source in ${script_dir}/common/*; do source ${script_source}; done 60 | 61 | # Target bosh 62 | bosh -n target $BOSH_DIRECTOR 63 | set +x 64 | bosh login $BOSH_USER $BOSH_PASSWORD 65 | set -x 66 | 67 | # Obtain bosh uuid for manifest generation 68 | BOSH_UUID=$(bosh --no-color status --uuid) 69 | cat > generated-stubs/director_uuid.yml < generated-stubs/aws-resources.json 78 | 79 | # Generate the cloud propeties stub 80 | SUBNET_ID=$(cat generated-stubs/aws-resources.json | jq -r .Resources.ConcourseSubnet) 81 | 82 | CONCOURSE_SECURITY_GROUP_ID=$(cat generated-stubs/aws-resources.json | jq -r .Resources.ConcourseSecurityGroup) 83 | CONCOURSE_SECURITY_GROUP_NAME=$(aws ec2 describe-security-groups --group-ids ${CONCOURSE_SECURITY_GROUP_ID} | jq .SecurityGroups[0].GroupName) 84 | 85 | AWS_ZONE=$(aws ec2 describe-subnets --subnet-ids $SUBNET_ID | jq -r .Subnets[0].AvailabilityZone) 86 | 87 | LOAD_BALANCER=$(cat generated-stubs/aws-resources.json | jq -r .Resources.ConcourseLoadBalancer) 88 | 89 | cat > generated-stubs/cloud_properties.yml < ${deployment_dir}/artifacts/deployments/concourse.yml 115 | spiff merge $script_dir/../templates/bosh/bosh-deployment-mask.yml \ 116 | $script_dir/../templates/concourse/concourse.yml \ 117 | generated-stubs/director_uuid.yml \ 118 | generated-stubs/cloud_properties.yml \ 119 | ${deployment_dir}/stubs/concourse/*.yml \ 120 | ${optional_datadog} \ 121 | ${optional_syslog} \ 122 | >> ${deployment_dir}/artifacts/deployments/concourse.yml 123 | 124 | # Upload binaries to BOSH director 125 | STEMCELL_PATH=$(cat $deployment_dir/stubs/concourse/binary_urls.json | jq -r .stemcell) 126 | CONCOURSE_RELEASE_PATH=$(cat $deployment_dir/stubs/concourse/binary_urls.json | jq -r .concourse_release) 127 | GARDEN_RELEASE_PATH=$(cat $deployment_dir/stubs/concourse/binary_urls.json | jq -r .garden_release) 128 | bosh -n upload stemcell $STEMCELL_PATH --skip-if-exists 129 | bosh -n upload release $CONCOURSE_RELEASE_PATH --skip-if-exists 130 | bosh -n upload release $GARDEN_RELEASE_PATH --skip-if-exists 131 | 132 | # Deploy Concourse 133 | bosh \ 134 | -n \ 135 | -d "${deployment_dir}/artifacts/deployments/concourse.yml" \ 136 | deploy 137 | 138 | # display result 139 | LOAD_BALANCER_DNSName=$(aws elb describe-load-balancers --load-balancer-names $LOAD_BALANCER| jq -r .LoadBalancerDescriptions[0].DNSName) 140 | echo "Concourse available at https://$LOAD_BALANCER_DNSName" 141 | 142 | # clean up 143 | rm -rf generated-stubs 144 | -------------------------------------------------------------------------------- /scripts/setup_aws_bosh_for_concourse: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e -x 4 | 5 | script_dir=$(dirname $0) 6 | 7 | if [ $# -ne 1 ]; then 8 | echo "Expected 1 argument (path to deployment directory), received $#" 9 | exit 1 10 | fi 11 | 12 | #remove trailing slash 13 | deployment_dir=${1%/} 14 | 15 | command -v aws >/dev/null || { echo "aws is required"; exit 1; } 16 | command -v bosh-init >/dev/null || { echo "bosh-init is required"; exit 1; } 17 | command -v jq >/dev/null || { echo "jq is required"; exit 1; } 18 | command -v spiff >/dev/null || { echo "spiff is required"; exit 1; } 19 | command -v openssl >/dev/null || { echo "openssl is required"; exit 1; } 20 | 21 | set +x 22 | source $deployment_dir/aws_environment 23 | 24 | if [ -z ${AWS_DEFAULT_REGION} ]; then 25 | echo 'AWS_DEFAULT_REGION is not set' 26 | exit 1 27 | fi 28 | 29 | if [ -z ${AWS_ACCESS_KEY_ID} ]; then 30 | echo 'AWS_ACCESS_KEY_ID is not set' 31 | exit 1 32 | fi 33 | 34 | if [ -z ${AWS_SECRET_ACCESS_KEY} ]; then 35 | echo 'AWS_SECRET_ACCESS_KEY is not set' 36 | exit 1 37 | fi 38 | set -x 39 | 40 | mkdir -p ${deployment_dir}/artifacts/certs 41 | mkdir -p ${deployment_dir}/artifacts/keypair 42 | mkdir -p ${deployment_dir}/artifacts/deployments 43 | mkdir -p ${deployment_dir}/generated-stubs/pipeline 44 | mkdir -p ${deployment_dir}/stubs/bosh 45 | mkdir -p generated-stubs 46 | 47 | for script_source in ${script_dir}/common/*; do source ${script_source}; done 48 | 49 | # create bosh ssl certificate 50 | if [ ! -e ${deployment_dir}/artifacts/certs/rootCA.key ]; then 51 | openssl genrsa -out ${deployment_dir}/artifacts/certs/rootCA.key 2048 52 | fi 53 | 54 | if [ ! -e ${deployment_dir}/artifacts/certs/rootCA.pem ]; then 55 | yes "" | openssl req -x509 -new -nodes -key ${deployment_dir}/artifacts/certs/rootCA.key -out ${deployment_dir}/artifacts/certs/rootCA.pem -days 99999 56 | fi 57 | 58 | # create keypair 59 | if ! aws ec2 describe-key-pairs --key-names bosh; then 60 | aws ec2 create-key-pair --key-name bosh | jq -r .KeyMaterial > ${deployment_dir}/artifacts/keypair/id_rsa_bosh 61 | fi 62 | 63 | # install certs for CF ELB 64 | if [ -e ${deployment_dir}/certs/cf.key ] && ! aws iam get-server-certificate --server-certificate-name cf; then 65 | aws iam upload-server-certificate --server-certificate-name cf --private-key file://${deployment_dir}/certs/cf.key --certificate-body file://${deployment_dir}/certs/cf.pem 66 | fi 67 | 68 | # install certs for Concourse ELB 69 | set +e 70 | CONCOURSE_HOSTNAME=$(cat ${deployment_dir}/cloud_formation/properties.json | \ 71 | jq -r '.|map({key: .ParameterKey, value: .ParameterValue}) | from_entries as $body | $body | .ELBRecordSetName' | grep -v "^null$" | grep -o ".*[^.]") 72 | set -e 73 | 74 | if ! aws iam get-server-certificate --server-certificate-name concourse; then 75 | if [ -e ${deployment_dir}/certs/concourse_chain.pem ]; then 76 | aws iam upload-server-certificate --server-certificate-name concourse --private-key file://${deployment_dir}/certs/concourse.key --certificate-body file://${deployment_dir}/certs/concourse.pem --certificate-chain file://${deployment_dir}/certs/concourse_chain.pem 77 | elif [ -e ${deployment_dir}/certs/concourse.pem ]; then 78 | aws iam upload-server-certificate --server-certificate-name concourse --private-key file://${deployment_dir}/certs/concourse.key --certificate-body file://${deployment_dir}/certs/concourse.pem 79 | elif [ -n ${CONCOURSE_HOSTNAME} ]; then 80 | generate_certificate "${deployment_dir}/certs" "concourse" ${CONCOURSE_HOSTNAME} 81 | aws iam upload-server-certificate --server-certificate-name concourse --private-key file://${deployment_dir}/certs/concourse.key --certificate-body file://${deployment_dir}/certs/concourse.pem 82 | fi 83 | fi 84 | 85 | # deploy infrastructure 86 | if ! aws cloudformation describe-stacks --stack-name concourse ; then 87 | if [ -e ${deployment_dir}/cloud_formation/properties.json ]; then 88 | aws cloudformation create-stack \ 89 | --stack-name concourse \ 90 | --template-body file://$script_dir/../templates/bootload-mega-ci/aws-for-concourse.json \ 91 | --parameters file://${deployment_dir}/cloud_formation/properties.json 92 | else 93 | aws cloudformation create-stack \ 94 | --stack-name concourse \ 95 | --template-body file://$script_dir/../templates/bootload-mega-ci/aws-for-concourse.json 96 | fi 97 | else 98 | if [ -e ${deployment_dir}/cloud_formation/properties.json ]; then 99 | aws cloudformation update-stack \ 100 | --stack-name concourse \ 101 | --template-body file://$script_dir/../templates/bootload-mega-ci/aws-for-concourse.json \ 102 | --parameters file://${deployment_dir}/cloud_formation/properties.json || true # update-stack is not idempotent :( 103 | else 104 | aws cloudformation update-stack \ 105 | --stack-name concourse \ 106 | --template-body file://$script_dir/../templates/bootload-mega-ci/aws-for-concourse.json || true # update-stack is not idempotent :( 107 | fi 108 | fi 109 | 110 | while aws cloudformation describe-stacks --stack-name concourse | grep StackStatus | grep IN_PROGRESS ; do 111 | echo "CloudFormation stack 'concourse' still in progress..." 112 | sleep 15 113 | done 114 | 115 | if aws cloudformation describe-stacks --stack-name concourse | grep StackStatus | grep ROLLBACK ; then 116 | echo 'Cloud formation failure' 117 | exit 1 118 | fi 119 | 120 | if ! aws cloudformation describe-stacks --stack-name concourse | grep StackStatus | grep COMPLETE ; then 121 | echo 'Cloud formation failure' 122 | exit 1 123 | fi 124 | 125 | # generate AWS resources stub for shared purposes 126 | aws cloudformation describe-stack-resources --stack-name concourse \ 127 | | jq '.StackResources|map({key: .LogicalResourceId, value: .PhysicalResourceId}) | from_entries as $body | {Resources: $body}' \ 128 | > generated-stubs/aws-resources.json 129 | 130 | BOSH_IP=$(cat generated-stubs/aws-resources.json | jq -r .Resources.MicroEIP) 131 | # generate certificate for bosh director 132 | if [ ! -e ${deployment_dir}/artifacts/certs/bosh.crt ]; then 133 | cat >generated-stubs/openssl-exts.conf <<-EOL 134 | extensions = san 135 | [san] 136 | subjectAltName = IP:${BOSH_IP} 137 | EOL 138 | 139 | openssl req -new -nodes -newkey rsa:2048 \ 140 | -out generated-stubs/bosh.csr -keyout ${deployment_dir}/artifacts/certs/bosh.key \ 141 | -subj "/C=US/O=BOSH/CN=${BOSH_IP}" 142 | 143 | openssl x509 -req -in generated-stubs/bosh.csr \ 144 | -CA ${deployment_dir}/artifacts/certs/rootCA.pem -CAkey ${deployment_dir}/artifacts/certs/rootCA.key -CAcreateserial \ 145 | -out ${deployment_dir}/artifacts/certs/bosh.crt -days 99999 \ 146 | -extfile generated-stubs/openssl-exts.conf 147 | fi 148 | 149 | cat > generated-stubs/bosh-certs.json < generated-stubs/security-groups.yml < generated-stubs/aws-credentials.yml < ${deployment_dir}/generated-stubs/pipeline/cf-resources.yml< ${deployment_dir}/artifacts/deployments/bosh.yml 208 | spiff merge $script_dir/../templates/bosh/bosh-init-mask.yml \ 209 | $script_dir/../templates/bosh/bosh-init.yml \ 210 | generated-stubs/aws-credentials.yml \ 211 | generated-stubs/aws-resources.json \ 212 | generated-stubs/bosh-certs.json \ 213 | generated-stubs/security-groups.yml \ 214 | ${deployment_dir}/stubs/bosh/bosh_passwords.yml \ 215 | ${deployment_dir}/stubs/bosh/resource_overrides.yml \ 216 | >> ${deployment_dir}/artifacts/deployments/bosh.yml 217 | 218 | # deploy BOSH 219 | bosh-init deploy ${deployment_dir}/artifacts/deployments/bosh.yml 220 | 221 | # display result 222 | echo "BOSH director is at $BOSH_IP" 223 | 224 | # generate director uuid stub for cf 225 | BOSH_DIRECTOR_UUID=$(bosh -t ${BOSH_IP} status --uuid) 226 | cat > ${deployment_dir}/generated-stubs/pipeline/bosh-director-uuid.yml < /dev/null 8 | ln -s "${ROOT}/mega-ci" 9 | pushd mega-ci > /dev/null 10 | ginkgo \ 11 | -p \ 12 | -r \ 13 | -race \ 14 | -randomizeAllSpecs \ 15 | -randomizeSuites \ 16 | "$@" \ 17 | . 18 | popd > /dev/null 19 | popd > /dev/null 20 | } 21 | 22 | main 23 | -------------------------------------------------------------------------------- /templates/bosh/bosh-deployment-mask.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: (( merge )) 3 | director_uuid: (( merge )) 4 | releases: (( merge )) 5 | networks: (( merge )) 6 | jobs: (( merge )) 7 | resource_pools: (( merge )) 8 | compilation: (( merge )) 9 | update: (( merge )) 10 | -------------------------------------------------------------------------------- /templates/bosh/bosh-init-mask.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: (( merge )) 3 | releases: (( merge )) 4 | resource_pools: (( merge )) 5 | disk_pools: (( merge )) 6 | networks: (( merge )) 7 | jobs: (( merge )) 8 | cloud_provider: (( merge )) 9 | -------------------------------------------------------------------------------- /templates/bosh/bosh-init.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: bosh 3 | 4 | releases: 5 | - name: bosh 6 | url: https://bosh.io/d/github.com/cloudfoundry/bosh?v=260 7 | sha1: f8f086974d9769263078fb6cb7927655744dacbc 8 | - name: bosh-aws-cpi 9 | url: https://bosh.io/d/github.com/cloudfoundry-incubator/bosh-aws-cpi-release?v=61 10 | sha1: 32da5ff428082bbe6b4ef50b6ba386d44b204baf 11 | 12 | resource_pools: 13 | - name: vms 14 | network: private 15 | stemcell: 16 | url: https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent?v=3312 17 | sha1: ea1d210b58a12d957d0b9b6a6cc538262ee4a924 18 | cloud_properties: 19 | instance_type: m3.xlarge 20 | ephemeral_disk: {size: 25_000, type: gp2} 21 | availability_zone: (( AWSCredentials.AWS_ZONE )) 22 | 23 | disk_pools: 24 | - name: disks 25 | disk_size: 20_000 26 | cloud_properties: {type: gp2} 27 | 28 | networks: 29 | - name: private 30 | type: manual 31 | subnets: 32 | - range: 10.0.0.0/24 33 | gateway: 10.0.0.1 34 | dns: [10.0.0.2] 35 | cloud_properties: {subnet: (( Resources.BOSHSubnet )) } 36 | - name: public 37 | type: vip 38 | 39 | jobs: 40 | - name: bosh 41 | instances: 1 42 | 43 | templates: 44 | - {name: nats, release: bosh} 45 | - {name: postgres, release: bosh} 46 | - {name: blobstore, release: bosh} 47 | - {name: director, release: bosh} 48 | - {name: health_monitor, release: bosh} 49 | - {name: registry, release: bosh} 50 | - {name: aws_cpi, release: bosh-aws-cpi} 51 | 52 | resource_pool: vms 53 | persistent_disk_pool: disks 54 | 55 | networks: 56 | - name: private 57 | static_ips: [10.0.0.6] 58 | default: [dns, gateway] 59 | - name: public 60 | static_ips: [(( Resources.MicroEIP ))] 61 | 62 | properties: 63 | nats: 64 | address: 127.0.0.1 65 | user: nats 66 | password: (( bosh_credentials.nats_password )) 67 | 68 | postgres: &db 69 | host: 127.0.0.1 70 | user: postgres 71 | password: (( bosh_credentials.postgres_password )) 72 | database: bosh 73 | adapter: postgres 74 | 75 | registry: 76 | address: 10.0.0.6 77 | host: 10.0.0.6 78 | db: *db 79 | http: 80 | user: admin 81 | password: (( bosh_credentials.registry_password )) 82 | port: 25777 83 | username: admin 84 | password: (( bosh_credentials.registry_password )) 85 | port: 25777 86 | 87 | blobstore: 88 | address: 10.0.0.6 89 | port: 25250 90 | provider: dav 91 | director: 92 | user: director 93 | password: (( bosh_credentials.director_password )) 94 | agent: 95 | user: agent 96 | password: (( bosh_credentials.agent_password )) 97 | 98 | director: 99 | address: 127.0.0.1 100 | name: my-bosh 101 | db: *db 102 | cpi_job: aws_cpi 103 | enable_dedicated_status_worker: true 104 | workers: 11 105 | flush_arp: true 106 | user_management: 107 | local: 108 | users: 109 | - name: admin 110 | password: ((bosh_credentials.director_password)) 111 | ssl: 112 | key: (( bosh_certs.key )) 113 | cert: (( bosh_certs.crt )) 114 | 115 | 116 | hm: 117 | director_account: 118 | user: admin 119 | password: (( bosh_credentials.director_password )) 120 | resurrector_enabled: true 121 | 122 | aws: &aws 123 | access_key_id: (( AWSCredentials.AWS_ACCESS_KEY_ID )) 124 | secret_access_key: (( AWSCredentials.AWS_SECRET_ACCESS_KEY )) 125 | default_key_name: bosh 126 | default_security_groups: [(( SecurityGroups.BOSH_SECURITY_GROUP_NAME ))] 127 | region: (( AWSCredentials.AWS_DEFAULT_REGION )) 128 | 129 | agent: 130 | mbus: (( "nats://nats:" bosh_credentials.nats_password "@10.0.0.6:4222" )) 131 | 132 | ntp: &ntp [0.pool.ntp.org, 1.pool.ntp.org] 133 | 134 | cloud_provider: 135 | template: {name: aws_cpi, release: bosh-aws-cpi} 136 | 137 | ssh_tunnel: 138 | host: (( Resources.MicroEIP )) 139 | port: 22 140 | user: vcap 141 | private_key: ../keypair/id_rsa_bosh 142 | 143 | mbus: (( "https://mbus:" bosh_credentials.mbus_password "@" Resources.MicroEIP ":6868" )) 144 | 145 | properties: 146 | aws: *aws 147 | agent: 148 | mbus: (( "https://mbus:" bosh_credentials.mbus_password "@0.0.0.0:6868" )) 149 | blobstore: {provider: local, path: /var/vcap/micro_bosh/data/cache} 150 | ntp: *ntp 151 | 152 | bosh_credentials: (( merge )) 153 | bosh_certs: (( merge )) 154 | AWSCredentials: (( merge )) 155 | Resources: (( merge )) 156 | SecurityGroups: (( merge )) 157 | -------------------------------------------------------------------------------- /templates/cf/cf-stub-mask.yml: -------------------------------------------------------------------------------- 1 | --- 2 | director_uuid: ~ 3 | meta: ~ 4 | compilation: ~ 5 | networks: ~ 6 | jobs: ~ 7 | properties: ~ 8 | resource_pools: ~ 9 | -------------------------------------------------------------------------------- /templates/concourse/concourse.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: concourse 3 | 4 | director_uuid: (( merge )) 5 | 6 | releases: 7 | - name: concourse 8 | version: latest 9 | - name: garden-linux 10 | version: latest 11 | 12 | networks: 13 | - name: concourse 14 | type: manual 15 | subnets: 16 | - range: 10.0.17.0/24 17 | reserved: [10.0.17.2 - 10.0.17.9] 18 | static: [10.0.17.31] 19 | dns: [10.0.0.2] 20 | gateway: 10.0.17.1 21 | cloud_properties: 22 | security_groups: 23 | - (( .cloud_properties.security_group_name )) 24 | subnet: (( .cloud_properties.subnet_id )) 25 | 26 | blackbox_config: &blackbox_config 27 | expvar: 28 | datadog: 29 | api_key: (( datadog_properties.api_key || nil )) 30 | syslog: 31 | destination: 32 | address: (( syslog_properties.address || nil )) 33 | transport: tls 34 | 35 | jobs: 36 | - name: discovery 37 | instances: 1 38 | resource_pool: discovery 39 | persistent_disk: 1024 40 | templates: 41 | - {release: concourse, name: consul-agent} 42 | - {release: concourse, name: blackbox} 43 | networks: 44 | - name: concourse 45 | static_ips: (( .networks.concourse.subnets.[0].static )) 46 | properties: 47 | blackbox: *blackbox_config 48 | consul: 49 | agent: 50 | mode: server 51 | 52 | - name: atc 53 | instances: 2 54 | resource_pool: orchestration 55 | templates: 56 | - {release: concourse, name: consul-agent} 57 | - {release: concourse, name: blackbox} 58 | - {release: concourse, name: atc} 59 | - {release: concourse, name: tsa} 60 | networks: 61 | - name: concourse 62 | update: 63 | serial: true 64 | max_in_flight: 1 65 | properties: 66 | blackbox: *blackbox_config 67 | atc: 68 | external_url: (( atc_credentials.external_url || nil )) 69 | github_auth: (( atc_credentials.github_auth || nil )) 70 | basic_auth_username: (( atc_credentials.basic_auth_username )) 71 | basic_auth_password: (( atc_credentials.basic_auth_password )) 72 | publicly_viewable: true 73 | postgresql: 74 | database: (( atc_credentials.db_name )) 75 | role: 76 | name: (( atc_credentials.db_user )) 77 | password: ((atc_credentials.db_password )) 78 | tsa: 79 | atc: 80 | username: (( atc_credentials.basic_auth_username )) 81 | password: (( atc_credentials.basic_auth_password )) 82 | consul: 83 | agent: 84 | servers: {lan: (( jobs.discovery.networks.concourse.static_ips ))} 85 | 86 | - name: postgresql 87 | instances: 1 88 | resource_pool: databases 89 | persistent_disk: 102400 90 | templates: 91 | - {release: concourse, name: consul-agent} 92 | - {release: concourse, name: blackbox} 93 | - {release: concourse, name: postgresql} 94 | networks: [{name: concourse}] 95 | properties: 96 | blackbox: *blackbox_config 97 | postgresql: 98 | databases: 99 | - name: (( atc_credentials.db_name )) 100 | roles: 101 | - name: (( atc_credentials.db_user )) 102 | password: ((atc_credentials.db_password )) 103 | consul: 104 | agent: 105 | servers: {lan: (( jobs.discovery.networks.concourse.static_ips ))} 106 | 107 | - name: worker 108 | instances: 2 109 | resource_pool: workers 110 | templates: 111 | - {release: concourse, name: consul-agent} 112 | - {release: concourse, name: blackbox} 113 | - {release: concourse, name: baggageclaim} 114 | - {release: concourse, name: groundcrew} 115 | - {release: garden-linux, name: garden} 116 | networks: [{name: concourse}] 117 | properties: 118 | blackbox: *blackbox_config 119 | garden: 120 | listen_network: tcp 121 | listen_address: 0.0.0.0:7777 122 | btrfs_store_size_mb: 100000 123 | consul: 124 | agent: 125 | servers: {lan: (( jobs.discovery.networks.concourse.static_ips ))} 126 | 127 | resource_pools: 128 | - name: orchestration 129 | network: concourse 130 | stemcell: &stemcell 131 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 132 | version: latest 133 | cloud_properties: 134 | instance_type: c3.large 135 | availability_zone: (( .cloud_properties.az )) 136 | elbs: 137 | - (( .cloud_properties.load_balancer )) 138 | 139 | - name: databases 140 | network: concourse 141 | stemcell: *stemcell 142 | cloud_properties: 143 | instance_type: c3.xlarge 144 | availability_zone: (( .cloud_properties.az )) 145 | 146 | - name: workers 147 | network: concourse 148 | stemcell: *stemcell 149 | cloud_properties: 150 | instance_type: c3.xlarge 151 | availability_zone: (( .cloud_properties.az )) 152 | ephemeral_disk: 153 | size: 1024000 154 | type: gp2 155 | 156 | - name: discovery 157 | network: concourse 158 | stemcell: *stemcell 159 | cloud_properties: 160 | instance_type: m3.medium 161 | availability_zone: (( .cloud_properties.az )) 162 | 163 | compilation: 164 | workers: 4 165 | network: concourse 166 | reuse_compilation_vms: true 167 | cloud_properties: 168 | instance_type: c3.large 169 | availability_zone: (( .cloud_properties.az )) 170 | 171 | update: 172 | canaries: 1 173 | max_in_flight: 3 174 | serial: false 175 | canary_watch_time: 1000-60000 176 | update_watch_time: 1000-60000 177 | 178 | cloud_properties: (( merge )) 179 | atc_credentials: (( merge )) 180 | datadog_properties: (( merge || nil )) 181 | syslog_properties: (( merge || nil )) 182 | -------------------------------------------------------------------------------- /templates/etcd/iaas-settings.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | EtcdSubnet: (( merge )) 3 | 4 | SecurityGroups: 5 | EtcdSecurityGroup: (( merge )) 6 | 7 | AvailabilityZones: 8 | EtcdSubnet: (( merge )) 9 | 10 | iaas_settings: 11 | stemcell: 12 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 13 | version: latest 14 | 15 | compilation_cloud_properties: 16 | instance_type: c3.large 17 | availability_zone: (( AvailabilityZones.EtcdSubnet )) 18 | ephemeral_disk: 19 | size: 10_240 20 | type: gp2 21 | 22 | resource_pool_cloud_properties: 23 | - name: etcd_z1 24 | cloud_properties: 25 | instance_type: c3.large 26 | availability_zone: (( AvailabilityZones.EtcdSubnet )) 27 | ephemeral_disk: 28 | size: 10_240 29 | type: gp2 30 | - name: etcd_z2 31 | cloud_properties: 32 | instance_type: c3.large 33 | availability_zone: (( AvailabilityZones.EtcdSubnet )) 34 | ephemeral_disk: 35 | size: 10_240 36 | type: gp2 37 | stemcell: 38 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 39 | version: latest 40 | 41 | subnet_configs: 42 | - name: etcd1 43 | subnets: 44 | - cloud_properties: 45 | subnet: (( Resources.EtcdSubnet )) 46 | security_groups: 47 | - (( SecurityGroups.EtcdSecurityGroup )) 48 | range: 10.0.16.0/24 49 | gateway: 10.0.16.1 50 | dns: [10.0.0.2] 51 | reserved: 52 | - 10.0.16.2-10.0.16.3 53 | - 10.0.16.10-10.0.16.254 54 | static: 55 | - 10.0.16.4-10.0.16.9 56 | - name: etcd2 57 | subnets: 58 | - cloud_properties: 59 | subnet: (( Resources.EtcdSubnet )) 60 | security_groups: 61 | - (( SecurityGroups.EtcdSecurityGroup )) 62 | range: 10.0.16.0/24 63 | gateway: 10.0.16.1 64 | dns: [10.0.0.2] 65 | reserved: 66 | - 10.0.16.2-10.0.16.3 67 | - 10.0.16.15-10.0.16.254 68 | static: 69 | - 10.0.16.10-10.0.16.14 70 | - name: compilation 71 | subnets: 72 | - cloud_properties: 73 | subnet: (( Resources.EtcdSubnet )) 74 | security_groups: 75 | - (( SecurityGroups.EtcdSecurityGroup )) 76 | range: 10.0.16.0/24 77 | gateway: 10.0.16.1 78 | dns: [10.0.0.2] 79 | reserved: 80 | - 10.0.16.2-10.0.16.3 81 | - 10.0.16.20-10.0.16.254 82 | static: 83 | - 10.0.16.15-10.0.16.19 84 | -------------------------------------------------------------------------------- /templates/final-release/bucket.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "S3 buckets for release blobs", 4 | 5 | "Parameters": { 6 | "BucketName": { 7 | "Description": "Name of the release blobs bucket", 8 | "Type": "String", 9 | "Default": "" 10 | } 11 | }, 12 | 13 | "Resources": { 14 | "BlobstoreUser": { 15 | "Type" : "AWS::IAM::User" 16 | }, 17 | "BlobstoreAccessKey": { 18 | "Type" : "AWS::IAM::AccessKey", 19 | "Properties": { 20 | "UserName": { "Ref": "BlobstoreUser" } 21 | } 22 | }, 23 | "ReleaseBlobsBucket": { 24 | "Type" : "AWS::S3::Bucket", 25 | "Properties" : { 26 | "BucketName" : { "Ref": "BucketName" } 27 | } 28 | }, 29 | "ReleaseBlobsBucketPolicy": { 30 | "Type" : "AWS::S3::BucketPolicy", 31 | "Properties" : { 32 | "PolicyDocument" : { 33 | "Statement" : [ { 34 | "Sid" : "AllowReadForEveryone", 35 | "Action" : [ 36 | "s3:GetObject" 37 | ], 38 | "Effect" : "Allow", 39 | "Principal" : "*", 40 | "Resource" : { "Fn::Join" : [ 41 | "", [ "arn:aws:s3:::", { "Ref" : "ReleaseBlobsBucket" } , "/*" ] 42 | ] } 43 | }, 44 | { 45 | "Sid" : "AllowList", 46 | "Action" : [ 47 | "s3:ListBucket" 48 | ], 49 | "Effect" : "Allow", 50 | "Principal" : "*", 51 | "Resource" : { "Fn::Join" : [ 52 | "", [ "arn:aws:s3:::", { "Ref" : "ReleaseBlobsBucket" } ] 53 | ] } 54 | }, 55 | { 56 | "Sid" : "AllowWrite", 57 | "Action" : [ 58 | "s3:PutObject" 59 | ], 60 | "Effect" : "Allow", 61 | "Principal" : { 62 | "AWS" : { "Fn::GetAtt" : [ "BlobstoreUser", "Arn" ] } 63 | }, 64 | "Resource" : { "Fn::Join" : [ 65 | "", [ "arn:aws:s3:::", { "Ref" : "ReleaseBlobsBucket" } , "/*" ] 66 | ] } 67 | } ] 68 | }, 69 | "Bucket" : { "Ref" : "ReleaseBlobsBucket" } 70 | } 71 | } 72 | }, 73 | "Outputs" : { 74 | "BlobstoreAccessKeyID": { 75 | "Description": "Access Key ID", 76 | "Value": { "Ref": "BlobstoreAccessKey" } 77 | }, 78 | "BlobstoreSecretAccessKey" : { 79 | "Description" : "Secret Key", 80 | "Value": { "Fn::GetAtt" : [ "BlobstoreAccessKey", "SecretAccessKey"]} 81 | }, 82 | "BlobstoreDomain" : { 83 | "Description": "Blobstore Domain", 84 | "Value": { "Ref": "ReleaseBlobsBucket" } 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /templates/turbulence/iaas-settings.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | EtcdSubnet: (( merge )) 3 | 4 | SecurityGroups: 5 | EtcdSecurityGroup: (( merge )) 6 | 7 | AvailabilityZones: 8 | EtcdSubnet: (( merge )) 9 | 10 | iaas_settings: 11 | stemcell: 12 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 13 | version: latest 14 | 15 | compilation_cloud_properties: 16 | instance_type: c3.large 17 | availability_zone: (( AvailabilityZones.EtcdSubnet )) 18 | ephemeral_disk: 19 | size: 10_240 20 | type: gp2 21 | 22 | resource_pool_cloud_properties: 23 | - name: turbulence 24 | cloud_properties: 25 | instance_type: m3.medium 26 | availability_zone: (( AvailabilityZones.EtcdSubnet )) 27 | 28 | stemcell: 29 | name: bosh-aws-xen-hvm-ubuntu-trusty-go_agent 30 | version: latest 31 | 32 | subnet_configs: 33 | - name: turbulence 34 | subnets: 35 | - cloud_properties: 36 | subnet: (( Resources.EtcdSubnet )) 37 | security_groups: 38 | - (( SecurityGroups.EtcdSecurityGroup )) 39 | range: 10.0.16.0/24 40 | gateway: 10.0.16.1 41 | dns: [10.0.0.2] 42 | reserved: 43 | - 10.0.16.2-10.0.16.20 44 | - 10.0.16.22-10.0.16.254 45 | static: 46 | - 10.0.16.21 47 | - name: compilation 48 | subnets: 49 | - cloud_properties: 50 | subnet: (( Resources.EtcdSubnet )) 51 | security_groups: 52 | - (( SecurityGroups.EtcdSecurityGroup )) 53 | range: 10.0.16.0/24 54 | gateway: 10.0.16.1 55 | dns: [10.0.0.2] 56 | reserved: 57 | - 10.0.16.2-10.0.16.14 58 | - 10.0.16.20-10.0.16.254 59 | -------------------------------------------------------------------------------- /templates/turbulence/turbulence-property-overrides.yml: -------------------------------------------------------------------------------- 1 | AWSCredentials: 2 | AWS_ACCESS_KEY_ID: (( merge )) 3 | AWS_SECRET_ACCESS_KEY: (( merge )) 4 | AWS_DEFAULT_REGION: (( merge )) 5 | 6 | bosh_config: 7 | director: (( merge )) 8 | password: (( merge )) 9 | 10 | cf_resources: 11 | security_group_name: (( merge )) 12 | 13 | turbulence_certificates: 14 | certificate: (( merge )) 15 | private_key: (( merge )) 16 | ca_cert: (( merge )) 17 | 18 | property_overrides: 19 | nats_password: password 20 | turbulence_api: 21 | password: turbulence-password 22 | certificate: (( turbulence_certificates.certificate )) 23 | private_key: (( turbulence_certificates.private_key )) 24 | director: 25 | host: (( bosh_config.director )) 26 | username: admin 27 | password: (( bosh_config.password )) 28 | ca_cert: (( turbulence_certificates.ca_cert )) 29 | aws: 30 | access_key_id: (( AWSCredentials.AWS_ACCESS_KEY_ID )) 31 | secret_access_key: (( AWSCredentials.AWS_SECRET_ACCESS_KEY )) 32 | default_key_name: bosh 33 | default_security_groups: 34 | - (( cf_resources.security_group_name )) 35 | region: (( AWSCredentials.AWS_DEFAULT_REGION )) 36 | registry: 37 | username: admin 38 | password: password 39 | host: 127.0.0.1 40 | port: 25777 --------------------------------------------------------------------------------