├── .gitignore ├── LICENSE ├── README.md ├── Selfsigned ├── csr.pem ├── my-certificate.pem └── my-private-key.pem ├── awsdeploy.py ├── awsdeploy1.7.1_prepared_example.yml ├── awsdeploy_example.yml ├── create_dbs.ddl ├── deployment.yml ├── dnsmapping.py ├── ert.yml ├── genhosts.py ├── installation-aws-1.7.yml ├── ipsec-addon-template.yml ├── opsman_mappings.yml ├── opsmanapi.py ├── pcf_1_7_cloudformation_singlefile.json ├── pivnet.py ├── requirements.txt ├── stemplate.py └── wait_util.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | installation_settings_post.yml 3 | installation_settings_pre.yml 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cfawsinit 2 | Automate Creation of CloudFoundry deployment in AWS 3 | 4 | Automates steps outlined in 5 | http://docs.pivotal.io/pivotalcf/customizing/cloudform.html 6 | 7 | ## Progress So far 8 | Given a configuration file you can create a fully working ops manager 9 | with Elastic Runtime tile staged. 10 | 11 | ## Features 12 | 1. Supports Ops Manager 1.6 and 1.7. 13 | 2. Uses network.pivotal.io / pivnet to resolve and fetch needed artifacts. 14 | 3. Idempotence 15 | 4. Creates Elastic Runtime 16 | 5. Registers correct dns entries for the domain 17 | 6. Supports HA configuration in 1.7 18 | 7. Supports installing ipsec bosh addon https://docs.pivotal.io/addon-ipsec/installing.html 19 | 20 | ## TODO 21 | 1. Autocreate self signed ssl cert and arn 22 | 2. Show first failure event when stack creation fails. 23 | 24 | ## Goals 25 | 1. Minimal input configuration file 26 | 2. Input file is resolved to a specific configuration file 27 | 3. Idempotence. The job can be killed and restarted anytime. 28 | 29 | ## Usage 30 | ```shell 31 | mjog@ mac ~/cfawsinit$ ./awsdeploy.py --help 32 | usage: awsdeploy [prepare|deploy] [-h] --action {prepare,deploy} [--cfg CFG] 33 | [--prepared-cfg PREPARED_CFG] 34 | [--timeout TIMEOUT] 35 | 36 | optional arguments: 37 | -h, --help show this help message and exit 38 | --action {prepare,deploy} 39 | --cfg CFG 40 | --prepared-cfg PREPARED_CFG 41 | --timeout TIMEOUT 42 | ``` 43 | ### Minimal input file (awsdeploy.yml) 44 | ```yml 45 | region: us-east-1 46 | email: email@gmail.com 47 | ssh_private_key_path: ~/.ssh/id_rsa 48 | ssh_key_name: mjog 49 | domain: "{ssh_key_name}{uid}.pcf-practice.com" 50 | PIVNET_TOKEN: AAAA-h6BBBBBCotwXFi 51 | ops-manager: 52 | version: latest 53 | beta-ok: true 54 | elastic-runtime: 55 | version: latest 56 | beta-ok: true 57 | ssl_cert_arn: arn:aws:iam::375783000519:server-certificate/mjogCertificate 58 | ``` 59 | ```shell 60 | mjog@ mac ~/cfawsinit $ ./awsdeploy.py --action prepare --cfg awsdeploy.yml --prepared-cfg awsout.yml 61 | ``` 62 | ### This command produces the following fully resolved yaml file 63 | The resolve (prepared) yaml file is used to deploy cloud foundry 64 | ```yml 65 | PIVNET_TOKEN: h6TTTTTTT 66 | __PREPARED__: true 67 | date: 2016-05-11 15:56:34.506636 68 | domain: mjog0f64e4.pcf-practice.com 69 | apps_domain: mjog0f64e4.pcf-practice.com 70 | system_domain: mjog0f64e4.pcf-practice.com 71 | elastic-runtime: 72 | beta-ok: false 73 | cloudformation-template: pcf_1_7_cloudformation.json 74 | cloudformation-template-url: https://network.pivotal.io/api/v2/products/elastic-runtime/releases/1730/product_files/4060/download 75 | cloudformation-template-version: 1.7.1 76 | image-build: 1.7.1-build.3 77 | image-file-url: https://network.pivotal.io/api/v2/products/elastic-runtime/releases/1730/product_files/4542/download 78 | image-filename: cf-1.7.1-build.3.pivotal 79 | version: 1.7.1 80 | template-params: 81 | 20VPCCidr: 10.0.0.0/16 82 | email: mjog@pivotal.io 83 | ops-manager: 84 | ami-id: ami-9cf508fc 85 | ami-name: pivotal-ops-manager-v1.7.1.0 86 | beta-ok: false 87 | version: 1.7.1.0 88 | opsman-password: keepitsimple 89 | opsman-username: admin 90 | rds-password: keepitsimple 91 | rds-username: dbadmin 92 | region: us-west-2 93 | ssh_key_name: mjog 94 | ssh_private_key_path: /Users/mjog/.ssh/piv-ec2-mjog.pem 95 | ssl_cert_file: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 96 | ssl_key_file: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-private-key.pem 97 | ssl_cert_arn: arn:aws:iam::375783000519:server-certificate/mjogCertificate 98 | skip_cert_verify: true 99 | ipsec_instance_certificate: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 100 | ipsec_instance_private_key: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-private-key.pem 101 | ipsec_ca_certificates: 102 | - /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 103 | ipsec_release: ipsec-1.0.0.tgz 104 | stack-name: mjog-pcf-0f64e4 105 | uid: 0f64e4 106 | _START_INSTALLS_: false 107 | ``` 108 | ### The prepared yaml file is used during deploy 109 | Many operations take a long time. You may press Ctrl-C and restart the same command later 110 | 111 | ```shell 112 | mjog@ mac ~/CFWORK/cfinit$ ./awsdeploy.py --action deploy --prepared-cfg ./awsout.yml 113 | Creating stack mjog-pcf-431699 114 | It takes about 22 minutes to create the stack 115 | ^CTraceback (most recent call last): 116 | KeyboardInterrupt 117 | 118 | mjog@ mac ~/CFWORK/cfinit$ ./awsdeploy.py --action deploy --prepared-cfg ./awsout.yml 119 | stack mjog-pcf-431699 is in state CREATE_IN_PROGRESS 120 | ^CTraceback (most recent call last): 121 | KeyboardInterrupt 122 | ``` 123 | After about 20 mins ... 124 | ```shell 125 | mjog@ mac ~/CFWORK/cfinit$ ./awsdeploy.py --action deploy --prepared-cfg ./awsout.yml 126 | stack mjog-pcf-431699 is in state CREATE_COMPLETE 127 | Waiting for instance to start i-2361c5b8 ... 128 | Admin user established. 129 | Configuring Ops Manager 130 | Applying Changes... 131 | Downloading (1.7.0.alpha4) cf-1.7.0-build.58.pivotal to ops manager... done 132 | Installing Elastic runtime (1.7.0.alpha4) cf-1.7.0-build.58.pivotal ... done 133 | Staged {u'product_version': u'1.7.0-build.58', u'name': u'cf'} 134 | Ops manager is now available at https://ec2-51-9-24-33.compute-1.amazonaws.com 135 | ``` 136 | 137 | After a loooong time, Success!! 138 | 139 | As always, if it times out waiting for a certain operation, restart it. 140 | Alternatively use --timeout parameter to give a very large timeout 141 | -------------------------------------------------------------------------------- /Selfsigned/csr.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICxjCCAa4CAQAwaTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5DMQ0wCwYDVQQH 3 | DARDYXJ5MRAwDgYDVQQKDAdQaXZvdGFsMQwwCgYDVQQLDANDRkExHjAcBgkqhkiG 4 | 9w0BCQEWD21qb2dAcGl2b3RhbC5pbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC 5 | AQoCggEBAMNSK9xKFmzUWhjyLRWolQ8ntMeYLy4IFGacJin8ml5rt9WeVJv413N7 6 | n8XnANCvRy/4o3q1pwQmQHhGVBvU/EoHXp/sqLTRlGpXq5q+PMDeq3XJI8A6wm6t 7 | NOXa6JS34nfODIm66nu47WfPUlMp48Mva1kJV9T5MTj+riCvOFUXOqNe9z/usXxH 8 | a44Z4D9KxJIVW6JskXD0+9lNJdfo0JgbiC4EnjphaiGli+z2VsBkV0UXTh4glLnR 9 | wG9qHc27kfSrLVKC5GFKL9yFa8IGop4mt58cei1gcGM6PIRxFzJqqSzVoD7jMm+4 10 | ZQysaa0nts/LNYoOw1Ia/E+D0Lc7TbsCAwEAAaAYMBYGCSqGSIb3DQEJAjEJDAdQ 11 | aXZvdGFsMA0GCSqGSIb3DQEBCwUAA4IBAQB4vGfICmNzdDZnLYOStXBMH1/shckL 12 | xkV93XCc/sstvF2DDQX54BIr63isvekWFb9UkyEpVYycvLGGjFx58rp7ntZWF1eL 13 | ldcI1vOzBkUe/obo90nkW6bwQZoPnjZpUGLaHNmWpct99OhKZ4WO0sUeto8pHiJP 14 | PYsL6G0xc+E64n8RhQoE/GMnI7Tznn2cShOjvikSHEMzEtbroYQkACMCeg1MqhAi 15 | ax36JejNvhFSr5LwXFNLy5c0u2ckOC9WDznAfaCOyN7N0834fxQIgqhZKYwbb+hC 16 | 8OPBqs/Dt5UVWdTsoWfxWx6QlVB3NhrYULcLipKzR9ohYFD+OyJuCBWY 17 | -----END CERTIFICATE REQUEST----- 18 | -------------------------------------------------------------------------------- /Selfsigned/my-certificate.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDTjCCAjYCCQDWZnlQGzgh4TANBgkqhkiG9w0BAQUFADBpMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCTkMxDTALBgNVBAcMBENhcnkxEDAOBgNVBAoMB1Bpdm90YWwx 4 | DDAKBgNVBAsMA0NGQTEeMBwGCSqGSIb3DQEJARYPbWpvZ0BwaXZvdGFsLmlvMB4X 5 | DTE1MTEyNDE2Mzg1MFoXDTE2MTEyMzE2Mzg1MFowaTELMAkGA1UEBhMCVVMxCzAJ 6 | BgNVBAgMAk5DMQ0wCwYDVQQHDARDYXJ5MRAwDgYDVQQKDAdQaXZvdGFsMQwwCgYD 7 | VQQLDANDRkExHjAcBgkqhkiG9w0BCQEWD21qb2dAcGl2b3RhbC5pbzCCASIwDQYJ 8 | KoZIhvcNAQEBBQADggEPADCCAQoCggEBAMNSK9xKFmzUWhjyLRWolQ8ntMeYLy4I 9 | FGacJin8ml5rt9WeVJv413N7n8XnANCvRy/4o3q1pwQmQHhGVBvU/EoHXp/sqLTR 10 | lGpXq5q+PMDeq3XJI8A6wm6tNOXa6JS34nfODIm66nu47WfPUlMp48Mva1kJV9T5 11 | MTj+riCvOFUXOqNe9z/usXxHa44Z4D9KxJIVW6JskXD0+9lNJdfo0JgbiC4Enjph 12 | aiGli+z2VsBkV0UXTh4glLnRwG9qHc27kfSrLVKC5GFKL9yFa8IGop4mt58cei1g 13 | cGM6PIRxFzJqqSzVoD7jMm+4ZQysaa0nts/LNYoOw1Ia/E+D0Lc7TbsCAwEAATAN 14 | BgkqhkiG9w0BAQUFAAOCAQEAtHkypj7tfO2E185OvA2FJd1nCtV1kpmsmM0gVGdT 15 | RXT1+5z0+4naRsFSYAsRfgXoEW3uyRbLpW17ke2p1spA8exOMc6w3UgZP3KFQ5ON 16 | x98CQfaajRbIqHdkIT/KD2eHJrpdXBmyXu9klZnEb4d0mLsaNRWyay1D5kwB3VA4 17 | o6UtWPt/jSO9vt9pYHAtn5W0SQV5WC7AaNGPh9SOlgV4V/+0Udt+5cqAMLBxj6gh 18 | BzZhFTuWK1L8py99G9cwCPziCceg20kUSofI06S1/NCtPoOhQiWhSOmKPYnvbF+h 19 | S+J3UUroQ29bkkxR8CktHkue8cEzNXRkwBNA/h/IuRAVRA== 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /Selfsigned/my-private-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAw1Ir3EoWbNRaGPItFaiVDye0x5gvLggUZpwmKfyaXmu31Z5U 3 | m/jXc3ufxecA0K9HL/ijerWnBCZAeEZUG9T8Sgden+yotNGUalermr48wN6rdckj 4 | wDrCbq005drolLfid84Mibrqe7jtZ89SUynjwy9rWQlX1PkxOP6uIK84VRc6o173 5 | P+6xfEdrjhngP0rEkhVbomyRcPT72U0l1+jQmBuILgSeOmFqIaWL7PZWwGRXRRdO 6 | HiCUudHAb2odzbuR9KstUoLkYUov3IVrwgainia3nxx6LWBwYzo8hHEXMmqpLNWg 7 | PuMyb7hlDKxprSe2z8s1ig7DUhr8T4PQtztNuwIDAQABAoIBAQCCKMsTtMd25JfZ 8 | 0eDbcwlKHSKqc67VFQtLSblg93WDgHL0RtuJSO2Chpp1TjhL2NIulJmUl2LiL/98 9 | Zxl7ppYtWvXiytmuDY/CwgOYTje8K2bFSjGp0z5odMIwfo3JNTUUs1nFmqrRoe2K 10 | 5SyuQNcH5hy6K7C9OwZjPcyZ+Uon4PVcEU+Cpoa8mN3wSW+S6EebxRCGLFs3TIVI 11 | PcvoM3Elcxjs24fkkkqZsamHzsqlsq3HokUtnJ0lf+JLcmKaW3ygIBRjf3fRF+xt 12 | 5OX/IQW6o6ugMKjGfrX4UKURgRPfJWmL6bJ0weEZ59BuaEkft+RMtV2A0SE1kFiy 13 | SfeoXYU5AoGBAO2wt0HtPhy0NWiRBxCRA2wTZpEYwibip+5SX0IFqsSx/cSMmj4q 14 | xvuN0FaJ4mrx9kK1BpmVJjErC5HXxdd12OvBcJNhJVXZemxwvHbNic+k2x/1LayR 15 | sSV0c1zzcq2ZIl7p9dkoLdHetnyQldHmPcfNMb+fykoZHDmFWLmXgeuFAoGBANJd 16 | 7U6pHvkgxcnLbBwiLZANXA0IMTFsZ3BjBQ3/HKrsPYhbWPAhXP51v7NS6MOoCh3/ 17 | kqxFTLHwt8Zo3GmUa0fei8cTK8VnplLY58LOwWPIyvjztoNVQ8Ya/QcY3MINGlp3 18 | kF3QoR6ugpbMzN9ZURCorqfKqECDFZjDXWPpLHg/AoGABFhMoUni7sdkiorMJENi 19 | WqFoKJLZSbiu9S5QS3arDnlqeCNR1n0Vshd/jXVrzCSsKcABZOFTF2cACSR2m1+u 20 | HXEly6vk8NLO6BiPeWR8dm0/DDCBKzxmjpa4XSSeHgpElJOWSOfxHnsBvvkto+6f 21 | hn17wVL8capP18VWP95DI7kCgYAE+a+5IorW/Y1v/l2qNKy7MXWx4TW4o4W2xMDD 22 | 2frFmBzEctShqUhXxFUqWpWiuwCyQO8pH+J2euj9ylEcNiRraLoJlUx9uLvYTcT9 23 | eIJeZ8tVI/53ELcvokfFuTLPbBvpc9Z4QYzt+taf1mwqBCTErhijY0mjbPY/zK8S 24 | w9sHZQKBgEpGfeDIV2S0h7poqN+8+Vb7+5lHbsdas9B3/KsOHqFD7/xaDaqWDGwW 25 | 64BOhKusPuAPhfR+QqjIJ9FHFSCkLNvJb5H2NpxPGm8jMEIcTgWky3JCdWHi8JMK 26 | dwNc6Ae0Q9kjM1MZqMaC20IBm5vlI1cb8adfV2Ws5XOR539Y8uc1 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /awsdeploy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from boto3.session import Session 4 | import botocore.exceptions 5 | import uuid 6 | import yaml 7 | from datetime import datetime 8 | import os 9 | import requests 10 | import requests.exceptions 11 | import pivnet 12 | import opsmanapi 13 | import wait_util 14 | import sys 15 | import json 16 | import dnsmapping 17 | 18 | 19 | # Othwerise urllib3 warns about 20 | # self signed certs 21 | requests.packages.urllib3.disable_warnings() 22 | 23 | THIS_DIR = os.path.dirname(os.path.abspath(__file__)) 24 | 25 | 26 | def get_stack_outputvars(stack, ec2): 27 | ops = {v['OutputKey']: v['OutputValue'] for v in stack.outputs} 28 | # group name is needed in a few places 29 | ops['PcfVmsSecurityGroupName'] = list( 30 | ec2.security_groups.filter( 31 | GroupIds=[ops['PcfVmsSecurityGroupId']]))[0].group_name 32 | 33 | if 'PcfVpc' in ops: 34 | vpc = ec2.Vpc(ops['PcfVpc']) 35 | ops['PcfPrivateSubnetAvailabilityZone'] = list(vpc.subnets.filter( 36 | SubnetIds=[ops['PcfPrivateSubnetId']]))[0].availability_zone 37 | return ops 38 | 39 | 40 | def get_stack(stackName, cff): 41 | try: 42 | stt = list(cff.stacks.filter(StackName=stackName)) 43 | return stt[0] 44 | except botocore.exceptions.ClientError as e: 45 | if "{} does not exist".format(stackName) in\ 46 | e.response['Error']['Message']: 47 | return None 48 | else: 49 | raise 50 | 51 | 52 | def create_stack(opts, ec2, cff, timeout=300): 53 | """ 54 | ensure idempotency 55 | 56 | returns a stack that is either in 57 | 'CREATE_COMPLETE' or 'CREATE_IN_PROGRESS' 58 | """ 59 | st = get_stack(opts['stack-name'], cff) 60 | 61 | if st is not None: 62 | msg = "stack {} is in state {}".format(st.name, st.stack_status) 63 | if st.stack_status in ('CREATE_IN_PROGRESS', 'CREATE_COMPLETE'): 64 | print msg 65 | return st 66 | else: 67 | raise Exception(msg) 68 | 69 | templateBody = open( 70 | opts['elastic-runtime']['cloudformation-template'], 'rt').read() 71 | templ = json.loads(templateBody) 72 | # stack does not exist create it 73 | args = {"01NATKeyPair": opts['ssh_key_name'], 74 | "05RdsUsername": opts['rds-username'], 75 | "06RdsPassword": opts['rds-password'], 76 | "07SSLCertificateARN": opts['ssl_cert_arn']} 77 | if 'opsman-template-url' in opts: 78 | args['08OpsManagerTemplate'] = opts['opsman-template-url'] 79 | if 'template-params' in opts['elastic-runtime']: 80 | args.update(opts['elastic-runtime']['template-params']) 81 | 82 | # filter out args that are not needed 83 | args = {k: v for k, v in args.items() if k in templ["Parameters"]} 84 | paramaters = [{"ParameterKey": k, "ParameterValue": v, 85 | "UsePreviousValue": True} for k, v in args.items()] 86 | tags = [{"Key": "email", "Value": opts["email"]}] 87 | st = cff.create_stack( 88 | StackName=opts['stack-name'], 89 | TemplateBody=templateBody, 90 | Tags=tags, 91 | Parameters=paramaters, 92 | Capabilities=['CAPABILITY_IAM']) 93 | 94 | print "Creating stack", opts['stack-name'], 95 | print ". Takes about 22 minutes to create the stack" 96 | return st 97 | 98 | 99 | def launch_ops_manager(opts, stack_vars, ec2): 100 | """ 101 | given a stack in CREATE_COMPLETE launch an ops manager 102 | """ 103 | insts = list(ec2.instances.filter( 104 | Filters=[{'Name': 'tag:Name', 'Values': ['Ops Manager']}, 105 | {'Name': 'tag:stack-name', 'Values': [opts['stack-name']]}, 106 | {'Name': 'instance-state-name', 'Values': ['running']}])) 107 | if len(insts) == 1: 108 | print insts 109 | print "Found running ops manager {} {}".format( 110 | insts[0].id, get_addr(insts[0])) 111 | return insts[0] 112 | elif len(insts) > 1: 113 | raise Exception("Several Ops Managers running {} for stack {}".format( 114 | insts, opts['stack-name'])) 115 | 116 | # no instance is running, create one 117 | inst = ec2.create_instances( 118 | ImageId=opts['ops-manager']['ami-id'], 119 | MinCount=1, 120 | MaxCount=1, 121 | KeyName=stack_vars.get('PcfKeyPairName', opts['ssh_key_name']), 122 | InstanceType='m3.large', 123 | NetworkInterfaces=[ 124 | {'DeviceIndex': 0, 125 | 'SubnetId': stack_vars['PcfPublicSubnetId'], 126 | 'Groups': [stack_vars['PcfOpsManagerSecurityGroupId']], 127 | 'AssociatePublicIpAddress': True}], 128 | BlockDeviceMappings=[ 129 | {"DeviceName": "/dev/sda1", 130 | "Ebs": {"VolumeSize": 100, 131 | "VolumeType": "gp2" 132 | } 133 | }] 134 | )[0] 135 | inst.wait_until_exists() 136 | inst.create_tags( 137 | Tags=[{'Key': 'Name', 'Value': 'Ops Manager'}, 138 | {'Key': 'stack-name', 'Value': opts['stack-name']}]) 139 | print "Waiting for Ops Manager to start", inst.id, "...", 140 | sys.stdout.flush() 141 | inst.wait_until_running() 142 | inst.reload() 143 | print get_addr(inst) 144 | return inst 145 | 146 | 147 | def configure_ops_manager(opts, stack_vars, inst, vpc): 148 | ops = opsmanapi.get( 149 | "https://"+get_addr(inst), 150 | opts['opsman-username'], 151 | opts['opsman-password'], 152 | os.path.expanduser(opts['ssh_private_key_path']), 153 | stack_vars, 154 | opts['region'], 155 | opts=opts, 156 | vpc=vpc) 157 | 158 | ops.setup() 159 | ops.login() 160 | ops.configure() 161 | return ops 162 | 163 | 164 | def wait_for_stack_ready(st, timeout): 165 | waitFor = wait_util.wait_while( 166 | lambda: st.stack_status == 'CREATE_IN_PROGRESS', 167 | lambda: st.reload()) 168 | waitFor(timeout) 169 | 170 | if st.stack_status == 'CREATE_COMPLETE': 171 | return True 172 | 173 | raise Exception("Stack {} is in bad state {}".format( 174 | st.name, st.stack_status)) 175 | 176 | 177 | def get_addr(inst): 178 | addr = inst.public_dns_name 179 | addr = addr or inst.public_ip_address 180 | return addr 181 | 182 | 183 | def wait_for_opsman_ready(inst, timeout): 184 | addr = get_addr(inst) 185 | 186 | def should_wait(): 187 | try: 188 | resp = requests.head( 189 | "https://{}/".format(addr), 190 | verify=False, timeout=1) 191 | return resp.status_code >= 400 192 | except requests.exceptions.RequestException as ex: 193 | pass 194 | except requests.HTTPError as ex: 195 | print ex 196 | return True 197 | 198 | waitFor = wait_util.wait_while(should_wait) 199 | waitFor(timeout) 200 | 201 | 202 | # 203 | # Main deploy driver function 204 | # 205 | def deploy(prepared_file, timeout=300): 206 | """ 207 | Topline driver 208 | idempotent 209 | """ 210 | opts = yaml.load(open(prepared_file, 'rt')) 211 | 212 | if '__PREPARED__' not in opts: 213 | raise Exception("using 'unprepared' file to deploy." 214 | " First run prepare on it") 215 | 216 | session = Session(profile_name=opts.get('profile_name'), 217 | region_name=opts['region']) 218 | ec2 = session.resource("ec2") 219 | cff = session.resource("cloudformation") 220 | route53 = session.client("route53") 221 | elb = session.client("elb") 222 | 223 | stack = create_stack(opts, ec2, cff) 224 | # ensure that stack is ready 225 | wait_for_stack_ready(stack, timeout) 226 | # 'arn:aws:cloudformation:us-east-1:375783000519:stack/mjog-pcf-42f062-OpsManStack-13R2QRZJCIPTB/ec4e3010-ef99-11e5-9206-500c28604c82' 227 | # opsman_stack_arn = next( 228 | # ll for ll in stack.resource_summaries.iterator() 229 | # if ll.logical_id == 'OpsManStack').physical_resource_id 230 | # opsman_stack = get_stack(opsman_stack_arn.split('/')[1], cff) 231 | stack_vars = get_stack_outputvars(stack, ec2) 232 | # stack_vars.update(get_stack_outputvars(stack, ec2)) 233 | ops_manager_inst = launch_ops_manager(opts, stack_vars, ec2) 234 | # ensure that ops manager is ready to receive requests 235 | wait_for_opsman_ready(ops_manager_inst, timeout) 236 | names = [] 237 | if 'apps_domain' not in opts: 238 | opts['apps_domain'] = "apps." + opts["domain"] 239 | else: 240 | names.append("*."+opts['apps_domain']) 241 | if 'system_domain' not in opts: 242 | opts['system_domain'] = "system." + opts["domain"] 243 | else: 244 | names.append("*."+opts['system_domain']) 245 | 246 | try: 247 | dnsmapping.map_ert_domain( 248 | stackname=opts['stack-name'], 249 | domain=opts['domain'], 250 | route53=route53, 251 | elb=elb, 252 | names=names) 253 | except Exception as ex: 254 | print ex 255 | print "Unable to create dns mappings, create manually" 256 | vpc = ec2.Vpc(stack_vars['PcfVpc']) 257 | ops = configure_ops_manager(opts, stack_vars, ops_manager_inst, vpc) 258 | ops.create_ert_databases(opts) 259 | print "Ops manager is now available at ", ops.url 260 | 261 | if not hasattr(ops, 'install_elastic_runtime'): 262 | print ops, "Does not support deploying elastic runtime on < 1.7" 263 | return 0 264 | 265 | ops.wait_for_deployed('p-bosh', timeout=timeout) 266 | ops.bosh("status") 267 | ops.configure_ipsec() 268 | ops.install_elastic_runtime(opts, timeout) 269 | ops.configure_elastic_runtime(opts, timeout) 270 | ops.bosh("vms", ignore_error='No deployments') 271 | ops.wait_for_deployed('cf', timeout=timeout) 272 | ops.wait_while_install_running(timeout=timeout) 273 | 274 | 275 | AMI_PREFIX = "pivotal-ops-manager-v" 276 | PCF_AWS_OWNERID = '364390758643' 277 | 278 | 279 | def resolve_versions(token, opsman, ert, ec2): 280 | """ 281 | resolve and return 2 new dicts which will replace the old 282 | """ 283 | piv = pivnet.Pivnet(token=token) 284 | opsman_vers = piv._latest( 285 | 'ops-manager', 286 | opsman['beta-ok'], 287 | opsman['version']) 288 | 289 | amidict = { 290 | e.name[len(AMI_PREFIX):]: e for e in 291 | ec2.images.filter( 292 | Owners=[PCF_AWS_OWNERID]) 293 | if e.name.startswith(AMI_PREFIX)} 294 | 295 | ami = next( 296 | amidict[vv['version']] for vv in opsman_vers 297 | if vv['version'] in amidict) 298 | 299 | opsman_out = {'version': opsman_vers[0]['version'], 300 | 'beta-ok': opsman['beta-ok'], 301 | 'ami-id': ami.image_id, 302 | 'ami-name': ami.name} 303 | elastic_runtime_ver = piv.latest('elastic-runtime', 304 | ert['beta-ok'], 305 | ert['version']) 306 | ert_out = {'version': elastic_runtime_ver['version'], 307 | 'beta-ok': ert['beta-ok']} 308 | 309 | files = piv.productfiles('elastic-runtime', elastic_runtime_ver['id']) 310 | cloudformation = next((f for f in files 311 | if f['aws_object_key'].endswith( 312 | "cloudformation.json")), 313 | None) 314 | if cloudformation is None: 315 | cloudformation = next((f for f in files 316 | if 'CloudFormation' in f['name']), 317 | None) 318 | if cloudformation is not None: 319 | filename, dn = piv.download( 320 | elastic_runtime_ver, cloudformation, quiet=True) 321 | ert_out['cloudformation-template-version'] = \ 322 | elastic_runtime_ver['version'] 323 | else: 324 | print "Could not find cloudformation template for ver = {}".format( 325 | elastic_runtime_ver['version']) 326 | print "Trying latest available" 327 | vv = piv.latest_file( 328 | 'elastic-runtime', 329 | ert['beta-ok'], 330 | ert['version'], 331 | selector=lambda x: 332 | x['aws_object_key'].endswith('cloudformation.json')) 333 | if vv is not None: 334 | vr, cloudformation = vv 335 | filename, dn = piv.download(vr, cloudformation, quiet=True) 336 | ert_out['cloudformation-template-version'] = vr['version'] 337 | print "Found cloudformation template for ver = {}".format( 338 | vr['version']) 339 | else: 340 | raise Exception( 341 | ("Could not find link for " 342 | "'cloudformation template for aws' in {} {}".format( 343 | elastic_runtime_ver, files))) 344 | 345 | ert_out['cloudformation-template'] = filename 346 | ert_out['cloudformation-template-url'] = \ 347 | pivnet.href(cloudformation, 'download') 348 | er = next((f for f in files if 'PCF Elastic Runtime' == f['name']), None) 349 | if er is None: 350 | raise Exception( 351 | "Could not find link for 'PCF Elastic Runtime' in " 352 | + str(elastic_runtime_ver) + " "+files) 353 | ert_out['image-file-url'] = \ 354 | pivnet.href(er, 'download') 355 | fname = os.path.basename(er['aws_object_key']) 356 | 357 | ert_out['image-filename'] = fname 358 | # extract build 359 | # cf-1.7.0-build.58.pivotal ==> 1.7.0-build.58 360 | ert_out['image-build'] = \ 361 | fname.partition('-')[2].rpartition('.')[0] 362 | 363 | return opsman_out, ert_out 364 | 365 | 366 | def prepare_deploy(infilename, outfilename): 367 | """ 368 | given infile.yml 369 | fully resolve it and produce outfile.yml 370 | """ 371 | infile = yaml.load(open(infilename, 'rt')) 372 | if 'uid' not in infile: 373 | infile['uid'] = str(uuid.uuid4())[:6] 374 | stack_name = infile['email'].partition('@')[0] + "-pcf-" + infile['uid'] 375 | date = datetime.utcnow() 376 | 377 | outfile = {k: v.format(**infile) 378 | if hasattr(v, 'format') 379 | else v for k, v in infile.items()} 380 | outfile['stack-name'] = stack_name 381 | outfile['date'] = date 382 | 383 | ec2 = Session(profile_name=None, 384 | region_name=outfile['region']).resource("ec2") 385 | 386 | opsman_ver, elastic_runtime_ver =\ 387 | resolve_versions(infile['PIVNET_TOKEN'], 388 | infile['ops-manager'], 389 | infile['elastic-runtime'], 390 | ec2) 391 | 392 | outfile['ops-manager'] = opsman_ver 393 | outfile['elastic-runtime'] = elastic_runtime_ver 394 | 395 | verify_ssh_key(ec2, outfile['ssh_private_key_path'], 396 | outfile['ssh_key_name']) 397 | 398 | outfile['__PREPARED__'] = True 399 | 400 | def set_if_empty(key, val): 401 | if key not in outfile: 402 | outfile[key] = val 403 | 404 | # default params 405 | set_if_empty('rds-username', 'dbadmin') 406 | set_if_empty('rds-password', 'keepitsimple') 407 | set_if_empty('opsman-username', 'admin') 408 | set_if_empty('opsman-password', 'keepitsimple') 409 | set_if_empty('_START_INSTALLS_', True) 410 | 411 | set_if_empty('apps_domain', "apps." + outfile["domain"]) 412 | set_if_empty('system_domain', "system." + outfile["domain"]) 413 | set_if_empty('skip_cert_verify', False) 414 | 415 | yamlout = open(outfilename, 'wt')\ 416 | if outfilename is not None \ 417 | else sys.stdout 418 | 419 | yaml.safe_dump(outfile, yamlout, 420 | indent=2, default_flow_style=False) 421 | return outfile 422 | 423 | 424 | def verify_ssh_key(ec2, key_file, ec2_keypair_name): 425 | """ 426 | ensure that the keypair matches 427 | """ 428 | ec2.KeyPair(ec2_keypair_name) 429 | # TODO verify this fingerprint with the key on disk 430 | open(os.path.expanduser(key_file), 'rt').read() 431 | 432 | 433 | def get_args(): 434 | import argparse 435 | argp = argparse.ArgumentParser("awsdeploy [prepare|deploy]") 436 | argp.add_argument('--action', choices=["prepare", "deploy"], required=True) 437 | argp.add_argument('--cfg') 438 | argp.add_argument('--prepared-cfg') 439 | argp.add_argument('--timeout', type=int, default=360) 440 | return argp 441 | 442 | 443 | def validate_creds(opts): 444 | session = Session(profile_name=opts.get('profile_name'), 445 | region_name=opts['region']) 446 | ec2 = session.resource("ec2") 447 | try: 448 | ec2.meta.client.describe_id_format() 449 | except botocore.exceptions.NoCredentialsError as ex: 450 | print "Missing ~/.aws/credentials ? missing profile_name from cfg file" 451 | print "http://boto3.readthedocs.org/en/latest/guide/configuration.html" 452 | print ex 453 | return False 454 | 455 | try: 456 | pivnet.Pivnet(token=opts['PIVNET_TOKEN']) 457 | except pivnet.AuthException as ex: 458 | print "Get API TOKEN from " 459 | print "https://network.pivotal.io/users/dashboard/edit-profile" 460 | print ex 461 | return False 462 | 463 | return True 464 | 465 | 466 | def main(argv): 467 | args = get_args().parse_args(argv) 468 | 469 | cfg = args.cfg or args.prepared_cfg 470 | if cfg is None: 471 | print "One of --cfg or --prepared-cfg is required" 472 | return -1 473 | opts = yaml.load(open(cfg, 'rt')) 474 | 475 | if validate_creds(opts) is False: 476 | return -1 477 | 478 | if args.action == 'prepare': 479 | prepare_deploy(args.cfg, args.prepared_cfg) 480 | elif args.action == 'deploy': 481 | deploy(args.prepared_cfg, timeout=args.timeout) 482 | 483 | return 0 484 | 485 | 486 | if __name__ == "__main__": 487 | sys.exit(main(sys.argv[1:])) 488 | -------------------------------------------------------------------------------- /awsdeploy1.7.1_prepared_example.yml: -------------------------------------------------------------------------------- 1 | PIVNET_TOKEN: h6TTTTTTT 2 | __PREPARED__: true 3 | date: 2016-05-11 15:56:34.506636 4 | domain: mjog0f64e4.pcf-practice.com 5 | apps_domain: mjog0f64e4.pcf-practice.com 6 | system_domain: mjog0f64e4.pcf-practice.com 7 | elastic-runtime: 8 | beta-ok: false 9 | cloudformation-template: pcf_1_7_cloudformation.json 10 | cloudformation-template-url: https://network.pivotal.io/api/v2/products/elastic-runtime/releases/1730/product_files/4060/download 11 | cloudformation-template-version: 1.7.1 12 | image-build: 1.7.1-build.3 13 | image-file-url: https://network.pivotal.io/api/v2/products/elastic-runtime/releases/1730/product_files/4542/download 14 | image-filename: cf-1.7.1-build.3.pivotal 15 | version: 1.7.1 16 | template-params: 17 | 20VPCCidr: 10.0.0.0/16 18 | email: mjog@pivotal.io 19 | ops-manager: 20 | ami-id: ami-9cf508fc 21 | ami-name: pivotal-ops-manager-v1.7.1.0 22 | beta-ok: false 23 | version: 1.7.1.0 24 | opsman-password: keepitsimple 25 | opsman-username: admin 26 | rds-password: keepitsimple 27 | rds-username: dbadmin 28 | region: us-west-2 29 | ssh_key_name: mjog 30 | ssh_private_key_path: /Users/mjog/.ssh/piv-ec2-mjog.pem 31 | ssl_cert_file: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 32 | ssl_key_file: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-private-key.pem 33 | ssl_cert_arn: arn:aws:iam::375783000519:server-certificate/mjogCertificate 34 | skip_cert_verify: true 35 | ipsec_instance_certificate: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 36 | ipsec_instance_private_key: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-private-key.pem 37 | ipsec_ca_certificates: 38 | - /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 39 | ipsec_release: ipsec-1.0.0.tgz 40 | stack-name: mjog-pcf-0f64e4 41 | uid: 0f64e4 42 | _START_INSTALLS_: false 43 | -------------------------------------------------------------------------------- /awsdeploy_example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | region: us-east-1 3 | email: mjog@pivotal.io 4 | ssh_key_name: mjog 5 | domain: "{ssh_key_name}{uid}.pcf-practice.com" 6 | apps_domain: "apps.{ssh_key_name}{uid}.pcf-practice.com" 7 | system_domain: "system.{ssh_key_name}{uid}.pcf-practice.com" 8 | PIVNET_TOKEN: h6vTzpeCotwXFi 9 | ops-manager: 10 | version: "1.7" 11 | beta-ok: true 12 | elastic-runtime: 13 | version: "1.7" 14 | beta-ok: true 15 | # If you have a real cert, put it here 16 | # otherwise this is a self signed cert 17 | ssh_private_key_path: /Users/mjog/.ssh/piv-ec2-mjog.pem 18 | ssl_cert_file: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 19 | ssl_key_file: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-private-key.pem 20 | ssl_cert_arn: arn:aws:iam::375783000519:server-certificate/mjogCertificate 21 | skip_cert_verify: true 22 | 23 | # If all ipsec related keys are specified 24 | # Ip sec add on is deployed. 25 | # Download the release from pivnet and specify the file 26 | ipsec_instance_certificate: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 27 | ipsec_instance_private_key: /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-private-key.pem 28 | ipsec_ca_certificates: 29 | - /Users/mjog/CFWORK/cfawsinit/Selfsigned/my-certificate.pem 30 | ipsec_release: ipsec-1.0.0.tgz 31 | 32 | # if this is false, the tool will configure ert / opsman 33 | # but will not "apply" 34 | _START_INSTALLS_: false 35 | -------------------------------------------------------------------------------- /create_dbs.ddl: -------------------------------------------------------------------------------- 1 | CREATE database IF NOT EXISTS uaa; 2 | CREATE database IF NOT EXISTS ccdb; 3 | CREATE database IF NOT EXISTS console; 4 | CREATE database IF NOT EXISTS notifications; 5 | CREATE database IF NOT EXISTS autoscale; 6 | CREATE database IF NOT EXISTS app_usage_service; 7 | -------------------------------------------------------------------------------- /deployment.yml: -------------------------------------------------------------------------------- 1 | PIVNET_TOKEN: 2 | __PREPARED__: true 3 | date: 2016-05-11 15:56:34.506636 4 | domain: pcf.cf.shaozhenpcf.com 5 | elastic-runtime: 6 | beta-ok: false 7 | cloudformation-template: pcf_1_7_cloudformation.json 8 | cloudformation-template-url: https://network.pivotal.io/api/v2/products/elastic-runtime/releases/1730/product_files/4060/download 9 | cloudformation-template-version: 1.7.1 10 | image-build: 1.7.1-build.3 11 | image-file-url: https://network.pivotal.io/api/v2/products/elastic-runtime/releases/1730/product_files/4542/download 12 | image-filename: cf-1.7.1-build.3.pivotal 13 | version: 1.7.1 14 | template-params: 15 | 12VpcCIDR: 10.91.144.0/21 16 | 13PublicSubnet: 10.91.144.0/28 17 | 20PublicSubnet2: 10.91.144.32/28 18 | 21PublicSubnet3: 10.91.144.64/28 19 | 14InfrastructureSubnet: 10.91.144.96/28 20 | 18RdsSubnet1: 10.91.144.128/28 21 | 19RdsSubnet2: 10.91.144.160/28 22 | 15PrivateSubnet1: 10.91.145.0/24 23 | 16PrivateSubnet2: 10.91.146.0/24 24 | 17PrivateSubnet3: 10.91.147.0/24 25 | email: sding@pivotal.io 26 | ops-manager: 27 | ami-id: ami-c03bd6ad 28 | ami-name: pivotal-ops-manager-v1.7.1.0 29 | beta-ok: false 30 | version: 1.7.1.0 31 | opsman-password: keepitsimple 32 | opsman-username: admin 33 | rds-password: boshbosh 34 | rds-username: bosh 35 | region: us-east-1 36 | ssh_key_name: pcf-sding 37 | ssh_private_key_path: /Users/sding/work/aws/personal/pcf-sding.pem 38 | ssl_cert_file: /Users/sding/work/source/cfawsinit/Selfsigned/my-certificate.pem 39 | ssl_key_file: /Users/sding/work/source/cfawsinit/Selfsigned/my-private-key.pem 40 | ssl_cert_arn: arn:aws:iam::375783000519:server-certificate/shaozhen-certificate 41 | skip_cert_verify: true 42 | stack-name: shaozhen 43 | uid: 0f64e4 44 | _START_INSTALLS_: false 45 | -------------------------------------------------------------------------------- /dnsmapping.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | 4 | def map_ert_domain(stackname, domain, lbname=None, 5 | route53=None, elb=None, names=None): 6 | """ 7 | maps *.domain, *.system.domain, *.apps.domain 8 | to stackname-pcf-elb load balancer 9 | """ 10 | route53 = route53 or boto3.client('route53') 11 | elb = elb or boto3.client('elb') 12 | prefixes = ["", "*.", "*.system.", "*.apps."] 13 | names = names or [prefix + domain for prefix in prefixes] 14 | 15 | if not domain.endswith('.'): 16 | domain += '.' 17 | 18 | zones = [z for z in route53.list_hosted_zones()['HostedZones'] if domain.endswith(z['Name'])] 19 | for zone in zones: 20 | # based on standard naming 21 | if zone['Config']['PrivateZone'] == False: 22 | lbname = stackname + "-pcf-elb" 23 | if zone['Config']['PrivateZone'] == True: 24 | lbname = stackname + "-pcf-elb-in" 25 | resp = elb.describe_load_balancers( 26 | LoadBalancerNames=[lbname]) 27 | 28 | if len(resp.get('LoadBalancerDescriptions', [])) == 0: 29 | raise Exception(lbname + " Loadbalacer could not be found") 30 | 31 | dnsname = resp['LoadBalancerDescriptions'][0]['DNSName'] 32 | 33 | if not zones: 34 | print domain + " Is not managed in route53" 35 | print "Manually map {} <-- {}".format(dnsname, names) 36 | return 37 | 38 | # ensure names are unique 39 | names = set(names) 40 | 41 | changes = [ 42 | { 43 | 'Action': 'UPSERT', 44 | 'ResourceRecordSet': { 45 | 'Name': name, 46 | 'Type': 'CNAME', 47 | 'TTL': 300, 48 | 'ResourceRecords': [ 49 | { 50 | 'Value': dnsname 51 | }, 52 | ], 53 | } 54 | } for name in names] 55 | 56 | route53.change_resource_record_sets( 57 | HostedZoneId=zone['Id'], 58 | ChangeBatch={ 59 | 'Comment': "for stack="+stackname, 60 | 'Changes': changes 61 | }) 62 | -------------------------------------------------------------------------------- /ert.yml: -------------------------------------------------------------------------------- 1 | --- 2 | products: 3 | - identifier: cf 4 | availability_zone_references: 5 | - az-(( PcfPrivateSubnetAvailabilityZone )) 6 | - az-(( PcfPrivateSubnet2AvailabilityZone )) 7 | - az-(( PcfPrivateSubnet3AvailabilityZone )) 8 | singleton_availability_zone_reference: az-(( PcfPrivateSubnetAvailabilityZone )) 9 | network_reference: network-(( PcfPrivateSubnetId )) 10 | properties: 11 | - identifier: logger_endpoint_port 12 | value: 4443 13 | - identifier: allow_cross_container_traffic 14 | value: true 15 | - identifier: networking_point_of_entry 16 | options: 17 | - identifier: external_ssl 18 | properties: 19 | - identifier: ssl_rsa_certificate 20 | value: 21 | cert_pem: (( v.ssl_cert )) 22 | private_key_pem: (( v.ssl_key )) 23 | - identifier: ssl_ciphers 24 | value: external_ssl 25 | - identifier: system_database 26 | options: 27 | - identifier: external 28 | properties: 29 | - identifier: port 30 | value: (( PcfRdsPort )) 31 | - identifier: host 32 | value: (( PcfRdsAddress )) 33 | - identifier: username 34 | value: (( PcfRdsUsername )) 35 | - identifier: password 36 | value: 37 | secret: (( PcfRdsPassword )) 38 | value: external 39 | - identifier: system_blobstore 40 | options: 41 | - identifier: external 42 | properties: 43 | - identifier: endpoint 44 | value: (( v.s3_endpoint )) 45 | - identifier: buildpacks_bucket 46 | value: (( PcfElasticRuntimeS3BuildpacksBucket )) 47 | - identifier: droplets_bucket 48 | value: (( PcfElasticRuntimeS3DropletsBucket )) 49 | - identifier: packages_bucket 50 | value: (( PcfElasticRuntimeS3PackagesBucket )) 51 | - identifier: resources_bucket 52 | value: (( PcfElasticRuntimeS3ResourcesBucket )) 53 | - identifier: access_key 54 | value: (( PcfIamUserAccessKey )) 55 | - identifier: secret_key 56 | value: 57 | secret: (( PcfIamUserSecretAccessKey )) 58 | value: external 59 | jobs: 60 | - identifier: cloud_controller 61 | properties: 62 | - identifier: system_domain 63 | value: (( Opts_system_domain )) 64 | - identifier: apps_domain 65 | value: (( Opts_apps_domain )) 66 | - identifier: ha_proxy 67 | instance: 68 | identifier: instances 69 | value: 0 70 | properties: 71 | - identifier: ssl_rsa_certificate 72 | value: 73 | cert_pem: (( v.ssl_cert )) 74 | private_key_pem: (( v.ssl_key )) 75 | - identifier: skip_cert_verify 76 | value: (( Opts_skip_cert_verify )) 77 | - identifier: ssl_ciphers 78 | - identifier: router 79 | elb_names: (( Opts_stack-name ))-pcf-elb,(( Opts_stack-name ))-pcf-elb-in 80 | instance: 81 | identifier: instances 82 | value: 3 83 | - identifier: diego_brain 84 | elb_names: (( Opts_stack-name ))-pcf-ssh-elb,(( Opts_stack-name ))-pcf-ssh-elb-in 85 | - identifier: nfs_server 86 | instance: 87 | identifier: instances 88 | value: 0 89 | - identifier: mysql_proxy 90 | instance: 91 | identifier: instances 92 | value: 0 93 | - identifier: mysql 94 | instance: 95 | identifier: instances 96 | value: 0 97 | - identifier: ccdb 98 | instance: 99 | identifier: instances 100 | value: 0 101 | - identifier: uaadb 102 | instance: 103 | identifier: instances 104 | value: 0 105 | - identifier: consoledb 106 | instance: 107 | identifier: instances 108 | value: 0 109 | -------------------------------------------------------------------------------- /genhosts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import socket 3 | import sys 4 | import yaml 5 | from boto3.session import Session 6 | 7 | 8 | def genhosts(elbip, sysdomain, outfile=sys.stdout): 9 | SYS_PREFIXES = [ 10 | 'console', 11 | 'uaa', 12 | 'apps', 13 | 'login', 14 | 'api', 15 | 'loggregator', 16 | 'doppler'] 17 | 18 | print >>outfile, elbip, sysdomain 19 | for prefix in SYS_PREFIXES: 20 | print >>outfile, elbip, prefix+"."+sysdomain 21 | 22 | 23 | def get_elbip(elb, stackname, lbsuffix="-pcf-elb"): 24 | lbname = stackname + lbsuffix 25 | resp = elb.describe_load_balancers( 26 | LoadBalancerNames=[lbname]) 27 | if len(resp.get('LoadBalancerDescriptions', [])) == 0: 28 | raise Exception(lbname + " Loadbalacer could not be found") 29 | dnsname = resp['LoadBalancerDescriptions'][0]['DNSName'] 30 | return socket.gethostbyname(dnsname) 31 | 32 | 33 | def get_args(): 34 | import argparse 35 | argp = argparse.ArgumentParser() 36 | argp.add_argument('--profile') 37 | argp.add_argument('--stack-name') 38 | argp.add_argument('--outfile') 39 | argp.add_argument('--prepared-cfg') 40 | argp.add_argument('--system-domain') 41 | argp.add_argument('--region', default='us-east-1') 42 | return argp 43 | 44 | 45 | def fix_args(args): 46 | if args.prepared_cfg is not None: 47 | opts = yaml.load(open(args.prepared_cfg, 'rt')) 48 | args.system_domain = args.system_domain or opts["system_domain"] 49 | args.stack_name = args.stack_name or opts["stack-name"] 50 | args.region = opts["region"] 51 | 52 | if args.outfile is not None: 53 | args.outfile = open(args.outfile, "wt") 54 | 55 | 56 | def genallhosts(elb, args): 57 | print >>args.outfile, "#"*16, 58 | print >>args.outfile, "Generated for /etc/hosts by cfawsinit", "#"*16 59 | genhosts( 60 | get_elbip(elb, args.stack_name), 61 | args.system_domain, 62 | args.outfile) 63 | print >>args.outfile, get_elbip(elb, args.stack_name, "-pcf-ssh-elb"), 64 | print >>args.outfile, "ssh."+args.system_domain 65 | print >>args.outfile, "#"*16, 66 | print >>args.outfile, "Generated for /etc/hosts by cfawsinit", "#"*16 67 | 68 | 69 | def main(argv): 70 | args = get_args().parse_args(argv) 71 | 72 | if args.prepared_cfg is None and\ 73 | args.system_domain is None: 74 | print ("Either --prepared-cfg or " 75 | "(--system-domain and --stack-name) are required") 76 | return -1 77 | fix_args(args) 78 | session = Session(profile_name=args.profile, region_name=args.region) 79 | elb = session.client("elb") 80 | genallhosts(elb, args) 81 | return 0 82 | 83 | 84 | if __name__ == "__main__": 85 | import sys 86 | sys.exit(main(sys.argv[1:])) 87 | -------------------------------------------------------------------------------- /installation-aws-1.7.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # based on /home/tempest-web/tempest/web/spec/fixtures/installations/current 3 | # (1.7) installation-with-p-bosh-for-aws.yml 4 | installation_schema_version: '1.7' 5 | infrastructure: 6 | type: aws 7 | icmp_checks_enabled: true 8 | director_configuration: 9 | # metrics_ip: 10 | resurrector_enabled: true 11 | ntp_servers: 12 | - 0.amazon.pool.ntp.org 13 | - 1.amazon.pool.ntp.org 14 | - 2.amazon.pool.ntp.org 15 | blobstore_type: s3 16 | s3_blobstore_options: 17 | endpoint: (( v.s3_endpoint )) 18 | bucket_name: (( PcfOpsManagerS3Bucket )) 19 | access_key: (( PcfIamUserAccessKey )) 20 | secret_key: (( PcfIamUserSecretAccessKey )) 21 | database_type: external 22 | external_database_options: 23 | host: (( PcfRdsAddress )) 24 | port: (( PcfRdsPort )) 25 | user: (( PcfRdsUsername )) 26 | password: (( PcfRdsPassword )) 27 | database: (( PcfRdsDBName )) 28 | networks: 29 | - guid: network-(( PcfPrivateSubnetId )) 30 | name: Default 31 | subnets: 32 | - guid: subnet-(( PcfPrivateSubnetId )) 33 | iaas_identifier: (( PcfPrivateSubnetId )) 34 | cidr: 10.0.16.0/20 35 | dns: 10.0.0.2 36 | gateway: 10.0.16.1 37 | reserved_ip_ranges: 10.0.16.1-10.0.16.9 38 | availability_zone_references: 39 | - az-(( PcfPrivateSubnetAvailabilityZone )) 40 | - guid: subnet-(( PcfPrivateSubnet2Id )) 41 | iaas_identifier: (( PcfPrivateSubnet2Id )) 42 | cidr: 10.0.32.0/20 43 | dns: 10.0.0.2 44 | gateway: 10.0.32.1 45 | reserved_ip_ranges: 10.0.32.1-10.0.32.9 46 | availability_zone_references: 47 | - az-(( PcfPrivateSubnet2AvailabilityZone )) 48 | - guid: subnet-(( PcfPrivateSubnet3Id )) 49 | iaas_identifier: (( PcfPrivateSubnet3Id )) 50 | cidr: 10.0.64.0/20 51 | dns: 10.0.0.2 52 | gateway: 10.0.64.1 53 | reserved_ip_ranges: 10.0.64.1-10.0.64.9 54 | availability_zone_references: 55 | - az-(( PcfPrivateSubnet3AvailabilityZone )) 56 | - guid: network-(( PcfInfrastructureSubnetId )) 57 | name: infrastructure 58 | subnets: 59 | - guid: subnet-(( PcfInfrastructureSubnetId )) 60 | iaas_identifier: (( PcfInfrastructureSubnetId )) 61 | cidr: 10.0.0.0/24 62 | dns: 10.0.0.2 63 | gateway: 10.0.0.1 64 | reserved_ip_ranges: 10.0.0.1-10.0.0.9 65 | availability_zone_references: 66 | - az-(( PcfInfrastructureSubnetAvailabilityZone )) 67 | iaas_configuration: 68 | region: (( v.region )) 69 | access_key_id: (( PcfIamUserAccessKey )) 70 | secret_access_key: (( PcfIamUserSecretAccessKey )) 71 | vpc_id: (( PcfVpc )) 72 | security_group: (( PcfVmsSecurityGroupId )) 73 | key_pair_name: (( PcfKeyPairName )) 74 | ssh_private_key: |- 75 | (( v.private_key )) 76 | availability_zones: 77 | - guid: az-(( PcfPrivateSubnetAvailabilityZone )) 78 | iaas_identifier: (( PcfPrivateSubnetAvailabilityZone )) 79 | - guid: az-(( PcfPrivateSubnet2AvailabilityZone )) 80 | iaas_identifier: (( PcfPrivateSubnet2AvailabilityZone )) 81 | - guid: az-(( PcfPrivateSubnet3AvailabilityZone )) 82 | iaas_identifier: (( PcfPrivateSubnet3AvailabilityZone )) 83 | 84 | products: 85 | - guid: p-bosh-guid 86 | installation_name: p-bosh-guid 87 | product_version: 1.7.0.0 88 | prepared: true 89 | jobs: 90 | - guid: director-guid 91 | installation_name: director 92 | vm_credentials: 93 | identity: vcap1 94 | properties: 95 | - value: 96 | identity: vcap 97 | identifier: agent_credentials 98 | - value: 99 | identity: registry 100 | identifier: registry_credentials 101 | - value: 102 | identity: director 103 | identifier: director_credentials 104 | - value: 105 | identity: nats 106 | identifier: nats_credentials 107 | - value: 108 | identity: redis 109 | identifier: redis_credentials 110 | - value: 111 | identity: postgres 112 | identifier: postgres_credentials 113 | - value: 114 | identity: blobstore 115 | identifier: blobstore_credentials 116 | - value: 117 | identity: health_monitor 118 | identifier: health_monitor_credentials 119 | - identifier: director_ssl 120 | instances: 121 | - value: 1 122 | identifier: instances 123 | resources: 124 | - value: 3072 125 | identifier: ram 126 | - value: 16384 127 | identifier: ephemeral_disk 128 | - value: 20480 129 | identifier: persistent_disk 130 | - value: 2 131 | identifier: cpu 132 | identifier: director 133 | identifier: p-bosh 134 | network_reference: network-(( PcfInfrastructureSubnetId )) 135 | singleton_availability_zone_reference: az-(( PcfPrivateSubnetAvailabilityZone )) 136 | availability_zone_references: 137 | - az-(( PcfPrivateSubnetAvailabilityZone )) 138 | - az-(( PcfPrivateSubnet2AvailabilityZone )) 139 | -------------------------------------------------------------------------------- /ipsec-addon-template.yml: -------------------------------------------------------------------------------- 1 | releases: 2 | - {name: ipsec, version: 1.0.0} 3 | 4 | addons: 5 | - name: ipsec-addon 6 | jobs: 7 | - name: ipsec 8 | release: ipsec 9 | properties: 10 | ipsec: 11 | ipsec_subnets: 12 | - 10.0.1.1/20 13 | no_ipsec_subnets: 14 | - 10.0.1.1/32 # gateway 15 | - 10.0.1.10/32 # bosh director 16 | instance_certificate: | 17 | -----BEGIN CERTIFICATE----- 18 | MIIEMDCCAhigAwIBAgIRAIvrBY2TttU/LeRhO+V1t0YwDQYJKoZIhvcNAQELBQAw 19 | ... 20 | -----END CERTIFICATE----- 21 | instance_private_key: | 22 | -----BEGIN RSA PRIVATE KEY----- 23 | MIIEogIBAAKCAQEAtAkBjrzr5x9g0aWgyDEmLd7m9u/ZzpK7UScfANLaN7JiNz3c 24 | ... 25 | -----END RSA PRIVATE KEY----- 26 | ca_certificates: 27 | - | 28 | -----BEGIN CERTIFICATE----- 29 | MIIFCTCCAvGgAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwl0ZXN0 30 | ... 31 | -----END CERTIFICATE----- 32 | - | 33 | -----BEGIN CERTIFICATE----- 34 | MIIFCTCCAvGgAwIBAgIBATAAYDVQQDEwl0ZXN0NBgkqhkiG9w0BAQsFADAUMRIwE 35 | ... 36 | -----END CERTIFICATE----- 37 | prestart_timeout: 30 38 | -------------------------------------------------------------------------------- /opsman_mappings.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - infrastructure/iaas_configuration: 3 | iaas_configuration[access_key_id]: $PcfIamUserAccessKey 4 | iaas_configuration[secret_access_key]: $PcfIamUserSecretAccessKey 5 | iaas_configuration[vpc_id]: $PcfVpc 6 | iaas_configuration[security_group]: $PcfVmsSecurityGroupName 7 | iaas_configuration[key_pair_name]: $PcfKeyPairName 8 | iaas_configuration[ssh_private_key]: $.private_key 9 | iaas_configuration[region]: $.region 10 | 11 | - infrastructure/director_configuration: 12 | director_configuration[ntp_servers_string]: 0.amazon.pool.ntp.org,1.amazon.pool.ntp.org,2.amazon.pool.ntp.org 13 | director_configuration[resurrector_enabled]: 1 14 | director_configuration[blobstore_type]: s3 15 | director_configuration[s3_blobstore_options][endpoint]: $.s3_endpoint 16 | director_configuration[s3_blobstore_options][bucket_name]: $PcfOpsManagerS3Bucket 17 | director_configuration[s3_blobstore_options][access_key]: $PcfIamUserAccessKey 18 | director_configuration[s3_blobstore_options][secret_key]: $PcfIamUserSecretAccessKey 19 | director_configuration[database_type]: external 20 | director_configuration[external_database_options][host]: $PcfRdsAddress 21 | director_configuration[external_database_options][port]: $PcfRdsPort 22 | director_configuration[external_database_options][user]: $PcfRdsUsername 23 | director_configuration[external_database_options][password]: $PcfRdsPassword 24 | director_configuration[external_database_options][database]: $PcfRdsDBName 25 | director_configuration[max_threads]: 5 26 | 27 | - infrastructure/availability_zones: 28 | availability_zones[availability_zones][][iaas_identifier]: $PcfPrivateSubnetAvailabilityZone 29 | 30 | - infrastructure/director/availability_zone_assignment: 31 | director_availability_zone_assignments[singleton_availability_zone]: $PcfPrivateSubnetAvailabilityZone 32 | 33 | - infrastructure/networks: 34 | __IGNORE_ERROR__: ignorable if ICMP is disabled 35 | network[networks][][name]: $PcfPrivateSubnetId 36 | network[networks][][iaas_network_identifier]: $PcfPrivateSubnetId 37 | network[networks][][subnet]: 10.0.16.0/20 38 | network[networks][][reserved_ip_ranges]: 10.0.16.1-10.0.16.9 39 | network[networks][][dns]: 10.0.0.2 40 | network[networks][][gateway]: 10.0.16.1 41 | 42 | - infrastructure/director/single_network_assignment: 43 | director_single_network_assignments[network]: $PcfPrivateSubnetId 44 | -------------------------------------------------------------------------------- /opsmanapi.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | from robobrowser import RoboBrowser 3 | from robobrowser import forms 4 | import yaml 5 | import urlparse 6 | import requests 7 | import requests.auth 8 | import os 9 | import time 10 | from StringIO import StringIO 11 | import copy 12 | import sys 13 | import stemplate 14 | import tempfile 15 | import wait_util 16 | import paramiko 17 | 18 | import pivnet 19 | 20 | # Othwerise urllib3 warns about 21 | # self signed certs 22 | requests.packages.urllib3.disable_warnings() 23 | 24 | 25 | THIS_DIR = os.path.dirname(os.path.abspath(__file__)) 26 | 27 | 28 | def get(*args, **kwargs): 29 | # bs4.find("span", {'class': 'version'}) 30 | resp = requests.head( 31 | args[0]+"/uaa/login", 32 | verify=False, 33 | allow_redirects=False) 34 | # somewhat of a hack 35 | # pre api ops manager does not have /api/v0 endpoints 36 | if resp.status_code == 404: 37 | return OpsManApi(*args, **kwargs) 38 | else: 39 | return OpsManApi17(*args, **kwargs) 40 | 41 | 42 | class OpsManApi(object): 43 | def __init__(self, url, username, password, private_key_file, 44 | stack_vars, region, opts, vpc): 45 | self.url = url 46 | self.username = username 47 | self.password = password 48 | self.auth = requests.auth.HTTPBasicAuth(username, password) 49 | self.private_key = open(private_key_file, "rt").read() 50 | if not self.private_key.endswith('\r\n\r\n'): 51 | self.private_key += '\r\n' 52 | 53 | self.self_signed_key = open( 54 | THIS_DIR+"/Selfsigned/my-private-key.pem", "rt").read() 55 | self.self_signed_cert = open( 56 | THIS_DIR+"/Selfsigned/my-certificate.pem", "rt").read() 57 | 58 | if 'ssl_cert_file' in opts: 59 | self.ssl_cert = open(opts['ssl_cert_file'], "rt").read() 60 | else: 61 | self.ssl_cert = self.self_signed_cert 62 | 63 | if 'ssl_key_file' in opts: 64 | self.ssl_key = open(opts['ssl_key_file'], "rt").read() 65 | else: 66 | self.ssl_key = self.self_signed_key 67 | 68 | self.browser = RoboBrowser(history=True) 69 | self.var = stack_vars 70 | self.region = region 71 | if region == 'us-east-1': 72 | self.s3_endpoint = "https://s3.amazonaws.com" 73 | else: 74 | self.s3_endpoint = "https://s3-{}.amazonaws.com".format(region) 75 | self.action_map_file = THIS_DIR+'/opsman_mappings.yml' 76 | self.opts = opts 77 | self._login = False 78 | if 'PcfKeyPairName' not in self.var: 79 | self.var['PcfKeyPairName'] = self.opts['ssh_key_name'] 80 | self._sshclient = None 81 | self.vpc = vpc 82 | self.director = None 83 | 84 | def setup(self): 85 | setup_data = {'setup[eula_accepted]': 'true', 86 | 'setup[password]': self.password, 87 | 'setup[password_confirmation]': self.password, 88 | 'setup[user_name]': self.username} 89 | resp = requests.post( 90 | self.url + "/api/setup", 91 | data=setup_data, 92 | verify=False, 93 | allow_redirects=False) 94 | 95 | if resp.status_code == 200: 96 | print "Admin user established", resp.json() 97 | elif resp.status_code == 422: 98 | jx = resp.json() 99 | if 'errors' in jx: 100 | raise Exception("Could not establish user: {}". 101 | format(jx['errors'])) 102 | else: 103 | print "Admin user is previously established" 104 | return self 105 | 106 | def login(self): 107 | self.browser.open(self.url + "/login", verify=False) 108 | form = self.browser.get_form(action='/login') 109 | form['login[user_name]'].value = self.username 110 | form['login[password]'].value = self.password 111 | self.browser.submit_form(form) 112 | if self.browser.response.status_code >= 400: 113 | raise Exception("Error login in {}\n{}". 114 | format(self.username, self.browser.response.text)) 115 | return self 116 | 117 | def is_prepared(self, product='p-bosh'): 118 | # check if this is previously prepared 119 | resp = requests.get( 120 | self.url+"/api/installation_settings", 121 | verify=False, 122 | auth=self.auth) 123 | if resp.status_code == 200: 124 | cfg = stemplate.Cfg(resp.json()) 125 | if cfg['products'][product].obj.get('prepared', False) is True: 126 | print product, "is previously prepared" 127 | return True 128 | 129 | return False 130 | 131 | def process_action(self, action, mappings): 132 | if self.is_prepared(): 133 | return self 134 | 135 | self.browser.open(self.url + "/", verify=False) 136 | form = None 137 | suffix = mappings.get('__edit__', "/edit") 138 | self.browser.open(self.url + "/" + action + suffix, verify=False) 139 | form = self.browser.get_form(action='/' + action) 140 | 141 | """ 142 | for suffix in ["/new", "/edit"]: 143 | self.browser.open(self.url + "/" + action + suffix, verify=False) 144 | raise Exception() 145 | form = self.browser.get_form(action='/' + action) 146 | if form is not None: 147 | break 148 | """ 149 | if form is None: 150 | raise Exception("Could not find form for action="+action) 151 | 152 | # forms use ruby hash style params 153 | print form 154 | for k, v in mappings.items(): 155 | if k.startswith("__"): 156 | continue 157 | if k not in form.keys() and "__force__" in mappings: 158 | field = forms.form._parse_fields( 159 | BeautifulSoup(''.format(k)) 160 | )[0] 161 | form.add_field(field) 162 | 163 | form[k].value = v 164 | print k, "=", v 165 | 166 | print form 167 | self.browser.submit_form(form) 168 | soup = BeautifulSoup(self.browser.response.text) 169 | # check if errors-block class is there in the output 170 | # ops manager sometime returns a 200 with and errors block 171 | # in html 172 | # HACK warning 173 | errblock = soup.select('.errors-block') 174 | 175 | if self.browser.response.status_code >= 400 or len(errblock) > 0: 176 | if '__IGNORE_ERROR__' not in mappings or \ 177 | mappings['__IGNORE_ERROR__'] not in errblock[0].text: 178 | raise Exception("Error submitting form " + 179 | self.browser.response.text) 180 | 181 | return self 182 | 183 | def _load_mappings(self, filename): 184 | """ 185 | load mappings and hydrate using self, stack_vars 186 | """ 187 | # TODO use self.resolve_yml after updating opsman_mappings.yml 188 | # template spiff style 189 | mappings = yaml.load(open(filename, 'rt')) 190 | for mapping in mappings: 191 | mp = mapping.values()[0] 192 | for key, val in mp.items(): 193 | if isinstance(val, (bool, int, long)): 194 | continue 195 | if val.startswith("$."): 196 | attrib = val[2:] 197 | if hasattr(self, attrib): 198 | mp[key] = getattr(self, attrib) 199 | else: 200 | raise Exception(val + " Is not provided" 201 | " as a mapping variable") 202 | elif val.startswith("$"): 203 | attrib = val[1:] 204 | if attrib in self.var: 205 | mp[key] = self.var[attrib] 206 | else: 207 | raise Exception(val + " Is not provided" 208 | " as a stack output variable") 209 | return mappings 210 | 211 | def configure(self, filename=None, action=None): 212 | filename = filename or self.action_map_file 213 | 214 | mappings = self._load_mappings(filename) 215 | for mapping in mappings: 216 | ac, mp = mapping.items()[0] 217 | if action is None or action == ac: 218 | self.process_action(ac, mp) 219 | 220 | self.apply_changes() 221 | return self 222 | 223 | def apply_changes(self): 224 | print "Applying Changes" 225 | self.browser.open(self.url, verify=False) 226 | soup = BeautifulSoup(self.browser.response.text) 227 | fx = soup.find('meta', {"name": 'csrf-token'}) 228 | csrf_token = fx.attrs["content"] 229 | 230 | rsp = self.browser.session.put(self.url+"/install", 231 | data={'authenticity_token': csrf_token}) 232 | 233 | if rsp.status_code == 422 and \ 234 | 'Ignore errors and start the install' in rsp.text: 235 | # This happens because of the icmp error 236 | sp = BeautifulSoup(rsp.text) 237 | inst_form = sp.find("form", {"action": "/install"}) 238 | if inst_form is None: 239 | raise Exception("Unable to complete installation") 240 | self.browser.submit_form(forms.form.Form(inst_form)) 241 | 242 | boshprefix = ( 243 | 'BUNDLE_GEMFILE=/home/tempest-web/tempest/web/vendor/bosh/Gemfile ' 244 | 'bundle exec bosh ') 245 | 246 | def boshlogin(self, out=None): 247 | """ 248 | ensure that bosh on opsmanager in logged in 249 | """ 250 | bosh_director = self.get_bosh_director() 251 | boshcmd = ( 252 | self.boshprefix + 253 | '-n ' 254 | '--ca-cert /var/tempest/workspaces/default/root_ca_certificate ' 255 | 'target {}').format(bosh_director) 256 | self.execute_on_opsman( 257 | self.opts, 258 | boshcmd, 259 | out) 260 | 261 | handle, bosh_cfg_path = tempfile.mkstemp() 262 | self.copy_from_opsman(self.opts, ".bosh_config", bosh_cfg_path) 263 | bosh_cfg = yaml.load(open(bosh_cfg_path, 'rt')) 264 | 265 | if 'auth' in bosh_cfg: 266 | all_set = True 267 | for dep, dd in bosh_cfg['auth'].items(): 268 | if 'access_token' not in dd: 269 | all_set = False 270 | if all_set: 271 | print "Bosh prior login present" 272 | self._login = True 273 | return self 274 | 275 | # set deployment and auth fields 276 | if 'deployment' not in bosh_cfg: 277 | deployed_products = { 278 | p['type']: p 279 | for p in self.getJSON("/api/v0/deployed/products")} 280 | if 'cf' in deployed_products: 281 | boshcmd = ( 282 | self.boshprefix + 283 | '-n deployment ' 284 | '/var/tempest/workspaces/' 285 | 'default/deployments/{}.yml').format( 286 | deployed_products['cf']['installation_name']) 287 | 288 | self.execute_on_opsman( 289 | self.opts, 290 | boshcmd, 291 | out) 292 | 293 | respjson = self.getJSON( 294 | "/api/v0/deployed/director/credentials/director_credentials") 295 | creds = respjson['credential']['value'] 296 | 297 | _, cred_path = tempfile.mkstemp() 298 | with open(cred_path, "wt") as handle: 299 | handle.write( 300 | creds['identity'] + 301 | '\n' + 302 | creds['password'] + 303 | '\n') 304 | handle.close() 305 | 306 | self.copy_to_opsman(self.opts, cred_path, "creds.txt") 307 | boshcmd = ( 308 | self.boshprefix + 309 | 'login < creds.txt') 310 | 311 | try: 312 | self.execute_on_opsman( 313 | self.opts, 314 | boshcmd, 315 | out) 316 | except Exception as ex: 317 | if 'Non-interactive UAA login is not supported'\ 318 | not in str(ex): 319 | raise 320 | 321 | self._login = True 322 | return self 323 | 324 | """ 325 | auth = bosh_cfg.get('auth', {}) 326 | target_auth = getUAA_Auth_Header( 327 | self.url, 328 | creds['identity'], 329 | creds['password'], 330 | client_id='bosh_cli' 331 | ) 332 | raise Exception() 333 | auth[bosh_cfg['target']] = target_auth 334 | bosh_cfg['auth'] = auth 335 | """ 336 | def bosh(self, cmd, out=None, ignore_error=None): 337 | if not self._login: 338 | self.boshlogin() 339 | 340 | boshcmd = ( 341 | self.boshprefix + 342 | cmd) 343 | try: 344 | return self.execute_on_opsman( 345 | self.opts, 346 | boshcmd, 347 | out) 348 | except Exception as ex: 349 | if ignore_error is not None and ignore_error\ 350 | not in str(ex): 351 | raise 352 | 353 | def execute_on_opsman(self, opts, cmd, out=None): 354 | stdin, stdout, stderr = self.sshclient.exec_command(cmd) 355 | sout = "" 356 | serr = "" 357 | while stdout.channel.exit_status_ready() is False: 358 | time.sleep(2) 359 | _sout = stdout.read() 360 | print _sout 361 | sout += _sout 362 | serr += stderr.read() 363 | 364 | if stdout.channel.exit_status != 0: 365 | raise Exception(cmd + " failed "+sout + serr) 366 | 367 | print serr 368 | return sout, serr 369 | 370 | @property 371 | def sshclient(self): 372 | if self._sshclient is None: 373 | host = urlparse.urlparse(self.url).netloc 374 | clnt = paramiko.SSHClient() 375 | clnt.set_missing_host_key_policy(paramiko.WarningPolicy()) 376 | clnt.connect( 377 | host, username="ubuntu", 378 | key_filename=self.opts['ssh_private_key_path']) 379 | self._sshclient = clnt 380 | 381 | return self._sshclient 382 | 383 | def copy_to_opsman(self, opts, source, target=None): 384 | scp = self.sshclient.open_sftp() 385 | target = target or os.path.basename(source) 386 | scp.put(source, target) 387 | 388 | def copy_from_opsman(self, opts, source, target): 389 | scp = self.sshclient.open_sftp() 390 | scp.get(source, target) 391 | 392 | def create_ert_databases(self, opts): 393 | file_name = 'create_dbs.ddl' 394 | cmd = 'mysql < {file_name}'.format(file_name=file_name) 395 | MY_CNF = ( 396 | '[client]\n' 397 | 'host={PcfRdsAddress}\n' 398 | 'user={PcfRdsUsername}\n' 399 | 'password={PcfRdsPassword}\n\n' 400 | ).format(**self.var) 401 | 402 | _, my_cnf_path = tempfile.mkstemp() 403 | with open(my_cnf_path, "wt") as _fll: 404 | _fll.write(MY_CNF) 405 | 406 | self.copy_to_opsman(opts, my_cnf_path, ".my.cnf") 407 | self.copy_to_opsman(opts, THIS_DIR+"/"+file_name, file_name) 408 | self.execute_on_opsman(opts, cmd) 409 | 410 | 411 | class AuthException(Exception): 412 | pass 413 | 414 | 415 | class CFAuthHandler(requests.auth.AuthBase): 416 | def __init__(self, username, password): 417 | self.username = username 418 | self.password = password 419 | self.uaa = None 420 | 421 | def __call__(self, req): 422 | if self.uaa is None: 423 | url = req.url[:-len(req.path_url)] 424 | self.uaa = self._uaa(url) 425 | 426 | req.headers['Authorization'] = "Bearer "+self.uaa['access_token'] 427 | return req 428 | 429 | def _uaa(self, url): 430 | resp = requests.post( 431 | url+"/uaa/oauth/token", 432 | verify=False, 433 | data={'grant_type': 'password', 434 | 'username': self.username, 435 | 'password': self.password}, 436 | auth=('opsman', '')) 437 | 438 | if resp.status_code != 200: 439 | exp = AuthException("Unable to authenticate ") 440 | exp.resp = resp 441 | raise exp 442 | 443 | return resp.json() 444 | 445 | 446 | def getUAA_Auth_Header(url, username, password, client_id="opsman"): 447 | # client_id = bosh_cli 448 | resp = requests.post( 449 | url+"/uaa/oauth/token", 450 | verify=False, 451 | data={'grant_type': 'password', 452 | 'username': username, 453 | 'password': password}, 454 | auth=(client_id, '')) 455 | 456 | if resp.status_code != 200: 457 | raise Exception("Unable to authenticate "+resp.text) 458 | 459 | return resp.json() 460 | 461 | 462 | class OpsManApi17(OpsManApi): 463 | 464 | def __init__(self, *args, **kwargs): 465 | super(OpsManApi17, self).__init__(*args, **kwargs) 466 | # self.action_map_file = THIS_DIR+'/opsman_mappings17.yml' 467 | self.action_map_file = THIS_DIR+'/installation-aws-1.7.yml' 468 | 469 | def setup(self, timeout=300): 470 | setup_data = { 471 | 'setup[eula_accepted]': 'true', 472 | 'setup[identity_provider]': 'internal', 473 | 'setup[decryption_passphrase]': self.password, 474 | 'setup[decryption_passphrase_confirmation]': self.password, 475 | 'setup[admin_password]': self.password, 476 | 'setup[admin_password_confirmation]': self.password, 477 | 'setup[admin_user_name]': self.username} 478 | 479 | resp = requests.post( 480 | self.url + "/api/v0/setup", 481 | data=setup_data, 482 | verify=False, 483 | allow_redirects=False) 484 | 485 | if resp.status_code == 200: 486 | print "Admin user established" 487 | 488 | def should_wait(): 489 | try: 490 | self.login() 491 | return False 492 | except Exception as ex: 493 | if hasattr(ex, 'resp') and\ 494 | ex.resp.status_code >= 400: 495 | return True 496 | 497 | raise 498 | 499 | print "Waiting for ops manager login" 500 | waiter = wait_util.wait_while(should_wait) 501 | waiter(timeout) 502 | 503 | elif resp.status_code == 422: 504 | jx = resp.json() 505 | if 'errors' in jx: 506 | raise Exception("Could not establish user: {}". 507 | format(jx['errors'])) 508 | else: 509 | print "Admin user is previously established" 510 | return self 511 | 512 | def login(self): 513 | self.auth = CFAuthHandler(self.username, self.password) 514 | resp = self.get("/eula") 515 | if resp.status_code >= 400: 516 | exp = Exception("Error login in {}\n{}". 517 | format(self.username, resp.text)) 518 | exp.resp = resp 519 | raise exp 520 | return self 521 | 522 | def post(self, uri, **kwargs): 523 | return requests.post( 524 | self.url+uri, 525 | verify=False, 526 | auth=self.auth, 527 | **kwargs) 528 | 529 | def getJSON(self, uri, **kwargs): 530 | resp = self.get(uri, **kwargs) 531 | if resp.status_code < 400: 532 | return resp.json() 533 | else: 534 | raise Exception(resp.text) 535 | 536 | def postJSON(self, uri, **kwargs): 537 | resp = self.post(uri, **kwargs) 538 | if resp.status_code < 400: 539 | return resp.json() 540 | else: 541 | raise Exception(resp.text) 542 | 543 | def get(self, uri, **kwargs): 544 | return requests.get( 545 | self.url+uri, 546 | verify=False, 547 | auth=self.auth, 548 | **kwargs) 549 | 550 | def resolve_yml(self, filename=None): 551 | filename = filename or self.action_map_file 552 | yobj = yaml.load(open(filename, 'rt')) 553 | var = copy.copy(self.var) 554 | if 'PcfKeyPairName' not in var: 555 | var['PcfKeyPairName'] = self.opts['ssh_key_name'] 556 | var.update({"Opts_"+k: v for k, v in self.opts.items()}) 557 | var['v'] = self 558 | stemplate.resolve( 559 | yobj, var, 560 | replacefn=lambda x: x.replace('(( ', '{').replace(' ))', '}')) 561 | 562 | buf = StringIO() 563 | yaml.safe_dump( 564 | yobj, buf, indent=2, default_flow_style=False) 565 | yamlfile = buf.getvalue() 566 | return yamlfile, yobj 567 | 568 | def get_ip_insubnet(self, cidr, num=1): 569 | ipr, _, mask = cidr.partition('/') 570 | vs = map(int, ipr.split('.')) 571 | addr = 0 572 | for idx, val in enumerate(vs[::-1]): 573 | addr += val << (8*idx) 574 | v2 = [] 575 | addr += num # make into gateway 576 | while addr > 0: 577 | v2.append(str(addr % 256)) 578 | addr = addr >> 8 579 | 580 | return '.'.join(v2[::-1]) 581 | 582 | def update_subnet(self, yobj, subnet_id, index): 583 | subnet = list(self.vpc.subnets.filter( 584 | SubnetIds=[subnet_id]))[0] 585 | subnet_gw = self.get_ip_insubnet(subnet.cidr_block) 586 | res_hosts = int( 587 | self.opts.get( 588 | "reserved_hosts", 589 | "9")) 590 | subnet_reserved = "{}-{}".format( 591 | subnet_gw, self.get_ip_insubnet(subnet.cidr_block, res_hosts)) 592 | 593 | dns = self.opts.get( 594 | 'dns', self.get_ip_insubnet(self.vpc.cidr_block, 2)) 595 | 596 | sb = stemplate.Cfg( 597 | yobj['infrastructure']['networks'][index], idfield='iaas_identifier') 598 | subnetobj = sb['subnets'][subnet.id].obj 599 | subnetobj['dns'] = dns 600 | subnetobj['cidr'] = subnet.cidr_block 601 | subnetobj['gateway'] = subnet_gw 602 | subnetobj['reserved_ip_ranges'] = subnet_reserved 603 | 604 | def update_boshnetworkinfo(self, yobj): 605 | self.update_subnet(yobj, self.var["PcfPrivateSubnetId"], 0) 606 | self.update_subnet(yobj, self.var["PcfPrivateSubnet2Id"], 0) 607 | self.update_subnet(yobj, self.var["PcfPrivateSubnet3Id"], 0) 608 | 609 | def update_infranetworkinfo(self, yobj): 610 | self.update_subnet(yobj, self.var["PcfInfrastructureSubnetId"], 1) 611 | 612 | def get_bosh_director(self): 613 | if self.director is None: 614 | current = self.getJSON("/api/installation_settings") 615 | # {'director-guid': {'az-us-west-2b': ['10.0.16.10']}} 616 | pbosh =\ 617 | next(v for k, v in 618 | current["ip_assignments"]["assignments"].items() 619 | if k.startswith('p-bosh')) 620 | 621 | boship = pbosh.values()[0].values()[0] 622 | self.director = boship[0] 623 | return self.director 624 | 625 | def configure_ipsec(self): 626 | if 'ipsec_release' not in self.opts: 627 | print "Ipsec config skipped" 628 | return self 629 | jx = yaml.load(open(THIS_DIR+"/ipsec-addon-template.yml")) 630 | ipsec = jx["addons"][0]["properties"]["ipsec"] 631 | ipsec["instance_certificate"] = open( 632 | self.opts["ipsec_instance_certificate"]).read() 633 | ipsec["instance_private_key"] = open( 634 | self.opts["ipsec_instance_private_key"]).read() 635 | ipsec["ca_certificates"] = [ 636 | open(fl).read() 637 | for fl in self.opts["ipsec_ca_certificates"]] 638 | 639 | current = self.getJSON("/api/installation_settings") 640 | # {'director-guid': {'az-us-west-2b': ['10.0.16.10']}} 641 | pbosh =\ 642 | next(v for k, v in 643 | current["ip_assignments"]["assignments"].items() 644 | if k.startswith('p-bosh')) 645 | 646 | boship = pbosh.values()[0].values()[0] 647 | 648 | # only 1 network in the installation 649 | nw = current["infrastructure"]["networks"][0] 650 | ipsec["ipsec_subnets"] = [s["cidr"] for s in nw["subnets"]] 651 | no_ipsec_subnets = [s["gateway"] for s in nw["subnets"]] + boship 652 | ipsec["no_ipsec_subnets"] = [s+"/32" for s in no_ipsec_subnets] 653 | 654 | # write the addon file 655 | outfilename = "ipsec-addon-prepared.yml" 656 | outfile = open(outfilename, "wt") 657 | yaml.safe_dump( 658 | jx, outfile, indent=2, default_flow_style=False) 659 | 660 | self.copy_to_opsman(self.opts, outfilename, outfilename) 661 | self.copy_to_opsman( 662 | self.opts, 663 | self.opts["ipsec_release"], 664 | "ipsec-release.tgz") 665 | self.bosh("upload release ipsec-release.tgz") 666 | self.bosh("update runtime-config "+outfilename) 667 | self.bosh("bosh releases") 668 | 669 | def configure(self, filename=None, action=None, force=False): 670 | force = force or '_FORCE_PREPARE_' in os.environ 671 | if force or not self.is_prepared(): 672 | _, yobj = self.resolve_yml(filename=filename) 673 | self.update_boshnetworkinfo(yobj) 674 | self.update_infranetworkinfo(yobj) 675 | # update network configuration 676 | buf = StringIO() 677 | yaml.safe_dump( 678 | yobj, buf, indent=2, default_flow_style=False) 679 | yamlfile = buf.getvalue() 680 | 681 | files = {'installation[file]': 682 | ('installation-integration-minimal.yml', 683 | yamlfile, 'text/yaml')} 684 | resp = requests.post( 685 | self.url+"/api/installation_settings", 686 | files=files, 687 | verify=False, 688 | auth=self.auth) 689 | if resp.status_code != 200: 690 | raise Exception("Unable to configure "+resp.text) 691 | 692 | # check if its either deployed or staged 693 | if self.is_deployed('p-bosh'): 694 | return self 695 | 696 | if self.opts.get("_START_INSTALLS_", True) is False and\ 697 | self.is_install_running() is False: 698 | raise Exception("Not Starting install per _START_INSTALLS_ flag\n" 699 | "Verify that the configuration is correct and " 700 | "manually start install") 701 | 702 | print "Starting Ops Manager Director install...", 703 | sys.stdout.flush() 704 | self.apply_changes(in_progress_ok=True) 705 | print "Done" 706 | return self 707 | 708 | # TODO enable errands, it is needed now 709 | def apply_changes(self, in_progress_ok=False, post_args=None): 710 | postdata = [('ignore_warnings', True)] 711 | if post_args is not None: 712 | postdata += post_args 713 | 714 | resp = requests.post( 715 | self.url+'/api/v0/installations', 716 | verify=False, 717 | data=postdata, 718 | auth=self.auth) 719 | if resp.status_code == 422: 720 | if 'install in progress' not in resp.text.lower(): 721 | print resp.text 722 | if in_progress_ok is True: 723 | return 724 | if resp.status_code != 200: 725 | raise Exception( 726 | "Unable to start install, status: {}, error: {}".format( 727 | resp.status_code, resp.text)) 728 | 729 | def stage_elastic_runtime(self, opts, timeout, products): 730 | # TODO if we are running in ec2, don't have to do this 731 | # ssh magic 732 | if 'cf' not in products: 733 | # upload and make available for staging 734 | filename = self._download_ert_to_opsman(opts) 735 | self._add_ert_to_opsman(opts, filename) 736 | 737 | def should_wait(): 738 | products.update({ 739 | p['name']: p 740 | for p in self.getJSON("/api/v0/available_products")}) 741 | return 'cf' not in products 742 | 743 | print "Waiting for elastic runtime to be available for staging" 744 | waiter = wait_util.wait_while(should_wait) 745 | waiter(timeout) 746 | 747 | self.postJSON("/api/v0/staged/products", data=products['cf']) 748 | 749 | staged_products = { 750 | p['type']: p 751 | for p in self.getJSON("/api/v0/staged/products")} 752 | print "Staged", products['cf'] 753 | return staged_products 754 | 755 | def _download_ert_to_opsman(self, opts): 756 | """ 757 | logon to opsman and download the 758 | ert file from pivnet 759 | 760 | it runs the command *from* ops manager 761 | so it can be locally uploaded 762 | """ 763 | piv = pivnet.Pivnet(token=self.opts['PIVNET_TOKEN']) 764 | rel, _, _ = opts['elastic-runtime']['image-file-url'].partition( 765 | 'product_files') 766 | 767 | resp = piv.post(rel+"eula_acceptance") 768 | if resp.status_code != 200: 769 | raise Exception( 770 | "Could not auto accept eula" + 771 | opts['elastic-runtime']['image-file-url'] + 772 | " " + str(resp.headers)) 773 | 774 | filename = opts['elastic-runtime']['image-filename'] 775 | ver = opts['elastic-runtime']['version'] 776 | print "Downloading ({}) {} to ops manager...".format(ver, filename), 777 | sys.stdout.flush() 778 | CMD = "" 779 | if '_NO_CACHE_' not in os.environ: 780 | CMD += '[[ -e {filename} ]] || ' 781 | CMD += ( 782 | 'wget -q -O {filename} --post-data="" ' 783 | '--header="Authorization: Token {token}" {url}') 784 | 785 | cmd = CMD.format( 786 | filename=filename, 787 | token=opts['PIVNET_TOKEN'], 788 | url=opts['elastic-runtime']['image-file-url']) 789 | 790 | self.execute_on_opsman(opts, cmd) 791 | print "done" 792 | return filename 793 | 794 | def _add_ert_to_opsman(self, opts, ert_file): 795 | # TODO ensure that ops manager is ready to install ert 796 | CMD = ( 797 | 'curl -s -k https://localhost/api/v0/available_products ' 798 | '-F \'product[file]=@{filename}\' ' 799 | '-X POST ' 800 | '-H "Authorization: {auth}"') 801 | 802 | cmd = CMD.format( 803 | filename=ert_file, 804 | auth="Bearer "+self.auth.uaa['access_token']) 805 | 806 | ver = opts['elastic-runtime']['version'] 807 | print "Installing Elastic runtime ({}) {} ...".format(ver, ert_file), 808 | sys.stdout.flush() 809 | self.execute_on_opsman(opts, cmd) 810 | print "done" 811 | 812 | def is_staged(self, product): 813 | products = { 814 | p['type']: p 815 | for p in self.getJSON( 816 | "/api/v0/staged/products")} 817 | return product in products 818 | 819 | def is_deployed(self, product): 820 | products = { 821 | p['type']: p 822 | for p in self.getJSON( 823 | "/api/v0/deployed/products")} 824 | return product in products 825 | 826 | def wait_for_deployed(self, product, timeout=400): 827 | """ 828 | wait until a product is deployed 829 | """ 830 | def should_wait(): 831 | return not self.is_deployed(product) 832 | 833 | if should_wait() is False: 834 | print product, "previously deployed" 835 | return 836 | 837 | print "Waiting for {} to deploy...".format(product), 838 | sys.stdout.flush() 839 | waitFor = wait_util.wait_while(should_wait) 840 | waitFor(timeout) 841 | print "done" 842 | 843 | def is_install_running(self): 844 | instno = self.find_lastest_install() 845 | 846 | if instno == -1: 847 | return False 848 | respjson = self.getJSON("/api/v0/installations/{}".format(instno)) 849 | return respjson.get("status", "success") == "running" 850 | 851 | def find_lastest_install(self): 852 | instno = -1 853 | respjson = self.getJSON("/api/v0/installations") 854 | if len(respjson["installations"]) > 0: 855 | instno = max([inst["id"] for inst in respjson["installations"]]) 856 | 857 | return instno 858 | 859 | def wait_while_install_running(self, timeout=400): 860 | """ 861 | if there is an ongoing install, wait for 862 | it to finish 863 | """ 864 | instno = self.find_lastest_install() 865 | 866 | if instno == -1: 867 | return 868 | 869 | def should_wait(): 870 | respjson = self.getJSON("/api/v0/installations/{}".format(instno)) 871 | return respjson.get("status", "success") == "running" 872 | 873 | print "Waiting while install {} is running...".format(instno), 874 | sys.stdout.flush() 875 | waitFor = wait_util.wait_while(should_wait) 876 | waitFor(timeout) 877 | print "done" 878 | print self.getJSON("/api/v0/installations/{}".format(instno)) 879 | 880 | def install_elastic_runtime(self, opts, timeout=400): 881 | """ 882 | idempotent just like everything else 883 | """ 884 | # prereq is 'p-bosh' is fully deployed 885 | 886 | self.wait_for_deployed('p-bosh', timeout=timeout) 887 | 888 | # check if it is previously installed. 889 | deployed_products = { 890 | p['type']: p 891 | for p in self.getJSON("/api/v0/deployed/products")} 892 | products = { 893 | p['name']: p 894 | for p in self.getJSON("/api/v0/available_products")} 895 | 896 | if 'cf' in deployed_products: 897 | print "Elastic runtime is deployed", products['cf'] 898 | return 899 | 900 | staged_products = { 901 | p['type']: p 902 | for p in self.getJSON("/api/v0/staged/products")} 903 | 904 | if 'cf' in staged_products: 905 | print "Elastic runtime ", products['cf']['product_version'], 906 | print "is previously staged" 907 | else: 908 | staged_products = self.stage_elastic_runtime( 909 | opts, timeout, products) 910 | return self 911 | 912 | def configure_elastic_runtime(self, opts, timeout=300, force=False): 913 | force = force or 'FORCE_PREPARE' in os.environ 914 | if not force and self.is_prepared('cf'): 915 | return self 916 | 917 | current = self.getJSON("/api/installation_settings") 918 | cfg_current = stemplate.Cfg(current) 919 | yaml.safe_dump( 920 | current, 921 | open('installation_settings_pre.yml', 'wt'), 922 | indent=2, default_flow_style=False) 923 | _, yobj = self.resolve_yml(filename=THIS_DIR+"/ert.yml") 924 | stemplate.cfgmerge( 925 | cfg_current, 926 | stemplate.Cfg(yobj)) 927 | yaml.safe_dump( 928 | current, 929 | open('installation_settings_post.yml', 'wt'), 930 | indent=2, default_flow_style=False) 931 | buf = StringIO() 932 | yaml.safe_dump( 933 | current, buf, indent=2, default_flow_style=False) 934 | yamlfile = buf.getvalue() 935 | files = {'installation[file]': 936 | ('installation-integration-minimal.yml', 937 | yamlfile, 'text/yaml')} 938 | prod_guid = cfg_current['products']['cf'].obj['guid'] 939 | # FIXME get this from the manifest 940 | # This can be done by unzip cf-1.7.0-build.167.pivotal metadata/cf.yml 941 | # and reading from post_deploy_errands key in that yml 942 | enabled_errands =\ 943 | ['smoke-tests', 'push-apps-manager', 'notifications', 944 | 'notifications-ui', 'autoscaling', 'autoscaling-register-broker'] 945 | post_args =\ 946 | [("enabled_errands[{}][post_deploy_errands][]". 947 | format(prod_guid), err) for err in enabled_errands] 948 | self.postJSON( 949 | "/api/installation_settings", 950 | files=files) 951 | 952 | if self.opts.get("_START_INSTALLS_", True) is False and\ 953 | self.is_install_running() is False: 954 | raise Exception("Not Starting install per _START_INSTALLS_ flag\n" 955 | "Verify that the configuration is correct and " 956 | "manually start install") 957 | 958 | self.apply_changes(post_args=post_args) 959 | return self 960 | -------------------------------------------------------------------------------- /pcf_1_7_cloudformation_singlefile.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Cloudformation template for configuring Pivotal Cloud Foundry on AWS", 4 | "Parameters": { 5 | "03OpsManagerIngress": { 6 | "Type": "String", 7 | "Default": "0.0.0.0/0", 8 | "Description": "CIDR range allowed to connect to Ops Manager instance" 9 | }, 10 | "04RdsDBName": { 11 | "Type": "String", 12 | "MinLength": "4", 13 | "Default": "bosh", 14 | "Description": "BOSH database name" 15 | }, 16 | "05RdsUsername": { 17 | "Type": "String", 18 | "Description": "BOSH database username" 19 | }, 20 | "06RdsPassword": { 21 | "Type": "String", 22 | "NoEcho": "true", 23 | "MinLength": "8", 24 | "Description": "BOSH database password" 25 | }, 26 | "07SSLCertificateARN": { 27 | "Type": "String", 28 | "Description": "ARN for pre-uploaded SSL certificate" 29 | }, 30 | "09ElbPrefix": { 31 | "Type": "String", 32 | "Default": "", 33 | "Description": "Prefix for the name of the ELBs generated. NOTE: Leave empty to use default prefix of AWS::StackName" 34 | }, 35 | "10AllowHttpOnElb": { 36 | "Type": "String", 37 | "Default": "true", 38 | "AllowedValues": ["true", "false"], 39 | "Description": "Allow HTTP traffic on PCF-ELB port 80. Default: true." 40 | }, 41 | "12VpcCIDR": { 42 | "Type": "String", 43 | "Default": "10.0.0.0/16", 44 | "Description": "VPC Network CIDR" 45 | }, 46 | "13PublicSubnet": { 47 | "Type": "String", 48 | "Default": "10.0.0.0/26", 49 | "Description": "Infrastructure Subnet CIDR" 50 | }, 51 | "14InfrastructureSubnet": { 52 | "Type": "String", 53 | "Default": "10.0.0.192/26", 54 | "Description": "Infrastructure Subnet CIDR" 55 | }, 56 | "15PrivateSubnet1": { 57 | "Type": "String", 58 | "Default": "10.0.2.0/24", 59 | "Description": "Private Subnet1" 60 | }, 61 | "16PrivateSubnet2": { 62 | "Type": "String", 63 | "Default": "10.0.3.0/24", 64 | "Description": "Private Subnet2" 65 | }, 66 | "17PrivateSubnet3": { 67 | "Type": "String", 68 | "Default": "10.0.4.0/24", 69 | "Description": "Private Subnet3" 70 | }, 71 | "18RdsSubnet1": { 72 | "Type": "String", 73 | "Default": "10.0.5.0/24", 74 | "Description": "Rds Subnet" 75 | }, 76 | "19RdsSubnet2": { 77 | "Type": "String", 78 | "Default": "10.0.6.0/24", 79 | "Description": "Rds Subnet2" 80 | }, 81 | "20PublicSubnet2": { 82 | "Type": "String", 83 | "Default": "10.0.0.64/26", 84 | "Description": "Public Subnet2 CIDR" 85 | }, 86 | "21PublicSubnet3": { 87 | "Type": "String", 88 | "Default": "10.0.0.128/26", 89 | "Description": "Public Subnet3 CIDR" 90 | } 91 | }, 92 | "Conditions": { 93 | "ElbPrefixProvided": { 94 | "Fn::Not": [ 95 | { 96 | "Fn::Equals" : [{"Ref" : "09ElbPrefix" }, ""] 97 | } 98 | ] 99 | }, 100 | "AllowHTTPTrafficOnELB": { 101 | "Fn::Equals" : [{"Ref" : "10AllowHttpOnElb" }, "true"] 102 | }, 103 | "CreateRDS": { 104 | "Fn::And":[ 105 | { "Fn::Not": [ { "Fn::Equals": [ { "Ref": "04RdsDBName"}, "" ]}]}, 106 | { "Fn::Not": [ { "Fn::Equals": [ { "Ref": "05RdsUsername"}, "" ]}]}, 107 | { "Fn::Not": [ { "Fn::Equals": [ { "Ref": "06RdsPassword"}, "" ]}]} 108 | ] 109 | } 110 | }, 111 | "Resources": { 112 | "PcfInternetGateway": { 113 | "Type": "AWS::EC2::InternetGateway" 114 | }, 115 | "PcfNatGateway" : { 116 | "Type" : "AWS::EC2::NatGateway", 117 | "Properties" : { 118 | "AllocationId" : { "Fn::GetAtt" : [ "PcfNatElasticIp", "AllocationId" ] }, 119 | "SubnetId": {"Ref": "PcfPublicSubnet"} 120 | } 121 | }, 122 | "PcfVpc": { 123 | "Type": "AWS::EC2::VPC", 124 | "Properties": { 125 | "CidrBlock": {"Ref": "12VpcCIDR"}, 126 | "EnableDnsSupport": "true", 127 | "EnableDnsHostnames": "true", 128 | "InstanceTenancy": "default", 129 | "Tags": [ 130 | { 131 | "Key": "Name", 132 | "Value": { 133 | "Fn::Join": [ "-", 134 | [ 135 | { "Ref" : "AWS::StackName" }, 136 | "pcf-vpc" 137 | ] 138 | ] 139 | } 140 | } 141 | ] 142 | } 143 | }, 144 | "PcfVpcGatewayAttachment": { 145 | "Type": "AWS::EC2::VPCGatewayAttachment", 146 | "DependsOn": [ 147 | "PcfInternetGateway", 148 | "PcfVpc" 149 | ], 150 | "Properties": { 151 | "InternetGatewayId": { 152 | "Ref": "PcfInternetGateway" 153 | }, 154 | "VpcId": { 155 | "Ref": "PcfVpc" 156 | } 157 | } 158 | }, 159 | "PcfPublicSubnet": { 160 | "Type": "AWS::EC2::Subnet", 161 | "Properties": { 162 | "AvailabilityZone": { 163 | "Fn::Select": [ 164 | "0", 165 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 166 | ] 167 | }, 168 | "CidrBlock": {"Ref": "13PublicSubnet"}, 169 | "VpcId": { 170 | "Ref": "PcfVpc" 171 | }, 172 | "Tags": [ 173 | { 174 | "Key": "Name", 175 | "Value": "pcf-public-subnet" 176 | }, 177 | { 178 | "Key": "subnet_type", 179 | "Value": "public" 180 | } 181 | ] 182 | } 183 | }, 184 | "PcfPublicSubnet2": { 185 | "Type": "AWS::EC2::Subnet", 186 | "Properties": { 187 | "AvailabilityZone": { 188 | "Fn::Select": [ 189 | "1", 190 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 191 | ] 192 | }, 193 | "CidrBlock": {"Ref": "20PublicSubnet2"}, 194 | "VpcId": { 195 | "Ref": "PcfVpc" 196 | }, 197 | "Tags": [ 198 | { 199 | "Key": "Name", 200 | "Value": "pcf-public-subnet2" 201 | }, 202 | { 203 | "Key": "subnet_type", 204 | "Value": "public" 205 | } 206 | ] 207 | } 208 | }, 209 | "PcfPublicSubnet3": { 210 | "Type": "AWS::EC2::Subnet", 211 | "Properties": { 212 | "AvailabilityZone": { 213 | "Fn::Select": [ 214 | "2", 215 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 216 | ] 217 | }, 218 | "CidrBlock": {"Ref": "21PublicSubnet3"}, 219 | "VpcId": { 220 | "Ref": "PcfVpc" 221 | }, 222 | "Tags": [ 223 | { 224 | "Key": "Name", 225 | "Value": "pcf-public-subnet3" 226 | }, 227 | { 228 | "Key": "subnet_type", 229 | "Value": "public" 230 | } 231 | ] 232 | } 233 | }, 234 | "PcfInfrastructureSubnet": { 235 | "Type": "AWS::EC2::Subnet", 236 | "Properties": { 237 | "AvailabilityZone": { 238 | "Fn::Select": [ 239 | "0", 240 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 241 | ] 242 | }, 243 | "CidrBlock": {"Ref": "14InfrastructureSubnet"}, 244 | "VpcId": { 245 | "Ref": "PcfVpc" 246 | }, 247 | "Tags": [ 248 | { 249 | "Key": "Name", 250 | "Value": "pcf-infrastructure-subnet" 251 | }, 252 | { 253 | "Key": "subnet_type", 254 | "Value": "bosh" 255 | } 256 | ] 257 | } 258 | }, 259 | "PcfPrivateSubnet": { 260 | "Type": "AWS::EC2::Subnet", 261 | "Properties": { 262 | "AvailabilityZone": { 263 | "Fn::Select": [ 264 | "0", 265 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 266 | ] 267 | }, 268 | "CidrBlock": {"Ref": "15PrivateSubnet1"}, 269 | "VpcId": { 270 | "Ref": "PcfVpc" 271 | }, 272 | "Tags": [ 273 | { 274 | "Key": "Name", 275 | "Value": "pcf-private-subnet" 276 | }, 277 | { 278 | "Key": "subnet_type", 279 | "Value": "private" 280 | } 281 | ] 282 | } 283 | }, 284 | "PcfPrivateSubnet2": { 285 | "Type": "AWS::EC2::Subnet", 286 | "Properties": { 287 | "AvailabilityZone": { 288 | "Fn::Select": [ 289 | "1", 290 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 291 | ] 292 | }, 293 | "CidrBlock": {"Ref": "16PrivateSubnet2"}, 294 | "VpcId": { 295 | "Ref": "PcfVpc" 296 | }, 297 | "Tags": [ 298 | { 299 | "Key": "Name", 300 | "Value": "pcf-private-subnet2" 301 | }, 302 | { 303 | "Key": "subnet_type", 304 | "Value": "private" 305 | } 306 | ] 307 | } 308 | }, 309 | "PcfPrivateSubnet3": { 310 | "Type": "AWS::EC2::Subnet", 311 | "Properties": { 312 | "AvailabilityZone": { 313 | "Fn::Select": [ 314 | "2", 315 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 316 | ] 317 | }, 318 | "CidrBlock": {"Ref": "17PrivateSubnet3"}, 319 | "VpcId": { 320 | "Ref": "PcfVpc" 321 | }, 322 | "Tags": [ 323 | { 324 | "Key": "Name", 325 | "Value": "pcf-private-subnet3" 326 | }, 327 | { 328 | "Key": "subnet_type", 329 | "Value": "private" 330 | } 331 | ] 332 | } 333 | }, 334 | "PcfNatElasticIp": { 335 | "Type": "AWS::EC2::EIP", 336 | "DependsOn": "PcfVpcGatewayAttachment", 337 | "Properties": { 338 | "Domain": "vpc" 339 | } 340 | }, 341 | "PcfOpsManagerS3Bucket": { 342 | "Type": "AWS::S3::Bucket", 343 | "Properties": { 344 | "Tags": [ 345 | { 346 | "Key": "Name", 347 | "Value": "PCF Ops Manager S3 Bucket" 348 | } 349 | ] 350 | } 351 | }, 352 | "PcfIamInstanceProfile": { 353 | "Type": "AWS::IAM::InstanceProfile", 354 | "Properties": { 355 | "Path": "/", 356 | "Roles": [ { 357 | "Ref": "PcfIamRole" 358 | } ] 359 | } 360 | }, 361 | "PcfIamRole": { 362 | "Type": "AWS::IAM::Role", 363 | "DependsOn": [ 364 | "PcfVpc", 365 | "PcfOpsManagerS3Bucket" 366 | ], 367 | "Properties": { 368 | "AssumeRolePolicyDocument": { 369 | "Version" : "2012-10-17", 370 | "Statement": [ { 371 | "Effect": "Allow", 372 | "Principal": { 373 | "Service": [ "ec2.amazonaws.com" ] 374 | }, 375 | "Action": [ "sts:AssumeRole" ] 376 | } ] 377 | }, 378 | "Policies": [ ] 379 | } 380 | }, 381 | "PcfIamPolicy" : { 382 | "Type": "AWS::IAM::ManagedPolicy", 383 | "Properties": { 384 | "PolicyDocument" : { 385 | "Version": "2012-10-17", 386 | "Statement": [ 387 | { 388 | "Effect": "Deny", 389 | "Action": [ 390 | "iam:Add*", 391 | "iam:Attach*", 392 | "iam:ChangePassword", 393 | "iam:Create*", 394 | "iam:DeactivateMFADevice", 395 | "iam:Delete*", 396 | "iam:Detach*", 397 | "iam:EnableMFADevice", 398 | "iam:GenerateCredentialReport", 399 | "iam:GenerateServiceLastAccessedDetails", 400 | "iam:GetAccessKeyLastUsed", 401 | "iam:GetAccountAuthorizationDetails", 402 | "iam:GetAccountPasswordPolicy", 403 | "iam:GetAccountSummary", 404 | "iam:GetContextKeysForCustomPolicy", 405 | "iam:GetContextKeysForPrincipalPolicy", 406 | "iam:GetCredentialReport", 407 | "iam:GetGroup", 408 | "iam:GetGroupPolicy", 409 | "iam:GetLoginProfile", 410 | "iam:GetOpenIDConnectProvider", 411 | "iam:GetPolicy", 412 | "iam:GetPolicyVersion", 413 | "iam:GetRole", 414 | "iam:GetRolePolicy", 415 | "iam:GetSAMLProvider", 416 | "iam:GetSSHPublicKey", 417 | "iam:GetServerCertificate", 418 | "iam:GetServiceLastAccessedDetails", 419 | "iam:GetUser", 420 | "iam:GetUserPolicy", 421 | "iam:List*", 422 | "iam:Put*", 423 | "iam:RemoveClientIDFromOpenIDConnectProvider", 424 | "iam:RemoveRoleFromInstanceProfile", 425 | "iam:RemoveUserFromGroup", 426 | "iam:ResyncMFADevice", 427 | "iam:SetDefaultPolicyVersion", 428 | "iam:SimulateCustomPolicy", 429 | "iam:SimulatePrincipalPolicy", 430 | "iam:Update*" 431 | ], 432 | "Resource": [ 433 | "*" 434 | ] 435 | }, 436 | { 437 | "Sid": "AllowToGetInfoAboutCurrentInstanceProfile", 438 | "Effect": "Allow", 439 | "Action": [ 440 | "iam:GetInstanceProfile" 441 | ], 442 | "Resource": [ 443 | { 444 | "Fn::GetAtt": [ 445 | "PcfIamInstanceProfile", 446 | "Arn" 447 | ] 448 | } 449 | ] 450 | 451 | }, 452 | { 453 | "Sid": "AllowToCreateInstanceWithCurrentInstanceProfile", 454 | "Effect": "Allow", 455 | "Action": [ 456 | "iam:PassRole" 457 | ], 458 | "Resource": [ 459 | { 460 | "Fn::GetAtt": [ 461 | "PcfIamRole", 462 | "Arn" 463 | ] 464 | } 465 | ] 466 | }, 467 | { 468 | "Sid": "OpsManagerS3Permissions", 469 | "Effect": "Allow", 470 | "Action": ["s3:*"], 471 | "Resource": [ 472 | { 473 | "Fn::Join": [ 474 | "", 475 | [ 476 | "arn:aws:s3:::", 477 | {"Ref": "PcfOpsManagerS3Bucket"} 478 | ] 479 | ] 480 | }, 481 | { 482 | "Fn::Join": [ 483 | "", 484 | [ 485 | "arn:aws:s3:::", 486 | {"Ref": "PcfOpsManagerS3Bucket"}, 487 | "/*" 488 | ] 489 | ] 490 | } 491 | ] 492 | }, 493 | { 494 | "Sid": "OpsManagerEc2Permissions", 495 | "Effect": "Allow", 496 | "Action": [ 497 | "ec2:DescribeAccountAttributes", 498 | "ec2:DescribeAddresses", 499 | "ec2:AssociateAddress", 500 | "ec2:DisassociateAddress", 501 | "ec2:DescribeAvailabilityZones", 502 | "ec2:DescribeImages", 503 | "ec2:DescribeInstances", 504 | "ec2:RunInstances", 505 | "ec2:RebootInstances", 506 | "ec2:TerminateInstances", 507 | "ec2:DescribeKeypairs", 508 | "ec2:DescribeRegions", 509 | "ec2:DescribeSnapshots", 510 | "ec2:CreateSnapshot", 511 | "ec2:DeleteSnapshot", 512 | "ec2:DescribeSecurityGroups", 513 | "ec2:DescribeSubnets", 514 | "ec2:DescribeVpcs", 515 | "ec2:CreateTags", 516 | "ec2:DescribeVolumes", 517 | "ec2:CreateVolume", 518 | "ec2:AttachVolume", 519 | "ec2:DeleteVolume", 520 | "ec2:DetachVolume" 521 | ], 522 | "Resource": ["*"] 523 | }, 524 | { 525 | "Sid": "OpsManagerElbPermissions", 526 | "Effect": "Allow", 527 | "Action": [ 528 | "elasticloadbalancing:DescribeLoadBalancers", 529 | "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", 530 | "elasticloadbalancing:RegisterInstancesWithLoadBalancer" 531 | ], 532 | "Resource": ["*"] 533 | } 534 | ] 535 | }, 536 | "Description" : {"Fn::Join": [ "", ["Managed policy for OpsManager and BOSH director in ", {"Ref": "AWS::StackName"}]]}, 537 | "Roles" : [ { "Ref" : "PcfIamRole" } ], 538 | "Users" : [ { "Ref" : "PcfIamUser" } ] 539 | } 540 | }, 541 | "PcfIamUser": { 542 | "Type": "AWS::IAM::User", 543 | "DependsOn": [ 544 | "PcfVpc", 545 | "PcfOpsManagerS3Bucket" 546 | ], 547 | "Properties": { 548 | "Policies": [] 549 | } 550 | }, 551 | "PcfIamUserAccessKey": { 552 | "Type": "AWS::IAM::AccessKey", 553 | "DependsOn": "PcfIamUser", 554 | "Properties": { 555 | "UserName": {"Ref": "PcfIamUser"} 556 | } 557 | }, 558 | "PcfOpsManagerSecurityGroup": { 559 | "Type": "AWS::EC2::SecurityGroup", 560 | "Properties": { 561 | "GroupDescription": "Ops Manager Security Group", 562 | "VpcId": { 563 | "Ref": "PcfVpc" 564 | }, 565 | "SecurityGroupIngress": [ 566 | { 567 | "IpProtocol": "tcp", 568 | "FromPort": "22", 569 | "ToPort": "22", 570 | "CidrIp": {"Ref": "03OpsManagerIngress"} 571 | }, 572 | { 573 | "IpProtocol": "tcp", 574 | "FromPort": "80", 575 | "ToPort": "80", 576 | "CidrIp": {"Ref": "03OpsManagerIngress"} 577 | }, 578 | { 579 | "IpProtocol": "tcp", 580 | "FromPort": "443", 581 | "ToPort": "443", 582 | "CidrIp": {"Ref": "03OpsManagerIngress"} 583 | }, 584 | { 585 | "IpProtocol": "tcp", 586 | "FromPort": "22", 587 | "ToPort": "22", 588 | "CidrIp": {"Ref": "12VpcCIDR"} 589 | }, 590 | { 591 | "IpProtocol": "tcp", 592 | "FromPort": "25555", 593 | "ToPort": "25555", 594 | "CidrIp": {"Ref": "12VpcCIDR"} 595 | }, 596 | { 597 | "IpProtocol": "tcp", 598 | "FromPort": "6868", 599 | "ToPort": "6868", 600 | "CidrIp": {"Ref": "12VpcCIDR"} 601 | } 602 | ] 603 | } 604 | }, 605 | "S3Endpoint" : { 606 | "Type" : "AWS::EC2::VPCEndpoint", 607 | "Properties" : { 608 | "RouteTableIds" : [ {"Ref" : "PcfInfrastructureRouteTable"}, {"Ref" : "PcfPrivateRouteTable"} ], 609 | "ServiceName" : { "Fn::Join": [ "", [ "com.amazonaws.", { "Ref": "AWS::Region" }, ".s3" ] ] }, 610 | "VpcId" : {"Ref" : "PcfVpc"} 611 | } 612 | }, 613 | "PcfVmsSecurityGroup": { 614 | "Type": "AWS::EC2::SecurityGroup", 615 | "Properties": { 616 | "GroupDescription": "PCF VMs Security Group", 617 | "VpcId": { 618 | "Ref": "PcfVpc" 619 | }, 620 | "SecurityGroupIngress": [ 621 | { 622 | "IpProtocol": "-1", 623 | "CidrIp": {"Ref": "12VpcCIDR"} 624 | } 625 | ] 626 | } 627 | }, 628 | "PcfMysqlSecurityGroup": { 629 | "Type": "AWS::EC2::SecurityGroup", 630 | "Properties": { 631 | "GroupDescription": "PCF MySQL Security Group", 632 | "VpcId": { 633 | "Ref": "PcfVpc" 634 | }, 635 | "SecurityGroupIngress": [ 636 | { 637 | "IpProtocol": "tcp", 638 | "FromPort": "3306", 639 | "ToPort": "3306", 640 | "CidrIp": {"Ref": "12VpcCIDR"} 641 | } 642 | ] 643 | } 644 | }, 645 | "PcfNatGatewayAcl" : { 646 | "Type": "AWS::EC2::NetworkAcl", 647 | "Properties": { 648 | "VpcId": { 649 | "Ref": "PcfVpc" 650 | } 651 | } 652 | }, 653 | "PcfNatGatewayAclRuleAllowAll": { 654 | "Type": "AWS::EC2::NetworkAclEntry", 655 | "Properties": { 656 | "CidrBlock": {"Ref" : "12VpcCIDR"}, 657 | "Egress": "false", 658 | "Icmp": { 659 | "Code": "-1", 660 | "Type": "-1" 661 | }, 662 | "NetworkAclId": { 663 | "Ref": "PcfNatGatewayAcl" 664 | }, 665 | "PortRange": { 666 | "From": "1", 667 | "To": "65535" 668 | }, 669 | "Protocol": "-1", 670 | "RuleAction": "allow", 671 | "RuleNumber": "1" 672 | } 673 | }, 674 | "PcfPublicRouteTable": { 675 | "Type": "AWS::EC2::RouteTable", 676 | "Properties": { 677 | "VpcId": { 678 | "Ref": "PcfVpc" 679 | } 680 | } 681 | }, 682 | "PcfPublicDefaultRoute": { 683 | "Type": "AWS::EC2::Route", 684 | "Properties": { 685 | "DestinationCidrBlock": "0.0.0.0/0", 686 | "GatewayId": { 687 | "Ref": "PcfInternetGateway" 688 | }, 689 | "RouteTableId": { 690 | "Ref": "PcfPublicRouteTable" 691 | } 692 | } 693 | }, 694 | "PcfPublicSubnetRouteTableAssociation": { 695 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 696 | "Properties": { 697 | "RouteTableId": { 698 | "Ref": "PcfPublicRouteTable" 699 | }, 700 | "SubnetId": { 701 | "Ref": "PcfPublicSubnet" 702 | } 703 | } 704 | }, 705 | "PcfPublicSubnet2RouteTableAssociation": { 706 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 707 | "Properties": { 708 | "RouteTableId": { 709 | "Ref": "PcfPublicRouteTable" 710 | }, 711 | "SubnetId": { 712 | "Ref": "PcfPublicSubnet2" 713 | } 714 | } 715 | }, 716 | "PcfPublicSubnet3RouteTableAssociation": { 717 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 718 | "Properties": { 719 | "RouteTableId": { 720 | "Ref": "PcfPublicRouteTable" 721 | }, 722 | "SubnetId": { 723 | "Ref": "PcfPublicSubnet3" 724 | } 725 | } 726 | }, 727 | "PcfInfrastructureRouteTable": { 728 | "Type": "AWS::EC2::RouteTable", 729 | "Properties": { 730 | "VpcId": { 731 | "Ref": "PcfVpc" 732 | } 733 | } 734 | }, 735 | "PcfInfrastructureDefaultRoute": { 736 | "Type": "AWS::EC2::Route", 737 | "Properties": { 738 | "DestinationCidrBlock": "0.0.0.0/0", 739 | "NatGatewayId": { 740 | "Ref": "PcfNatGateway" 741 | }, 742 | "RouteTableId": { 743 | "Ref": "PcfInfrastructureRouteTable" 744 | } 745 | } 746 | }, 747 | "PcfInfrastructureSubnetRouteTableAssociation": { 748 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 749 | "Properties": { 750 | "RouteTableId": { 751 | "Ref": "PcfInfrastructureRouteTable" 752 | }, 753 | "SubnetId": { 754 | "Ref": "PcfInfrastructureSubnet" 755 | } 756 | } 757 | }, 758 | "PcfPrivateRouteTable": { 759 | "Type": "AWS::EC2::RouteTable", 760 | "Properties": { 761 | "VpcId": { 762 | "Ref": "PcfVpc" 763 | } 764 | } 765 | }, 766 | "PcfPrivateRouteTableAssociation": { 767 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 768 | "Properties": { 769 | "RouteTableId": { 770 | "Ref": "PcfPrivateRouteTable" 771 | }, 772 | "SubnetId": { 773 | "Ref": "PcfPrivateSubnet" 774 | } 775 | } 776 | }, 777 | "PcfPrivate2RouteTableAssociation": { 778 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 779 | "Properties": { 780 | "RouteTableId": { 781 | "Ref": "PcfPrivateRouteTable" 782 | }, 783 | "SubnetId": { 784 | "Ref": "PcfPrivateSubnet2" 785 | } 786 | } 787 | }, 788 | "PcfPrivate3RouteTableAssociation": { 789 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 790 | "Properties": { 791 | "RouteTableId": { 792 | "Ref": "PcfPrivateRouteTable" 793 | }, 794 | "SubnetId": { 795 | "Ref": "PcfPrivateSubnet3" 796 | } 797 | } 798 | }, 799 | "PcfRdsSubnet1": { 800 | "Type": "AWS::EC2::Subnet", 801 | "Condition": "CreateRDS", 802 | "Properties": { 803 | "AvailabilityZone": { 804 | "Fn::Select": [ 805 | "0", 806 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 807 | ] 808 | }, 809 | "CidrBlock": {"Ref": "18RdsSubnet1"}, 810 | "VpcId": { 811 | "Ref": "PcfVpc" 812 | }, 813 | "Tags": [ 814 | { 815 | "Key": "Name", 816 | "Value": "pcf-rds-subnet-1" 817 | }, 818 | { 819 | "Key": "subnet_type", 820 | "Value": "rds" 821 | } 822 | ] 823 | } 824 | }, 825 | "PcfRdsSubnet2": { 826 | "Type": "AWS::EC2::Subnet", 827 | "Condition": "CreateRDS", 828 | "Properties": { 829 | "AvailabilityZone": { 830 | "Fn::Select": [ 831 | "1", 832 | {"Fn::GetAZs": {"Ref": "AWS::Region"}} 833 | ] 834 | }, 835 | "CidrBlock": {"Ref": "19RdsSubnet2"}, 836 | "VpcId": { 837 | "Ref": "PcfVpc" 838 | }, 839 | "Tags": [ 840 | { 841 | "Key": "Name", 842 | "Value": "pcf-rds-subnet-2" 843 | }, 844 | { 845 | "Key": "subnet_type", 846 | "Value": "rds" 847 | } 848 | ] 849 | } 850 | }, 851 | "PcfRdsSubnetGroup": { 852 | "Type": "AWS::RDS::DBSubnetGroup", 853 | "Condition": "CreateRDS", 854 | "Properties": { 855 | "DBSubnetGroupDescription": "PCF RDS Subnet Group", 856 | "SubnetIds": [ 857 | {"Ref": "PcfRdsSubnet1"}, 858 | {"Ref": "PcfRdsSubnet2"} 859 | ] 860 | } 861 | }, 862 | "PcfRds": { 863 | "Type": "AWS::RDS::DBInstance", 864 | "Condition": "CreateRDS", 865 | "Properties": { 866 | "AllocatedStorage": "100", 867 | "DBInstanceClass": "db.m3.large", 868 | "Engine": "MySQL", 869 | "EngineVersion": "5.6.22", 870 | "MultiAZ": "True", 871 | "DBName": {"Ref": "04RdsDBName"}, 872 | "Iops": "1000", 873 | "MasterUsername": {"Ref": "05RdsUsername"}, 874 | "MasterUserPassword": {"Ref": "06RdsPassword"}, 875 | "PubliclyAccessible": "False", 876 | "VPCSecurityGroups": [{"Ref": "PcfMysqlSecurityGroup"}], 877 | "DBSubnetGroupName": {"Ref": "PcfRdsSubnetGroup"} 878 | } 879 | }, 880 | "PcfElbSecurityGroup": { 881 | "Type": "AWS::EC2::SecurityGroup", 882 | "Properties": { 883 | "VpcId": {"Ref": "PcfVpc"}, 884 | "GroupDescription": "ELB Security Group", 885 | "SecurityGroupIngress": [ 886 | { 887 | "CidrIp": "0.0.0.0/0", 888 | "FromPort": "80", 889 | "IpProtocol": "tcp", 890 | "ToPort": "80" 891 | }, 892 | { 893 | "CidrIp": "0.0.0.0/0", 894 | "FromPort": "443", 895 | "IpProtocol": "tcp", 896 | "ToPort": "443" 897 | }, 898 | { 899 | "CidrIp": "0.0.0.0/0", 900 | "FromPort": "4443", 901 | "IpProtocol": "tcp", 902 | "ToPort": "4443" 903 | } 904 | ] 905 | } 906 | }, 907 | "PcfElb": { 908 | "Type": "AWS::ElasticLoadBalancing::LoadBalancer", 909 | "DependsOn": [ 910 | "PcfElbSecurityGroup" 911 | ], 912 | "Properties": { 913 | "CrossZone": true, 914 | "HealthCheck": { 915 | "HealthyThreshold": "10", 916 | "Interval": "30", 917 | "Target": "TCP:80", 918 | "Timeout": "5", 919 | "UnhealthyThreshold": "2" 920 | }, 921 | "ConnectionSettings": { 922 | "IdleTimeout": 3600 923 | }, 924 | "LoadBalancerName": { "Fn::Join" : [ 925 | "", 926 | [ { "Fn::If" : [ 927 | "ElbPrefixProvided", 928 | { "Ref" : "09ElbPrefix" }, 929 | { "Ref" : "AWS::StackName" } 930 | ]}, 931 | "-pcf-elb" 932 | ] 933 | ] }, 934 | "Listeners": { "Fn::If" : [ 935 | "AllowHTTPTrafficOnELB", 936 | [ 937 | { 938 | "InstancePort": "80", 939 | "LoadBalancerPort": "80", 940 | "Protocol": "http", 941 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 942 | }, 943 | { 944 | "InstancePort": "80", 945 | "LoadBalancerPort": "443", 946 | "Protocol": "https", 947 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 948 | }, 949 | { 950 | "InstancePort": "80", 951 | "LoadBalancerPort": "4443", 952 | "Protocol": "ssl", 953 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 954 | } 955 | ], 956 | [ 957 | { 958 | "InstancePort": "80", 959 | "LoadBalancerPort": "443", 960 | "Protocol": "https", 961 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 962 | }, 963 | { 964 | "InstancePort": "80", 965 | "LoadBalancerPort": "4443", 966 | "Protocol": "ssl", 967 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 968 | } 969 | ] 970 | ]}, 971 | "Scheme": "internet-facing", 972 | "SecurityGroups": [ 973 | { 974 | "Ref": "PcfElbSecurityGroup" 975 | } 976 | ], 977 | "Subnets": [ 978 | {"Ref": "PcfPublicSubnet"}, 979 | {"Ref": "PcfPublicSubnet2"}, 980 | {"Ref": "PcfPublicSubnet3"} 981 | ] 982 | } 983 | }, 984 | "PcfElbSshSecurityGroup": { 985 | "Type": "AWS::EC2::SecurityGroup", 986 | "Properties": { 987 | "VpcId": {"Ref": "PcfVpc"}, 988 | "GroupDescription": "ELB Security Group", 989 | "SecurityGroupIngress": [ 990 | { 991 | "CidrIp": "0.0.0.0/0", 992 | "FromPort": "2222", 993 | "IpProtocol": "tcp", 994 | "ToPort": "2222" 995 | } 996 | ] 997 | } 998 | }, 999 | "PcfElbSsh": { 1000 | "Type": "AWS::ElasticLoadBalancing::LoadBalancer", 1001 | "DependsOn": [ 1002 | "PcfElbSshSecurityGroup" 1003 | ], 1004 | "Properties": { 1005 | "CrossZone": true, 1006 | "HealthCheck": { 1007 | "HealthyThreshold": "10", 1008 | "Interval": "30", 1009 | "Target": "TCP:2222", 1010 | "Timeout": "5", 1011 | "UnhealthyThreshold": "2" 1012 | }, 1013 | "ConnectionSettings": { 1014 | "IdleTimeout": 3600 1015 | }, 1016 | "LoadBalancerName": { "Fn::Join" : [ 1017 | "", 1018 | [ { "Fn::If" : [ 1019 | "ElbPrefixProvided", 1020 | { "Ref" : "09ElbPrefix" }, 1021 | { "Ref" : "AWS::StackName" } 1022 | ]}, 1023 | "-pcf-ssh-elb" 1024 | ] 1025 | ] }, 1026 | "Listeners": [ 1027 | { 1028 | "InstancePort": "2222", 1029 | "LoadBalancerPort": "2222", 1030 | "Protocol": "tcp" 1031 | } 1032 | ], 1033 | "Scheme": "internet-facing", 1034 | "SecurityGroups": [ 1035 | { 1036 | "Ref": "PcfElbSshSecurityGroup" 1037 | } 1038 | ], 1039 | "Subnets": [ 1040 | {"Ref": "PcfPublicSubnet"}, 1041 | {"Ref": "PcfPublicSubnet2"}, 1042 | {"Ref": "PcfPublicSubnet3"} 1043 | ] 1044 | } 1045 | }, 1046 | "PcfInternalElb": { 1047 | "Type": "AWS::ElasticLoadBalancing::LoadBalancer", 1048 | "DependsOn": [ 1049 | "PcfElbSecurityGroup" 1050 | ], 1051 | "Properties": { 1052 | "CrossZone": true, 1053 | "HealthCheck": { 1054 | "HealthyThreshold": "10", 1055 | "Interval": "30", 1056 | "Target": "TCP:80", 1057 | "Timeout": "5", 1058 | "UnhealthyThreshold": "2" 1059 | }, 1060 | "ConnectionSettings": { 1061 | "IdleTimeout": 3600 1062 | }, 1063 | "LoadBalancerName": { "Fn::Join" : [ 1064 | "", 1065 | [ { "Fn::If" : [ 1066 | "ElbPrefixProvided", 1067 | { "Ref" : "09ElbPrefix" }, 1068 | { "Ref" : "AWS::StackName" } 1069 | ]}, 1070 | "-pcf-elb-in" 1071 | ] 1072 | ] }, 1073 | "Listeners": { "Fn::If" : [ 1074 | "AllowHTTPTrafficOnELB", 1075 | [ 1076 | { 1077 | "InstancePort": "80", 1078 | "LoadBalancerPort": "80", 1079 | "Protocol": "http", 1080 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 1081 | }, 1082 | { 1083 | "InstancePort": "80", 1084 | "LoadBalancerPort": "443", 1085 | "Protocol": "https", 1086 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 1087 | }, 1088 | { 1089 | "InstancePort": "80", 1090 | "LoadBalancerPort": "4443", 1091 | "Protocol": "ssl", 1092 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 1093 | } 1094 | ], 1095 | [ 1096 | { 1097 | "InstancePort": "80", 1098 | "LoadBalancerPort": "443", 1099 | "Protocol": "https", 1100 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 1101 | }, 1102 | { 1103 | "InstancePort": "80", 1104 | "LoadBalancerPort": "4443", 1105 | "Protocol": "ssl", 1106 | "SSLCertificateId": {"Ref": "07SSLCertificateARN"} 1107 | } 1108 | ] 1109 | ]}, 1110 | "Scheme": "internal", 1111 | "SecurityGroups": [ 1112 | { 1113 | "Ref": "PcfElbSecurityGroup" 1114 | } 1115 | ], 1116 | "Subnets": [ 1117 | {"Ref": "PcfPublicSubnet"}, 1118 | {"Ref": "PcfPublicSubnet2"}, 1119 | {"Ref": "PcfPublicSubnet3"} 1120 | ] 1121 | } 1122 | }, 1123 | "PcfInternalElbSsh": { 1124 | "Type": "AWS::ElasticLoadBalancing::LoadBalancer", 1125 | "DependsOn": [ 1126 | "PcfElbSshSecurityGroup" 1127 | ], 1128 | "Properties": { 1129 | "CrossZone": true, 1130 | "HealthCheck": { 1131 | "HealthyThreshold": "10", 1132 | "Interval": "30", 1133 | "Target": "TCP:2222", 1134 | "Timeout": "5", 1135 | "UnhealthyThreshold": "2" 1136 | }, 1137 | "ConnectionSettings": { 1138 | "IdleTimeout": 3600 1139 | }, 1140 | "LoadBalancerName": { "Fn::Join" : [ 1141 | "", 1142 | [ { "Fn::If" : [ 1143 | "ElbPrefixProvided", 1144 | { "Ref" : "09ElbPrefix" }, 1145 | { "Ref" : "AWS::StackName" } 1146 | ]}, 1147 | "-pcf-ssh-elb-in" 1148 | ] 1149 | ] }, 1150 | "Listeners": [ 1151 | { 1152 | "InstancePort": "2222", 1153 | "LoadBalancerPort": "2222", 1154 | "Protocol": "tcp" 1155 | } 1156 | ], 1157 | "Scheme": "internal", 1158 | "SecurityGroups": [ 1159 | { 1160 | "Ref": "PcfElbSshSecurityGroup" 1161 | } 1162 | ], 1163 | "Subnets": [ 1164 | {"Ref": "PcfPublicSubnet"}, 1165 | {"Ref": "PcfPublicSubnet2"}, 1166 | {"Ref": "PcfPublicSubnet3"} 1167 | ] 1168 | } 1169 | }, 1170 | "PcfErtS3BucketIamPolicy": { 1171 | "Type" : "AWS::IAM::Policy", 1172 | "Properties" : { 1173 | "PolicyName" : "PcfErtPolicy", 1174 | "PolicyDocument" : { 1175 | "Version": "2012-10-17", 1176 | "Statement": [ 1177 | { 1178 | "Sid": "ElasticRuntimeS3Permissions", 1179 | "Effect": "Allow", 1180 | "Action": ["s3:*"], 1181 | "Resource": [ 1182 | { 1183 | "Fn::Join": [ 1184 | "", 1185 | [ 1186 | "arn:aws:s3:::", 1187 | {"Ref": "PcfElasticRuntimeS3BuildpacksBucket"} 1188 | ] 1189 | ] 1190 | }, 1191 | { 1192 | "Fn::Join": [ 1193 | "", 1194 | [ 1195 | "arn:aws:s3:::", 1196 | {"Ref": "PcfElasticRuntimeS3BuildpacksBucket"}, 1197 | "/*" 1198 | ] 1199 | ] 1200 | }, 1201 | { 1202 | "Fn::Join": [ 1203 | "", 1204 | [ 1205 | "arn:aws:s3:::", 1206 | {"Ref": "PcfElasticRuntimeS3DropletsBucket"} 1207 | ] 1208 | ] 1209 | }, 1210 | { 1211 | "Fn::Join": [ 1212 | "", 1213 | [ 1214 | "arn:aws:s3:::", 1215 | {"Ref": "PcfElasticRuntimeS3DropletsBucket"}, 1216 | "/*" 1217 | ] 1218 | ] 1219 | }, 1220 | { 1221 | "Fn::Join": [ 1222 | "", 1223 | [ 1224 | "arn:aws:s3:::", 1225 | {"Ref": "PcfElasticRuntimeS3PackagesBucket"} 1226 | ] 1227 | ] 1228 | }, 1229 | { 1230 | "Fn::Join": [ 1231 | "", 1232 | [ 1233 | "arn:aws:s3:::", 1234 | {"Ref": "PcfElasticRuntimeS3PackagesBucket"}, 1235 | "/*" 1236 | ] 1237 | ] 1238 | }, 1239 | { 1240 | "Fn::Join": [ 1241 | "", 1242 | [ 1243 | "arn:aws:s3:::", 1244 | {"Ref": "PcfElasticRuntimeS3ResourcesBucket"} 1245 | ] 1246 | ] 1247 | }, 1248 | { 1249 | "Fn::Join": [ 1250 | "", 1251 | [ 1252 | "arn:aws:s3:::", 1253 | {"Ref": "PcfElasticRuntimeS3ResourcesBucket"}, 1254 | "/*" 1255 | ] 1256 | ] 1257 | } 1258 | ] 1259 | } 1260 | ] 1261 | }, 1262 | "Users" : [ {"Ref": "PcfIamUser"} ] 1263 | } 1264 | }, 1265 | "PcfElasticRuntimeS3BuildpacksBucket": { 1266 | "Type": "AWS::S3::Bucket", 1267 | "Properties": { 1268 | "Tags": [ 1269 | { 1270 | "Key": "Name", 1271 | "Value": "PCF Elastic Runtime S3 Buildpacks Bucket" 1272 | } 1273 | ] 1274 | } 1275 | }, 1276 | "PcfElasticRuntimeS3DropletsBucket": { 1277 | "Type": "AWS::S3::Bucket", 1278 | "Properties": { 1279 | "Tags": [ 1280 | { 1281 | "Key": "Name", 1282 | "Value": "PCF Elastic Runtime S3 Droplets Bucket" 1283 | } 1284 | ] 1285 | } 1286 | }, 1287 | "PcfElasticRuntimeS3PackagesBucket": { 1288 | "Type": "AWS::S3::Bucket", 1289 | "Properties": { 1290 | "Tags": [ 1291 | { 1292 | "Key": "Name", 1293 | "Value": "PCF Elastic Runtime S3 Packages Bucket" 1294 | } 1295 | ] 1296 | } 1297 | }, 1298 | "PcfElasticRuntimeS3ResourcesBucket": { 1299 | "Type": "AWS::S3::Bucket", 1300 | "Properties": { 1301 | "Tags": [ 1302 | { 1303 | "Key": "Name", 1304 | "Value": "PCF Elastic Runtime S3 Resources Bucket" 1305 | } 1306 | ] 1307 | } 1308 | } 1309 | }, 1310 | "Outputs": { 1311 | "PcfElbDnsName": { 1312 | "Value": { "Fn::GetAtt": [ "PcfElb", "DNSName"]} 1313 | }, 1314 | "PcfElbSshDnsName": { 1315 | "Value": { "Fn::GetAtt": [ "PcfElbSsh", "DNSName"]} 1316 | }, 1317 | "PcfInternalElbDnsName": { 1318 | "Value": { "Fn::GetAtt": [ "PcfInternalElb", "DNSName"]} 1319 | }, 1320 | "PcfInternalElbSshDnsName": { 1321 | "Value": { "Fn::GetAtt": [ "PcfInternalElbSsh", "DNSName"]} 1322 | }, 1323 | "PcfVpc": { 1324 | "Value": {"Ref": "PcfVpc"} 1325 | }, 1326 | "PcfIamUserName": { 1327 | "Value": {"Ref": "PcfIamUser"} 1328 | }, 1329 | "PcfIamUserAccessKey": { 1330 | "Value": {"Ref": "PcfIamUserAccessKey"} 1331 | }, 1332 | "PcfIamUserSecretAccessKey": { 1333 | "Value": { "Fn::GetAtt": [ "PcfIamUserAccessKey", "SecretAccessKey"]} 1334 | }, 1335 | "PcfIamInstanceProfile": { 1336 | "Value": {"Ref": "PcfIamInstanceProfile"} 1337 | }, 1338 | "PcfOpsManagerS3Bucket": { 1339 | "Value": {"Ref": "PcfOpsManagerS3Bucket"} 1340 | }, 1341 | "PcfElasticRuntimeS3BuildpacksBucket": { 1342 | "Value": {"Ref": "PcfElasticRuntimeS3BuildpacksBucket"} 1343 | }, 1344 | "PcfElasticRuntimeS3DropletsBucket": { 1345 | "Value": {"Ref": "PcfElasticRuntimeS3DropletsBucket"} 1346 | }, 1347 | "PcfElasticRuntimeS3PackagesBucket": { 1348 | "Value": {"Ref": "PcfElasticRuntimeS3PackagesBucket"} 1349 | }, 1350 | "PcfElasticRuntimeS3ResourcesBucket": { 1351 | "Value": {"Ref": "PcfElasticRuntimeS3ResourcesBucket"} 1352 | }, 1353 | "PcfVmsSecurityGroupId": { 1354 | "Value": {"Ref": "PcfVmsSecurityGroup"} 1355 | }, 1356 | "PcfOpsManagerSecurityGroupId": { 1357 | "Value": {"Ref": "PcfOpsManagerSecurityGroup"} 1358 | }, 1359 | "PcfPrivateSubnetId": { 1360 | "Value": {"Ref": "PcfPrivateSubnet"} 1361 | }, 1362 | "PcfPrivateSubnet2Id": { 1363 | "Value": {"Ref": "PcfPrivateSubnet2"} 1364 | }, 1365 | "PcfPrivateSubnet3Id": { 1366 | "Value": {"Ref": "PcfPrivateSubnet3"} 1367 | }, 1368 | "PcfPrivateSubnetAvailabilityZone": { 1369 | "Value": { "Fn::GetAtt" : [ "PcfPrivateSubnet", "AvailabilityZone" ] } 1370 | }, 1371 | "PcfPrivateSubnet2AvailabilityZone": { 1372 | "Value": { "Fn::GetAtt" : [ "PcfPrivateSubnet2", "AvailabilityZone" ] } 1373 | }, 1374 | "PcfPrivateSubnet3AvailabilityZone": { 1375 | "Value": { "Fn::GetAtt" : [ "PcfPrivateSubnet3", "AvailabilityZone" ] } 1376 | }, 1377 | "PcfPublicSubnetId": { 1378 | "Value": {"Ref": "PcfPublicSubnet"} 1379 | }, 1380 | "PcfPublicSubnet2Id": { 1381 | "Value": {"Ref": "PcfPublicSubnet2"} 1382 | }, 1383 | "PcfPublicSubnet3Id": { 1384 | "Value": {"Ref": "PcfPublicSubnet3"} 1385 | }, 1386 | "PcfPublicSubnetAvailabilityZone": { 1387 | "Value": { "Fn::GetAtt" : [ "PcfPublicSubnet", "AvailabilityZone" ] } 1388 | }, 1389 | "PcfPublicSubnet2AvailabilityZone": { 1390 | "Value": { "Fn::GetAtt" : [ "PcfPublicSubnet2", "AvailabilityZone" ] } 1391 | }, 1392 | "PcfPublicSubnet3AvailabilityZone": { 1393 | "Value": { "Fn::GetAtt" : [ "PcfPublicSubnet3", "AvailabilityZone" ] } 1394 | }, 1395 | "PcfInfrastructureSubnetId": { 1396 | "Value": {"Ref": "PcfInfrastructureSubnet"} 1397 | }, 1398 | "PcfInfrastructureSubnetAvailabilityZone": { 1399 | "Value": { "Fn::GetAtt" : [ "PcfInfrastructureSubnet", "AvailabilityZone" ] } 1400 | }, 1401 | "PcfRdsAddress": { 1402 | "Condition": "CreateRDS", 1403 | "Value": { "Fn::GetAtt": [ "PcfRds", "Endpoint.Address"]} 1404 | }, 1405 | "PcfRdsPort": { 1406 | "Condition": "CreateRDS", 1407 | "Value": {"Fn::GetAtt": [ "PcfRds", "Endpoint.Port"] } 1408 | }, 1409 | "PcfRdsUsername": { 1410 | "Condition": "CreateRDS", 1411 | "Value": {"Ref": "05RdsUsername"} 1412 | }, 1413 | "PcfRdsPassword": { 1414 | "Condition": "CreateRDS", 1415 | "Value": {"Ref": "06RdsPassword"} 1416 | }, 1417 | "PcfRdsDBName": { 1418 | "Condition": "CreateRDS", 1419 | "Value": {"Ref": "04RdsDBName"} 1420 | } 1421 | } 1422 | } 1423 | -------------------------------------------------------------------------------- /pivnet.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import os 3 | import urllib 4 | from pkg_resources import parse_version 5 | import sys 6 | import time 7 | 8 | 9 | class AuthException(Exception): 10 | pass 11 | 12 | 13 | class Pivnet(object): 14 | 15 | def __init__(self, token=None, url_base=None): 16 | self.url_base = url_base or 'https://network.pivotal.io/api/v2' 17 | self.token = token or os.getenv('PIVNET_TOKEN') 18 | self.auth_header = {"Authorization": "Token {}".format(self.token)} 19 | self._validate_() 20 | 21 | def _validate_(self): 22 | """ ensure that you can logon to pivnet """ 23 | if self.token is None: 24 | raise AuthException("PIVNET_TOKEN env var is not exported") 25 | ans = self.get("{}/authentication".format(self.url_base)) 26 | if ans.status_code != 200: 27 | raise AuthException(ans.text) 28 | 29 | def get(self, url, **kwargs): 30 | return requests.get(url, headers=self.auth_header, **kwargs) 31 | 32 | def post(self, url, **kwargs): 33 | return requests.post(url, headers=self.auth_header, **kwargs) 34 | 35 | def _latest(self, product, include_unreleased=False, version=None): 36 | """ https://network.pivotal.io 37 | /api/v2/products/elastic-runtime/releases 38 | """ 39 | if version is not None and version.lower() == 'latest': 40 | version = None 41 | 42 | ans = self.get( 43 | "{}/products/{}/releases".format(self.url_base, product)) 44 | releases = {parse_version(r['version']): r 45 | for r in ans.json()['releases']} 46 | vers = releases.keys() 47 | if include_unreleased is False: 48 | vers = [v for v in vers if v.is_prerelease is False] 49 | 50 | if version is not None: 51 | vers = [v for v in vers if v.base_version.startswith(version)] 52 | 53 | if len(vers) == 0: 54 | raise Exception("No version matched product={}," 55 | "v={}, {}".format( 56 | product, version, releases.keys())) 57 | 58 | return [releases[v] for v in sorted(vers, reverse=True)] 59 | 60 | def latest(self, product, include_unreleased=False, version=None): 61 | """ https://network.pivotal.io 62 | /api/v2/products/elastic-runtime/releases 63 | """ 64 | return self._latest(product, include_unreleased, version)[0] 65 | 66 | def latest_file(self, product, include_unreleased, version, selector): 67 | vers = self._latest(product, include_unreleased, version) 68 | 69 | for ver in vers: 70 | for fl in self.productfiles(product, ver['id']): 71 | if selector(fl): 72 | return ver, fl 73 | 74 | def productfiles(self, product, releaseNumber): 75 | return self.get("{}/products/{}/releases/{}/product_files".format( 76 | self.url_base, product, releaseNumber)).json()['product_files'] 77 | 78 | def files(self, product, releaseNumber): 79 | return self.get("{}/products/{}/releases/{}/product_files".format( 80 | self.url_base, product, releaseNumber)).json() 81 | 82 | def acceptEULA(self, verDict): 83 | # eula acceptance per spec 84 | print "Accepting EULA for the relase" 85 | resp = self.post(href(verDict, 'eula_acceptance'), 86 | allow_redirects=False) 87 | if resp.status_code != 200: 88 | raise Exception("Could not auto accept eula" + 89 | href(verDict, 'eula_acceptance') + 90 | " " + str(resp.headers)) 91 | 92 | """ 'https://network.pivotal.io/api/v2/products/ 93 | elastic-runtime/releases/1530/product_files/2946/download' 94 | """ 95 | def download(self, ver, filedict, filename=None, quiet=False): 96 | filename = filename or os.path.basename(filedict['aws_object_key']) 97 | resp = self.post(href(filedict, 'download'), allow_redirects=False) 98 | if resp.status_code == 451: 99 | self.acceptEULA(ver) 100 | resp = self.post(href(filedict, 'download'), allow_redirects=False) 101 | 102 | if resp.status_code != 302: 103 | raise Exception( 104 | "Could not download " + 105 | href(filedict, 'download') + " " 106 | + str(resp.headers)) 107 | 108 | class _progress_hook(object): 109 | lpr = 10 110 | started = False 111 | tm = time.time() 112 | 113 | def __call__(self, nblocks, block_size, size): 114 | if self.started is False: 115 | self.started = True 116 | if quiet is False: 117 | print " size: ", size 118 | if (100.0 * nblocks * block_size)/size > self.lpr: 119 | tm_end = time.time() 120 | if quiet is False: 121 | print >> sys.stderr, \ 122 | self.lpr, " ({} kBps)".format( 123 | int((nblocks * block_size) / 124 | (1000.0*(tm_end-self.tm)))), 125 | self.lpr += 10 126 | 127 | if quiet is False: 128 | print "\nDownloading ", filename, 129 | return filename, urllib.urlretrieve( 130 | resp.headers['location'], filename, _progress_hook()) 131 | 132 | 133 | def href(obj, key): 134 | return obj['_links'][key]['href'] 135 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | pyyaml 3 | robobrowser 4 | bs4 5 | mako 6 | sh 7 | requests 8 | paramiko 9 | -------------------------------------------------------------------------------- /stemplate.py: -------------------------------------------------------------------------------- 1 | def number(obj): 2 | try: 3 | return int(obj) 4 | except ValueError: 5 | try: 6 | return float(obj) 7 | except ValueError: 8 | return obj 9 | 10 | 11 | def resolve(kx, var, replacefn=None, idset=None): 12 | idset = idset or set() 13 | for k, v in kv(kx): 14 | if isinstance(v, (str, unicode)): 15 | if replacefn is not None: 16 | v = replacefn(v) 17 | # check for direct replacement first 18 | vx = v.strip()[1:-1] 19 | if vx in var: 20 | kx[k] = number(var[vx]) 21 | else: 22 | kx[k] = v.format(**var) 23 | continue 24 | if isinstance(v, (int, long, bool)): 25 | continue 26 | vid = id(v) 27 | if vid not in idset: 28 | idset.add(vid) 29 | resolve(v, var, replacefn=replacefn, idset=idset) 30 | 31 | 32 | def kv(obj): 33 | if isinstance(obj, dict): 34 | return obj.iteritems() 35 | elif isinstance(obj, list): 36 | return enumerate(obj) 37 | elif obj is None: 38 | return enumerate([]) 39 | else: 40 | raise Exception("{} {}".format(obj, type(obj))) 41 | 42 | 43 | def isElementry(val): 44 | return isinstance( 45 | val, 46 | (int, long, bool, str, unicode)) 47 | 48 | 49 | def isTerminal(val): 50 | if isElementry(val) is True: 51 | return True 52 | if isinstance(val, list): 53 | return all(isElementry(vv) for vv in val) 54 | 55 | return False 56 | 57 | 58 | def cfgmerge(dest, src): 59 | dest_keys = dest.keys() 60 | for key in src.keys(): 61 | if key == src.idfield: 62 | continue 63 | val = src[key] 64 | if key == 'value' or isTerminal(val.obj): 65 | dest[key] = val.obj 66 | continue 67 | if key in dest_keys: 68 | cfgmerge(dest[key], src[key]) 69 | else: 70 | print( 71 | "{} key {} is missing from {} ".format( 72 | src, 73 | key, 74 | dest_keys)) 75 | dest[key] = val.obj 76 | 77 | 78 | class Cfg(object): 79 | def __init__(self, obj, idfield='identifier', key=""): 80 | self.obj = obj 81 | self.idfield = idfield 82 | self.key = key 83 | 84 | def __getitem__(self, key): 85 | if isinstance(self.obj, dict): 86 | return Cfg(self.obj[key], self.idfield, self.key+"."+key) 87 | if isinstance(self.obj, list): 88 | if isinstance(key, (str, unicode)): 89 | val = next(vv for vv in self.obj if vv[self.idfield] == key) 90 | return Cfg(val, self.idfield, self.key+"."+key) 91 | elif isinstance(key, (int, long)): 92 | return Cfg(self.obj[key], self.idfield, self.key+"."+key) 93 | 94 | def __setitem__(self, key, val): 95 | if isinstance(self.obj, dict): 96 | self.obj[key] = val 97 | 98 | def keys(self): 99 | if isinstance(self.obj, dict): 100 | return self.obj.keys() 101 | elif isinstance(self.obj, list): 102 | return [v[self.idfield] for v in self.obj] 103 | 104 | def get(self, jpath): 105 | vx = self 106 | for comp in jpath.split('.'): 107 | if comp == '': 108 | continue 109 | vx = vx[comp] 110 | return vx 111 | 112 | def __repr__(self): 113 | return self.key 114 | -------------------------------------------------------------------------------- /wait_util.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class TimeoutException(Exception): 5 | pass 6 | 7 | 8 | def wait_while(condition, refresh=lambda: True): 9 | def waitfor(timeout): 10 | waited = 0 11 | SLEEPTIME = 5 12 | 13 | refresh() 14 | if not condition(): 15 | return True 16 | 17 | while waited < timeout and condition(): 18 | time.sleep(SLEEPTIME) 19 | waited += SLEEPTIME 20 | refresh() 21 | if condition(): 22 | raise TimeoutException() 23 | 24 | return waitfor 25 | --------------------------------------------------------------------------------