├── runtime.txt ├── README.md ├── manifest.yml ├── demiurge ├── api │ ├── __init__.py │ └── clusters.py ├── __init__.py ├── cert.py ├── swagger │ └── clusters.yaml ├── cli.py └── aws.py ├── Dockerfile ├── setup.py ├── .gitignore └── LICENSE /runtime.txt: -------------------------------------------------------------------------------- 1 | python-2.7.11 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # demiurge 2 | Kubernetes Clusters Creator 3 | -------------------------------------------------------------------------------- /manifest.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | --- 17 | applications: 18 | - command: demiurge 19 | buildpack: https://github.com/cloudfoundry/python-buildpack#v1.5.8 20 | -------------------------------------------------------------------------------- /demiurge/api/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | # Copyright (c) 2016 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from . import clusters 18 | 19 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=100 20 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | FROM python:2.7-slim 17 | MAINTAINER Maciej Strzelecki 18 | RUN mkdir -p /usr/src/app 19 | WORKDIR /usr/src/app 20 | COPY . /usr/src/app 21 | RUN pip install . 22 | ENV PORT 8080 23 | ENTRYPOINT ["demiurge"] 24 | EXPOSE $PORT 25 | -------------------------------------------------------------------------------- /demiurge/__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | # Copyright (c) 2016 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import connexion 18 | from connexion.resolver import RestyResolver 19 | from flask_httpauth import HTTPBasicAuth 20 | 21 | __version__ = '0.8.3' 22 | 23 | APP = connexion.App(__name__, specification_dir='swagger/', arguments={'version': __version__}) 24 | APPLICATION = APP.app 25 | AUTH = HTTPBasicAuth() 26 | 27 | APPLICATION.config['USERS'] = {} 28 | 29 | @AUTH.get_password 30 | def get_password(username): 31 | return APPLICATION.config['USERS'].get(username) 32 | 33 | def main(): 34 | APP.add_api('clusters.yaml', resolver=RestyResolver(__name__ + '.api')) 35 | APP.run() 36 | 37 | if __name__ == '__main__': 38 | main() 39 | 40 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=100 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | # Copyright (c) 2016 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from setuptools import setup, find_packages 18 | 19 | setup( 20 | name='demiurge', 21 | version='0.8.3', 22 | packages=find_packages(), 23 | install_requires=[ 24 | 'awacs==0.5.4', 25 | 'boto3==1.3.1', 26 | 'click==6.6', 27 | 'connexion==1.0.103', 28 | 'fauxfactory==2.0.9', 29 | 'Flask-HTTPAuth', 30 | 'troposphere==1.6.0', 31 | 'pyopenssl', 32 | 'cffi==1.7.0' 33 | ], 34 | author='Maciej Strzelecki', 35 | author_email='maciej.strzelecki@intel.com', 36 | license='Apache License, Version 2.0', 37 | entry_points={ 38 | 'console_scripts': [ 39 | 'demiurge = demiurge.cli:cli', 40 | ], 41 | }, 42 | ) 43 | 44 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=100 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim,python 3 | 4 | ### Vim ### 5 | # swap 6 | [._]*.s[a-w][a-z] 7 | [._]s[a-w][a-z] 8 | # session 9 | Session.vim 10 | # temporary 11 | .netrwhist 12 | *~ 13 | # auto-generated tag files 14 | tags 15 | 16 | 17 | ### Python ### 18 | # Byte-compiled / optimized / DLL files 19 | __pycache__/ 20 | *.py[cod] 21 | *$py.class 22 | 23 | # C extensions 24 | *.so 25 | 26 | # Distribution / packaging 27 | .Python 28 | env/ 29 | build/ 30 | develop-eggs/ 31 | dist/ 32 | downloads/ 33 | eggs/ 34 | .eggs/ 35 | lib/ 36 | lib64/ 37 | parts/ 38 | sdist/ 39 | var/ 40 | *.egg-info/ 41 | .installed.cfg 42 | *.egg 43 | 44 | # PyInstaller 45 | # Usually these files are written by a python script from a template 46 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 47 | *.manifest 48 | *.spec 49 | 50 | # Installer logs 51 | pip-log.txt 52 | pip-delete-this-directory.txt 53 | 54 | # Unit test / coverage reports 55 | htmlcov/ 56 | .tox/ 57 | .coverage 58 | .coverage.* 59 | .cache 60 | nosetests.xml 61 | coverage.xml 62 | *,cover 63 | .hypothesis/ 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | *.log 71 | local_settings.py 72 | 73 | # Flask instance folder 74 | instance/ 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | target/ 81 | 82 | # IPython Notebook 83 | .ipynb_checkpoints 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # dotenv 89 | .env 90 | 91 | # Cloud Foundry Python Buildpack 92 | vendor/ 93 | 94 | # Shared credentials file 95 | .aws/credentials 96 | 97 | # AWS config file 98 | .aws/config 99 | 100 | # Boto2 config file 101 | boto.cfg 102 | .boto 103 | -------------------------------------------------------------------------------- /demiurge/cert.py: -------------------------------------------------------------------------------- 1 | from OpenSSL import crypto, SSL 2 | 3 | VALIDITY = 60*60*24*365*5 4 | 5 | def create_cert(cn, san_list=None, sign_key=None, sign_cert=None, ca=False): 6 | generated_key = crypto.PKey() 7 | generated_key.generate_key(crypto.TYPE_RSA, 2048) 8 | 9 | generated_req = crypto.X509Req() 10 | generated_req.get_subject().CN = cn 11 | generated_req.set_pubkey(generated_key) 12 | generated_req.sign(generated_key, 'sha1') 13 | 14 | generated_cert = crypto.X509() 15 | 16 | if san_list: 17 | generated_cert.add_extensions([crypto.X509Extension( 18 | "subjectAltName", False, ", ".join(san_list) 19 | )]) 20 | 21 | generated_cert.gmtime_adj_notBefore(0) 22 | generated_cert.gmtime_adj_notAfter(VALIDITY) 23 | generated_cert.set_subject(generated_req.get_subject()) 24 | generated_cert.set_pubkey(generated_req.get_pubkey()) 25 | 26 | issuer_cert = sign_cert if sign_cert else generated_cert 27 | generated_cert.set_issuer(issuer_cert.get_subject()) 28 | 29 | generated_cert.sign(sign_key if sign_key else generated_key, 'sha1') 30 | 31 | if ca: 32 | generated_cert.add_extensions([ 33 | crypto.X509Extension("basicConstraints", True, "CA:TRUE"), 34 | crypto.X509Extension("subjectKeyIdentifier", False, "hash", subject=generated_cert), 35 | ]) 36 | generated_cert.add_extensions([ 37 | crypto.X509Extension("authorityKeyIdentifier", False, "keyid:always", 38 | issuer=generated_cert) 39 | ]) 40 | 41 | pem_key = crypto.dump_privatekey(crypto.FILETYPE_PEM, generated_key) 42 | pem_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, generated_cert) 43 | 44 | return (pem_key, pem_cert, generated_key, generated_cert) 45 | 46 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=100 47 | -------------------------------------------------------------------------------- /demiurge/swagger/clusters.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | --- 17 | 18 | swagger: '2.0' 19 | 20 | info: 21 | title: Kubernetes Clusters Creator 22 | license: 23 | name: Apache 2.0 24 | url: http://www.apache.org/licenses/LICENSE-2.0.html 25 | version: '{{ version }}' 26 | 27 | paths: 28 | /clusters: 29 | get: 30 | responses: 31 | 200: 32 | description: Fetch a list of clusters 33 | schema: 34 | $ref: '#/definitions/Clusters' 35 | security: 36 | - basic: [] 37 | '/clusters/{cluster_name}': 38 | put: 39 | parameters: 40 | - name: cluster_name 41 | in: path 42 | required: true 43 | type: string 44 | responses: 45 | 202: 46 | description: Create a new cluster 47 | 409: 48 | description: Create a new cluster 49 | security: 50 | - basic: [] 51 | get: 52 | parameters: 53 | - name: cluster_name 54 | in: path 55 | required: true 56 | type: string 57 | responses: 58 | 200: 59 | description: Fetch a cluster by name 60 | schema: 61 | $ref: '#/definitions/Cluster' 62 | 204: 63 | description: Fetch a cluster by name 64 | 404: 65 | description: Fetch a cluster by name 66 | security: 67 | - basic: [] 68 | delete: 69 | parameters: 70 | - name: cluster_name 71 | in: path 72 | required: true 73 | type: string 74 | responses: 75 | 204: 76 | description: Delete a cluster by name 77 | security: 78 | - basic: [] 79 | 80 | definitions: 81 | Cluster: 82 | type: object 83 | properties: 84 | cluster_name: 85 | type: string 86 | api_server: 87 | type: string 88 | username: 89 | type: string 90 | password: 91 | type: string 92 | consul_http_api: 93 | type: string 94 | Clusters: 95 | type: array 96 | items: 97 | $ref: '#/definitions/Cluster' 98 | 99 | securityDefinitions: 100 | basic: 101 | type: basic 102 | -------------------------------------------------------------------------------- /demiurge/cli.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | # Copyright (c) 2016 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import logging 18 | 19 | import click 20 | 21 | from . import __version__, APP, APPLICATION, main 22 | 23 | @click.command() 24 | @click.option('--debug/--no-debug', '-d', default=False) 25 | @click.option('--port', '-p', envvar='PORT', default=8080) 26 | 27 | @click.option('--username', envvar='USERNAME', required=True, 28 | help='Username for basic authentication.') 29 | @click.option('--password', envvar='PASSWORD', required=True, 30 | help='Password for basic authentication.') 31 | 32 | @click.option('--region-name', envvar='AWS_DEFAULT_REGION', default='us-west-2', 33 | help='The region to use.') 34 | @click.option('--aws-access-key-id', envvar='AWS_ACCESS_KEY_ID', 35 | help='The AWS Access Key ID.') 36 | @click.option('--aws-secret-access-key', envvar='AWS_SECRET_ACCESS_KEY', 37 | help='The AWS Secret Access Key.') 38 | 39 | @click.option('--os-username', envvar='OS_USERNAME', help='Your OpenStack username.') 40 | @click.option('--os-password', envvar='OS_PASSWORD', help='Your OpenStack password.') 41 | @click.option('--os-tenant-id', envvar='OS_TENANT_ID', help='Your OpenStack tenant.') 42 | @click.option('--os-auth-url', envvar='OS_AUTH_URL', help='Your OpenStack auth endpoint.') 43 | 44 | @click.option('--vpc', envvar='VPC', required=True, 45 | help='VPC ID of your exsiting Virtual Private Cloud (VPC) where you want to deploy ' 46 | 'Kubernetes clusters.') 47 | @click.option('--subnet', envvar='SUBNET', required=True, 48 | help='Subnet ID of the existing subnet in your VPC where you want to deploy ' 49 | 'Kubernetes nodes.') 50 | @click.option('--key-name', envvar='KEY_NAME', required=True, 51 | help='Name of an existing EC2 Key Pair. Kubernetes instances will launch with ' 52 | 'this Key Pair.') 53 | @click.option('--consul-dc', envvar='CONSUL_DC', required=True, 54 | help='The datacenter in which the Consul agent is running.') 55 | @click.option('--consul-join', envvar='CONSUL_JOIN', required=True, 56 | help='Address of another Consul agent to join.') 57 | def cli(debug, port, username, password, **kwargs): 58 | logging.basicConfig(level=logging.DEBUG if debug else logging.INFO) 59 | 60 | APP.debug = debug 61 | APP.port = port 62 | 63 | APPLICATION.config['USERS'][username] = password 64 | 65 | APPLICATION.config['AWS_DEFAULT_REGION_NAME'] = kwargs['region_name'] 66 | APPLICATION.config['AWS_ACCESS_KEY_ID'] = kwargs['aws_access_key_id'] 67 | APPLICATION.config['AWS_SECRET_ACCESS_KEY'] = kwargs['aws_secret_access_key'] 68 | 69 | APPLICATION.config['VPC'] = kwargs['vpc'] 70 | APPLICATION.config['SUBNET'] = kwargs['subnet'] 71 | APPLICATION.config['KEY_NAME'] = kwargs['key_name'] 72 | APPLICATION.config['CONSUL_DC'] = kwargs['consul_dc'] 73 | APPLICATION.config['CONSUL_JOIN'] = kwargs['consul_join'] 74 | 75 | main() 76 | 77 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=100 78 | -------------------------------------------------------------------------------- /demiurge/api/clusters.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | # Copyright (c) 2016 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import re 18 | from time import sleep 19 | 20 | import boto3 21 | from botocore.exceptions import ClientError 22 | import fauxfactory 23 | from connexion import NoContent 24 | import logging 25 | 26 | from .. import APP, APPLICATION, AUTH 27 | from ..aws import TEMPLATE 28 | 29 | logger = logging.getLogger('clusters.api') 30 | 31 | CLIENT = boto3.client( 32 | 'cloudformation', 33 | region_name=APPLICATION.config.get('AWS_DEFAULT_REGION_NAME'), 34 | aws_access_key_id=APPLICATION.config.get('AWS_ACCESS_KEY_ID'), 35 | aws_secret_access_key=APPLICATION.config.get('AWS_SECRET_ACCESS_KEY'), 36 | ) 37 | 38 | STACK_NAME = 'TAP-Kubernetes-{}' 39 | MAX_RETRIES = 10 40 | 41 | def __cluster(stack): 42 | cluster = {} 43 | 44 | if 'Parameters' not in stack: 45 | return None 46 | 47 | for parameter in stack['Parameters']: 48 | if parameter['ParameterKey'] == 'ClusterName': 49 | cluster['cluster_name'] = parameter['ParameterValue'] 50 | elif parameter['ParameterKey'] == 'Username': 51 | cluster['username'] = parameter['ParameterValue'] 52 | elif parameter['ParameterKey'] == 'Password': 53 | cluster['password'] = parameter['ParameterValue'] 54 | elif parameter['ParameterKey'] == 'KubernetesServiceNetwork': 55 | cluster['kubernetes_service_network'] = parameter['ParameterValue'] 56 | elif (parameter['ParameterKey'] == 'VPC' and 57 | parameter['ParameterValue'] != APPLICATION.config['VPC']): 58 | return None 59 | 60 | if 'Outputs' in stack: 61 | for output in stack['Outputs']: 62 | if output['OutputKey'] == 'APIServer': 63 | cluster['api_server'] = output['OutputValue'] 64 | if output['OutputKey'] == 'ConsulHTTPAPI': 65 | cluster['consul_http_api'] = output['OutputValue'] 66 | 67 | return cluster 68 | 69 | @AUTH.login_required 70 | def search(): 71 | clusters = [] 72 | 73 | response = CLIENT.describe_stacks() 74 | 75 | for stack in response['Stacks']: 76 | if re.match(r'(CREATE|UPDATE)_COMPLETE', stack['StackStatus']): 77 | cluster = __cluster(stack) 78 | 79 | if cluster: 80 | clusters.append(cluster) 81 | 82 | return clusters, 200 83 | 84 | @AUTH.login_required 85 | def get(cluster_name): 86 | response = CLIENT.describe_stacks() 87 | 88 | for stack in response['Stacks']: 89 | if stack['StackName'] == STACK_NAME.format(cluster_name): 90 | if re.match(r'(CREATE|UPDATE)_COMPLETE', stack['StackStatus']): 91 | return __cluster(stack), 200 92 | elif re.match(r'(CREATE|UPDATE)_IN_PROGRESS', stack['StackStatus']): 93 | return NoContent, 204 94 | elif re.match(r'DELETE_(IN_PROGRESS|COMPLETE)', stack['StackStatus']): 95 | return NoContent, 404 96 | else: 97 | error_msg = stack['StackName'] + ': ' + stack['StackStatus'] 98 | if 'StackStatusReason' in stack: 99 | error_msg += ': ' + stack['StackStatusReason'] 100 | logger.error(error_msg) 101 | return NoContent, 404 102 | 103 | return NoContent, 404 104 | 105 | def __get_next_network(): 106 | clusters = [] 107 | networks = [] 108 | response = CLIENT.describe_stacks() 109 | stacks = response['Stacks'] 110 | for stack in stacks: 111 | if stack is not None: 112 | cluster = (__cluster(stack)) 113 | if cluster: 114 | clusters.append(cluster) 115 | 116 | for cluster in clusters: 117 | if 'kubernetes_service_network' in cluster: 118 | networks.append(cluster['kubernetes_service_network']) 119 | 120 | for subnet_no in range(1, 254): 121 | subnet = '10.3.{}.0/24'.format(subnet_no) 122 | if not subnet in networks: 123 | return subnet_no 124 | 125 | @AUTH.login_required 126 | def put(cluster_name): 127 | subnet_no = __get_next_network() 128 | if subnet_no is None: 129 | return NoContent, 409 130 | 131 | try: 132 | CLIENT.create_stack( 133 | StackName=STACK_NAME.format(cluster_name), 134 | TemplateBody=TEMPLATE.to_json(), 135 | Parameters=[ 136 | { 137 | 'ParameterKey': 'KubernetesServiceNetwork', 138 | 'ParameterValue': '10.3.{}.0/24'.format(subnet_no), 139 | }, 140 | { 141 | 'ParameterKey': 'KubernetesServiceNetworkMin', 142 | 'ParameterValue': '10.3.{}.0'.format(subnet_no), 143 | }, 144 | { 145 | 'ParameterKey': 'KubernetesServiceNetworkMax', 146 | 'ParameterValue': '10.3.{}.0'.format(subnet_no), 147 | }, 148 | { 149 | 'ParameterKey': 'VPC', 150 | 'ParameterValue': APPLICATION.config['VPC'], 151 | }, 152 | { 153 | 'ParameterKey': 'Subnet', 154 | 'ParameterValue': APPLICATION.config['SUBNET']}, 155 | { 156 | 'ParameterKey': 'KeyName', 157 | 'ParameterValue': APPLICATION.config['KEY_NAME'], 158 | }, 159 | { 160 | 'ParameterKey': 'ClusterName', 161 | 'ParameterValue': cluster_name, 162 | }, 163 | { 164 | 'ParameterKey': 'Password', 165 | 'ParameterValue': fauxfactory.gen_string('alphanumeric', 16), 166 | }, 167 | { 168 | 'ParameterKey': 'ConsulDC', 169 | 'ParameterValue': APPLICATION.config['CONSUL_DC'], 170 | }, 171 | { 172 | 'ParameterKey': 'ConsulJoin', 173 | 'ParameterValue': APPLICATION.config['CONSUL_JOIN'], 174 | }, 175 | ], 176 | DisableRollback=APP.debug, 177 | Capabilities=[ 178 | 'CAPABILITY_IAM', 179 | ], 180 | ) 181 | except ClientError as exception: 182 | if exception.response['Error']['Code'] == 'AlreadyExistsException': 183 | return NoContent, 409 184 | else: 185 | raise 186 | 187 | in_progress = False 188 | retries = 0 189 | 190 | while not in_progress and retries < MAX_RETRIES: 191 | response = CLIENT.describe_stacks(StackName=STACK_NAME.format(cluster_name)) 192 | 193 | for stack in response['Stacks']: 194 | in_progress = bool(re.match(r'(CREATE|UPDATE)_IN_PROGRESS', stack['StackStatus'])) 195 | 196 | # SEE: http://docs.aws.amazon.com/general/latest/gr/api-retries.html 197 | secs = 2**retries*0.1 198 | sleep(secs) 199 | retries += 1 200 | 201 | return NoContent, 202 if in_progress else 500 202 | 203 | @AUTH.login_required 204 | def delete(cluster_name): 205 | CLIENT.delete_stack( 206 | StackName=STACK_NAME.format(cluster_name), 207 | ) 208 | 209 | return NoContent, 204 210 | 211 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=100 212 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /demiurge/aws.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # pylint: disable=missing-docstring,invalid-name 3 | # Copyright (c) 2016 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # SEE: https://coreos.com/kubernetes/docs/latest/getting-started.html 19 | 20 | # pylint: disable=wildcard-import, unused-wildcard-import 21 | from troposphere.constants import * 22 | # pylint: enable=wildcard-import, unused-wildcard-import 23 | 24 | from troposphere import (AWS_REGION, ec2, iam, Base64, FindInMap, Join, Parameter, Ref, Template, 25 | autoscaling, policies, elasticloadbalancing, GetAtt, Output) 26 | from cert import create_cert 27 | 28 | import awacs.ec2 29 | import awacs.iam 30 | import awacs.sts 31 | 32 | TEMPLATE = Template() 33 | 34 | TEMPLATE.add_version('2010-09-09') 35 | 36 | # CoreOS 991.2.0 37 | # SEE: https://coreos.com/os/docs/latest/booting-on-ec2.html#beta 38 | TEMPLATE.add_mapping('RegionMap', { 39 | EU_CENTRAL_1: {'AMI': 'ami-e83ddb87'}, 40 | AP_NORTHEAST_1: {'AMI': 'ami-67e9fd09'}, 41 | SA_EAST_1: {'AMI': 'ami-9666eafa'}, 42 | AP_SOUTHEAST_2: {'AMI': 'ami-9a7d5ef9'}, 43 | AP_SOUTHEAST_1: {'AMI': 'ami-b8d319db'}, 44 | US_EAST_1: {'AMI': 'ami-cfaba5a5'}, 45 | US_WEST_2: {'AMI': 'ami-141df674'}, 46 | US_WEST_1: {'AMI': 'ami-6c037e0c'}, 47 | EU_WEST_1: {'AMI': 'ami-d149cea2'}, 48 | }) 49 | 50 | VPC = TEMPLATE.add_parameter(Parameter( 51 | 'VPC', 52 | Type=VPC_ID, 53 | )) 54 | 55 | SUBNET = TEMPLATE.add_parameter(Parameter( 56 | 'Subnet', 57 | Type=SUBNET_ID, 58 | )) 59 | 60 | KUBERNETES_SERVICE_NETWORK = TEMPLATE.add_parameter(Parameter( 61 | 'KubernetesServiceNetwork', 62 | Type=STRING, 63 | Default='10.3.0.0/24', 64 | )) 65 | 66 | KUBERNETES_SERVICE_NETWORK_MIN = TEMPLATE.add_parameter(Parameter( 67 | 'KubernetesServiceNetworkMin', 68 | Type=STRING, 69 | Default='10.3.0.0', 70 | )) 71 | 72 | KUBERNETES_SERVICE_NETWORK_MAX = TEMPLATE.add_parameter(Parameter( 73 | 'KubernetesServiceNetworkMax', 74 | Type=STRING, 75 | Default='10.3.0.0', 76 | )) 77 | 78 | FLANNEL_NETWORK = TEMPLATE.add_parameter(Parameter( 79 | 'FlannelNetwork', 80 | Type=STRING, 81 | Default='10.1.0.0/16', 82 | )) 83 | 84 | FLANNEL_SUBNET_LEN = TEMPLATE.add_parameter(Parameter( 85 | 'FlannelSubnetLen', 86 | Type=NUMBER, 87 | Default='24', 88 | )) 89 | 90 | FLANNEL_SUBNET_MIN = TEMPLATE.add_parameter(Parameter( 91 | 'FlannelSubnetMin', 92 | Type=STRING, 93 | Default='10.1.0.0', 94 | )) 95 | 96 | FLANNEL_SUBNET_MAX = TEMPLATE.add_parameter(Parameter( 97 | 'FlannelSubnetMax', 98 | Type=STRING, 99 | Default='10.1.24.0', 100 | )) 101 | 102 | CLUSTER_NAME = TEMPLATE.add_parameter(Parameter( 103 | 'ClusterName', 104 | Type=STRING, 105 | Default='kubernetes', 106 | )) 107 | 108 | USERNAME = TEMPLATE.add_parameter(Parameter( 109 | 'Username', 110 | Type=STRING, 111 | Default='admin', 112 | )) 113 | 114 | PASSWORD = TEMPLATE.add_parameter(Parameter( 115 | 'Password', 116 | Type=STRING, 117 | Default='admin', 118 | )) 119 | 120 | CONSUL_DC = TEMPLATE.add_parameter(Parameter( 121 | 'ConsulDC', 122 | Type=STRING, 123 | Default='dc1', 124 | )) 125 | 126 | CONSUL_JOIN = TEMPLATE.add_parameter(Parameter( 127 | 'ConsulJoin', 128 | Type=STRING, 129 | )) 130 | 131 | DOCKER_GRAPH_SIZE = TEMPLATE.add_parameter(Parameter( 132 | 'DockerGraphSize', 133 | Type=NUMBER, 134 | Default='120', 135 | )) 136 | 137 | ROLE = TEMPLATE.add_resource(iam.Role( 138 | 'Role', 139 | AssumeRolePolicyDocument=awacs.aws.Policy( 140 | Statement=[ 141 | awacs.aws.Statement( 142 | Effect=awacs.aws.Allow, 143 | Action=[awacs.sts.AssumeRole], 144 | Principal=awacs.aws.Principal('Service', ['ec2.amazonaws.com']), 145 | ), 146 | ], 147 | ), 148 | )) 149 | 150 | POLICY = TEMPLATE.add_resource(iam.PolicyType( 151 | 'Policy', 152 | PolicyName='coreos', 153 | PolicyDocument=awacs.aws.Policy( 154 | Statement=[ 155 | awacs.aws.Statement( 156 | Effect=awacs.aws.Allow, 157 | Action=[ 158 | awacs.ec2.EC2Action('Describe*'), 159 | awacs.ec2.EC2Action('CreateTags'), 160 | awacs.ec2.EC2Action('AttachVolume'), 161 | awacs.ec2.EC2Action('CreateVolume'), 162 | awacs.ec2.EC2Action('DeleteVolume'), 163 | awacs.ec2.EC2Action('DetachVolume'), 164 | awacs.ec2.EC2Action('SecurityGroup'), 165 | awacs.ec2.EC2Action('CreateRoute'), 166 | awacs.ec2.EC2Action('DeleteRoute'), 167 | awacs.ec2.EC2Action('ReplaceRoute'), 168 | awacs.ec2.EC2Action('ModifyInstanceAttribute'), 169 | ], 170 | Resource=['*'], 171 | ), 172 | awacs.aws.Statement( 173 | Effect=awacs.aws.Allow, 174 | Action=[ 175 | awacs.aws.Action('autoscaling', 'Describe*'), 176 | awacs.aws.Action('elasticloadbalancing', 'Describe*'), 177 | ], 178 | Resource=['*'], 179 | ), 180 | ], 181 | ), 182 | Roles=[Ref(ROLE)], 183 | )) 184 | 185 | INSTANCE_PROFILE = TEMPLATE.add_resource(iam.InstanceProfile( 186 | 'InstanceProfile', 187 | Roles=[Ref(ROLE)], 188 | )) 189 | 190 | INSTANCE_TYPE = TEMPLATE.add_parameter(Parameter( 191 | 'InstanceType', 192 | Type=STRING, 193 | Default=M4_LARGE, 194 | AllowedValues=[M4_LARGE, M4_XLARGE, M4_2XLARGE, M4_4XLARGE, M4_10XLARGE], 195 | )) 196 | 197 | KEY_NAME = TEMPLATE.add_parameter(Parameter( 198 | 'KeyName', 199 | Type=KEY_PAIR_NAME, 200 | )) 201 | 202 | SECURITY_GROUP = TEMPLATE.add_resource(ec2.SecurityGroup( 203 | 'SecurityGroup', 204 | GroupDescription='Kubernetes Security Group', 205 | SecurityGroupIngress=[ 206 | ec2.SecurityGroupRule( 207 | IpProtocol='tcp', 208 | FromPort='22', 209 | ToPort='22', 210 | CidrIp='0.0.0.0/0', 211 | ), 212 | ec2.SecurityGroupRule( 213 | IpProtocol='tcp', 214 | FromPort='5672', 215 | ToPort='5672', 216 | CidrIp='0.0.0.0/0', 217 | ), 218 | ec2.SecurityGroupRule( 219 | IpProtocol='tcp', 220 | FromPort='8080', 221 | ToPort='8080', 222 | CidrIp='0.0.0.0/0', 223 | ), 224 | ec2.SecurityGroupRule( 225 | IpProtocol='tcp', 226 | FromPort='8301', 227 | ToPort='8301', 228 | CidrIp='0.0.0.0/0', 229 | ), 230 | ec2.SecurityGroupRule( 231 | IpProtocol='udp', 232 | FromPort='8301', 233 | ToPort='8301', 234 | CidrIp='0.0.0.0/0', 235 | ), 236 | ec2.SecurityGroupRule( 237 | IpProtocol='tcp', 238 | FromPort='9080', 239 | ToPort='9080', 240 | CidrIp='0.0.0.0/0', 241 | ), 242 | ec2.SecurityGroupRule( 243 | IpProtocol='tcp', 244 | FromPort='30000', 245 | ToPort='32767', 246 | CidrIp='0.0.0.0/0', 247 | ), 248 | ], 249 | SecurityGroupEgress=[ 250 | ec2.SecurityGroupRule( 251 | IpProtocol='-1', 252 | FromPort='-1', 253 | ToPort='-1', 254 | CidrIp='0.0.0.0/0', 255 | ), 256 | ], 257 | VpcId=Ref(VPC), 258 | )) 259 | 260 | TEMPLATE.add_resource(ec2.SecurityGroupIngress( 261 | 'etcdClientCommunicationSecurityGroupIngress', 262 | IpProtocol='tcp', 263 | FromPort='2379', 264 | ToPort='2379', 265 | SourceSecurityGroupId=Ref(SECURITY_GROUP), 266 | GroupId=Ref(SECURITY_GROUP), 267 | )) 268 | 269 | TEMPLATE.add_resource(ec2.SecurityGroupIngress( 270 | 'etcdServerToServerCommunicationSecurityGroupIngress', 271 | IpProtocol='tcp', 272 | FromPort='2380', 273 | ToPort='2380', 274 | SourceSecurityGroupId=Ref(SECURITY_GROUP), 275 | GroupId=Ref(SECURITY_GROUP), 276 | )) 277 | 278 | TEMPLATE.add_resource(ec2.SecurityGroupIngress( 279 | 'flannelVXLANSecurityGroupIngress', 280 | IpProtocol='udp', 281 | FromPort='8472', 282 | ToPort='8472', 283 | SourceSecurityGroupId=Ref(SECURITY_GROUP), 284 | GroupId=Ref(SECURITY_GROUP), 285 | )) 286 | 287 | API_SERVER_SECURITY_GROUP = TEMPLATE.add_resource(ec2.SecurityGroup( 288 | 'ServerSecurityGroup', 289 | GroupDescription='Kubernetes API Server Security Group', 290 | SecurityGroupIngress=[ 291 | ec2.SecurityGroupRule( 292 | IpProtocol='tcp', 293 | FromPort='443', 294 | ToPort='443', 295 | CidrIp='0.0.0.0/0', 296 | ), 297 | ], 298 | VpcId=Ref(VPC), 299 | )) 300 | 301 | API_SERVER_LOAD_BALANCER = TEMPLATE.add_resource(elasticloadbalancing.LoadBalancer( 302 | 'APIServerLoadBalancer', 303 | HealthCheck=elasticloadbalancing.HealthCheck( 304 | Target='TCP:443', 305 | HealthyThreshold='3', 306 | UnhealthyThreshold='5', 307 | Interval='30', 308 | Timeout='5', 309 | ), 310 | Listeners=[ 311 | elasticloadbalancing.Listener( 312 | LoadBalancerPort='443', 313 | InstancePort='443', 314 | Protocol='TCP', 315 | ), 316 | ], 317 | Scheme='internal', 318 | SecurityGroups=[Ref(API_SERVER_SECURITY_GROUP)], 319 | Subnets=[Ref(SUBNET)], 320 | )) 321 | 322 | CONSUL_HTTP_API_SECURITY_GROUP = TEMPLATE.add_resource(ec2.SecurityGroup( 323 | 'ConsulHTTPAPISecurityGroup', 324 | GroupDescription='Consul HTTP API Security Group', 325 | SecurityGroupIngress=[ 326 | ec2.SecurityGroupRule( 327 | IpProtocol='tcp', 328 | FromPort='8500', 329 | ToPort='8500', 330 | CidrIp='0.0.0.0/0', 331 | ), 332 | ], 333 | VpcId=Ref(VPC), 334 | )) 335 | 336 | CONSUL_HTTP_API_LOAD_BALANCER = TEMPLATE.add_resource(elasticloadbalancing.LoadBalancer( 337 | 'ConsulHTTPAPILoadBalancer', 338 | HealthCheck=elasticloadbalancing.HealthCheck( 339 | Target='TCP:8500', 340 | HealthyThreshold='3', 341 | UnhealthyThreshold='5', 342 | Interval='30', 343 | Timeout='5', 344 | ), 345 | Listeners=[ 346 | elasticloadbalancing.Listener( 347 | LoadBalancerPort='8500', 348 | InstancePort='8500', 349 | Protocol='HTTP', 350 | ), 351 | ], 352 | Scheme='internal', 353 | SecurityGroups=[Ref(CONSUL_HTTP_API_SECURITY_GROUP)], 354 | Subnets=[Ref(SUBNET)], 355 | )) 356 | 357 | (CA_KEY_PEM, CA_CERT_PEM, CA_KEY, CA_CERT) = create_cert('kube-ca') 358 | api_san_list = [ 359 | 'DNS:kubernetes', 360 | 'DNS:kubernetes.default', 361 | 'DNS:kubernetes.default.svc', 362 | 'DNS:kubernetes.default.svc.cluster.local', 363 | 'DNS:*.*.elb.amazonaws.com', 364 | 'DNS:*.*.compute.internal', 365 | 'DNS:*.ec2.internal', 366 | 'IP:10.3.0.1', 367 | ] 368 | (API_KEY_PEM, API_CERT_PEM, API_KEY, API_CERT) = create_cert('kube-apiserver', 369 | san_list=api_san_list, sign_key=CA_KEY, sign_cert=CA_CERT) 370 | 371 | LAUNCH_CONFIGURATION = TEMPLATE.add_resource(autoscaling.LaunchConfiguration( 372 | 'LaunchConfiguration', 373 | BlockDeviceMappings=[ 374 | ec2.BlockDeviceMapping( 375 | DeviceName='/dev/sdb', 376 | Ebs=ec2.EBSBlockDevice( 377 | VolumeSize=Ref(DOCKER_GRAPH_SIZE), 378 | ) 379 | ), 380 | ], 381 | IamInstanceProfile=Ref(INSTANCE_PROFILE), 382 | ImageId=FindInMap('RegionMap', Ref(AWS_REGION), 'AMI'), 383 | InstanceType=Ref(INSTANCE_TYPE), 384 | KeyName=Ref(KEY_NAME), 385 | SecurityGroups=[ 386 | Ref(SECURITY_GROUP), 387 | Ref(API_SERVER_SECURITY_GROUP), 388 | Ref(CONSUL_HTTP_API_SECURITY_GROUP) 389 | ], 390 | UserData=Base64(Join('', [ 391 | '#cloud-config\n\n', 392 | 'coreos:\n', 393 | ' update:\n', 394 | ' reboot-strategy: off\n', 395 | ' etcd2:\n', 396 | ' advertise-client-urls: http://$private_ipv4:2379\n', 397 | ' initial-advertise-peer-urls: http://$private_ipv4:2380\n', 398 | ' listen-client-urls: http://0.0.0.0:2379\n', 399 | ' listen-peer-urls: http://$private_ipv4:2380\n', 400 | ' units:\n', 401 | ' - name: update-engine.service\n', 402 | ' command: stop\n', 403 | ' - name: locksmithd.service\n', 404 | ' command: stop\n', 405 | ' - name: format-ephemeral.service\n', 406 | ' command: start\n', 407 | ' content: |\n', 408 | ' [Unit]\n', 409 | ' Description=Formats the ephemeral drive\n', 410 | ' After=dev-xvdb.device\n', 411 | ' Requires=dev-xvdb.device\n', 412 | ' [Service]\n', 413 | ' Type=oneshot\n', 414 | ' RemainAfterExit=yes\n', 415 | ' ExecStart=/usr/sbin/wipefs -f /dev/xvdb\n', 416 | ' ExecStart=/usr/sbin/mkfs.ext4 -F /dev/xvdb\n', 417 | ' - name: var-lib-docker.mount\n', 418 | ' command: start\n', 419 | ' content: |\n', 420 | ' [Unit]\n', 421 | ' Description=Mount ephemeral to /var/lib/docker\n', 422 | ' Requires=format-ephemeral.service\n', 423 | ' After=format-ephemeral.service\n', 424 | ' [Mount]\n', 425 | ' What=/dev/xvdb\n', 426 | ' Where=/var/lib/docker\n', 427 | ' Type=ext4\n', 428 | ' - name: docker.service\n', 429 | ' drop-ins:\n', 430 | ' - name: 10-wait-docker.conf\n', 431 | ' content: |\n', 432 | ' [Unit]\n', 433 | ' After=var-lib-docker.mount\n', 434 | ' Requires=var-lib-docker.mount\n', 435 | ' - name: etcd-peers.service\n', 436 | ' command: start\n', 437 | ' content: |\n', 438 | ' [Unit]\n', 439 | ' Description=Write a file with the etcd peers that we should bootstrap to\n', 440 | ' After=docker.service\n' 441 | ' Requires=docker.service\n\n', 442 | ' [Service]\n', 443 | ' Type=oneshot\n', 444 | ' RemainAfterExit=yes\n', 445 | ' ExecStart=/usr/bin/docker pull monsantoco/etcd-aws-cluster:latest\n', 446 | ' ExecStart=/usr/bin/docker run --rm=true -v /etc/sysconfig/:/etc/sysconfig/ ', 447 | 'monsantoco/etcd-aws-cluster:latest\n', 448 | ' - name: etcd2.service\n' 449 | ' command: start\n', 450 | ' drop-ins:\n' 451 | ' - name: 30-etcd_peers.conf\n', 452 | ' content: |\n', 453 | ' [Unit]\n', 454 | ' After=etcd-peers.service\n' 455 | ' Requires=etcd-peers.service\n\n', 456 | ' [Service]\n', 457 | ' # Load the other hosts in the etcd leader autoscaling group from file\n', 458 | ' EnvironmentFile=/etc/sysconfig/etcd-peers\n', 459 | ' - name: fleet.service\n', 460 | ' command: start\n', 461 | ' - name: docker-flannel.service\n', 462 | ' content: |\n', 463 | ' [Unit]\n\n', 464 | ' [Service]\n', 465 | ' Type=oneshot\n', 466 | ' RemainAfterExit=yes\n', 467 | ' Environment="DOCKER_HOST=unix:///var/run/early-docker.sock"\n', 468 | ' Environment="FLANNEL_VER=0.5.5"\n', 469 | ' Environment="FLANNEL_IMG=quay.io/coreos/flannel"\n', 470 | ' ExecStart=/usr/bin/docker run --net=host --rm --volume=/run:/run \\\n', 471 | ' ${FLANNEL_IMG}:${FLANNEL_VER} \\\n', 472 | ' /opt/bin/mk-docker-opts.sh -d /run/flannel_docker_opts.env \\\n', 473 | ' -f /run/flannel/networks/flannel.env -i\n', 474 | ' ExecStart=/usr/bin/systemctl restart docker.service\n', 475 | ' - name: docker-flannel.path\n', 476 | ' command: start\n', 477 | ' content: |\n', 478 | ' [Unit]\n\n', 479 | ' [Path]\n', 480 | ' PathExists=/run/flannel/networks/flannel.env\n', 481 | ' PathModified=/run/flannel/networks/flannel.env\n\n', 482 | ' [Install]\n', 483 | ' WantedBy=multi-user.target\n', 484 | ' - name: flanneld.service\n', 485 | ' command: start\n', 486 | ' drop-ins:\n', 487 | ' - name: 50-network-config.conf\n', 488 | ' content: |\n', 489 | ' [Service]\n', 490 | ' Type=simple\n', 491 | ' ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/flannel/config \'{ "Network": "', 492 | Ref(FLANNEL_NETWORK), '", "SubnetLen": ', Ref(FLANNEL_SUBNET_LEN), ', "SubnetMin": "', 493 | Ref(FLANNEL_SUBNET_MIN), '", "SubnetMax": "', Ref(FLANNEL_SUBNET_MAX), '" }\'\n', 494 | ' ExecStartPre=/usr/bin/etcdctl set /coreos.com/network/k8s/config \'{ "Network": "', 495 | Ref(KUBERNETES_SERVICE_NETWORK), '", "SubnetLen": 24, "SubnetMin": "', 496 | Ref(KUBERNETES_SERVICE_NETWORK_MIN), '", "SubnetMax": "', Ref(KUBERNETES_SERVICE_NETWORK_MAX), 497 | '", "Backend": {"Type": "aws-vpc"}}\'\n', 498 | ' ExecStart=\n', 499 | ' ExecStart=/usr/bin/docker run --net=host --privileged=true --rm \\\n', 500 | ' --volume=/run/flannel:/run/flannel \\\n', 501 | ' --env=AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \\\n', 502 | ' --env=AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \\\n', 503 | ' --env-file=${FLANNEL_ENV_FILE} \\\n', 504 | ' --volume=/usr/share/ca-certificates:/etc/ssl/certs:ro \\\n', 505 | ' --volume=${ETCD_SSL_DIR}:${ETCD_SSL_DIR}:ro \\\n', 506 | ' ${FLANNEL_IMG}:${FLANNEL_VER} /opt/bin/flanneld --ip-masq=true \\\n', 507 | ' --networks=flannel,k8s\n\n', 508 | ' ExecStartPost=\n', 509 | ' - name: kubelet.service\n', 510 | ' command: start\n', 511 | ' drop-ins:\n', 512 | ' - name: local.conf\n', 513 | ' content: |\n', 514 | ' [Service]\n', 515 | ' Environment="RKT_OPTS=--volume=resolv,kind=host,source=/etc/resolv.conf ', 516 | '--mount volume=resolv,target=/etc/resolv.conf"\n', 517 | ' Environment=KUBELET_VERSION=v1.2.2_coreos.0\n', 518 | ' ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests\n\n', 519 | ' ExecStart=\n', 520 | ' ExecStart=/usr/lib/coreos/kubelet-wrapper \\\n', 521 | ' --api-servers=http://127.0.0.1:8080 \\\n', 522 | ' --allow-privileged=true \\\n', 523 | ' --cluster-dns=', Ref(CONSUL_JOIN), ' \\\n', 524 | ' --cloud-provider=aws \\\n', 525 | ' --config=/etc/kubernetes/manifests\n', 526 | ' Restart=always\n', 527 | ' RestartSec=10\n', 528 | ' - name: kube-system.service\n', 529 | ' command: start\n', 530 | ' content: |\n', 531 | ' [Unit]\n', 532 | ' After=kubelet.service\n', 533 | ' Requires=kubelet.service\n\n', 534 | ' [Service]\n', 535 | ' Type=oneshot\n', 536 | ' ExecStart=/bin/sh -c \'while true; do curl -H "Content-Type: application/json" ', 537 | '-XPOST -d\\\'{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kube-system"}}\\\'', 538 | ' -sS "http://127.0.0.1:8080/api/v1/namespaces" && break || sleep 20; done\'\n', 539 | 'write_files:\n', 540 | ' - path: /etc/kubernetes/ssl/ca.pem\n', 541 | ' permissions: \'0600\'\n', 542 | ' content: |\n', 543 | ''.join([' {}\n'.format(line) for line in CA_CERT_PEM.split('\n')]).rstrip('\n '), 544 | '\n', 545 | ' - path: /etc/kubernetes/ssl/apiserver.pem\n', 546 | ' permissions: \'0600\'\n', 547 | ' content: |\n', 548 | ''.join([' {}\n'.format(line) for line in API_CERT_PEM.split('\n')]).rstrip('\n '), 549 | '\n', 550 | ' - path: /etc/kubernetes/ssl/apiserver-key.pem\n', 551 | ' permissions: \'0600\'\n', 552 | ' content: |\n', 553 | ''.join([' {}\n'.format(line) for line in API_KEY_PEM.split('\n')]).rstrip('\n '), 554 | '\n', 555 | ' - path: /etc/kubernetes/manifests/kube-apiserver.yaml\n', 556 | ' content: |\n', 557 | ' apiVersion: v1\n', 558 | ' kind: Pod\n', 559 | ' metadata:\n', 560 | ' name: kube-apiserver\n', 561 | ' namespace: kube-system\n', 562 | ' spec:\n', 563 | ' hostNetwork: true\n', 564 | ' containers:\n', 565 | ' - name: kube-apiserver\n', 566 | ' image: quay.io/coreos/hyperkube:v1.2.2_coreos.0\n', 567 | ' command:\n', 568 | ' - /hyperkube\n', 569 | ' - apiserver\n', 570 | ' - --etcd-servers=http://127.0.0.1:2379\n', 571 | ' - --allow-privileged=true\n', 572 | ' - --service-cluster-ip-range=', Ref(KUBERNETES_SERVICE_NETWORK), '\n', 573 | ' - --secure-port=443\n', 574 | ' - --admission-control=NamespaceLifecycle,LimitRanger,SecurityContextDeny,', 575 | 'ResourceQuota,ServiceAccount\n', 576 | ' - --runtime-config=extensions/v1beta1/deployments=true,', 577 | 'extensions/v1beta1/daemonsets=true\n', 578 | ' - --external-hostname=', GetAtt(API_SERVER_LOAD_BALANCER, 'DNSName'), '\n', 579 | ' - --basic-auth-file=/srv/kubernetes/basic_auth.csv\n', 580 | ' - --cloud-provider=aws\n', 581 | ' - --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem\n', 582 | ' - --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem\n', 583 | ' - --client-ca-file=/etc/kubernetes/ssl/ca.pem\n', 584 | ' - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem\n', 585 | ' ports:\n', 586 | ' - containerPort: 443\n', 587 | ' hostPort: 443\n', 588 | ' name: https\n', 589 | ' - containerPort: 8080\n', 590 | ' hostPort: 8080\n', 591 | ' name: local\n', 592 | ' volumeMounts:\n', 593 | ' - mountPath: /etc/kubernetes/ssl\n', 594 | ' name: ssl-certs-generated\n', 595 | ' readOnly: true\n', 596 | ' - mountPath: /etc/ssl/certs\n', 597 | ' name: ssl-certs-host\n', 598 | ' readOnly: true\n', 599 | ' - mountPath: /srv/kubernetes/basic_auth.csv\n', 600 | ' name: basic-auth-file\n', 601 | ' readOnly: true\n', 602 | ' volumes:\n', 603 | ' - hostPath:\n', 604 | ' path: /etc/kubernetes/ssl\n', 605 | ' name: ssl-certs-generated\n', 606 | ' - hostPath:\n', 607 | ' path: /usr/share/ca-certificates\n', 608 | ' name: ssl-certs-host\n', 609 | ' - hostPath:\n', 610 | ' path: /srv/kubernetes/basic_auth.csv\n', 611 | ' name: basic-auth-file\n', 612 | ' - path: /etc/kubernetes/manifests/kube-proxy.yaml\n', 613 | ' content: |\n', 614 | ' apiVersion: v1\n', 615 | ' kind: Pod\n', 616 | ' metadata:\n', 617 | ' name: kube-proxy\n', 618 | ' namespace: kube-system\n', 619 | ' spec:\n', 620 | ' hostNetwork: true\n', 621 | ' containers:\n', 622 | ' - name: kube-proxy\n', 623 | ' image: quay.io/coreos/hyperkube:v1.2.2_coreos.0\n', 624 | ' command:\n', 625 | ' - /hyperkube\n', 626 | ' - proxy\n', 627 | ' - --master=http://127.0.0.1:8080\n', 628 | ' - --proxy-mode=iptables\n', 629 | ' securityContext:\n', 630 | ' privileged: true\n', 631 | ' volumeMounts:\n', 632 | ' - mountPath: /etc/ssl/certs\n', 633 | ' name: ssl-certs-host\n', 634 | ' readOnly: true\n', 635 | ' volumes:\n', 636 | ' - hostPath:\n', 637 | ' path: /usr/share/ca-certificates\n', 638 | ' name: ssl-certs-host\n', 639 | ' - path: /etc/kubernetes/manifests/kube-controller-manager.yaml\n', 640 | ' content: |\n', 641 | ' apiVersion: v1\n', 642 | ' kind: Pod\n', 643 | ' metadata:\n', 644 | ' name: kube-controller-manager\n', 645 | ' namespace: kube-system\n', 646 | ' spec:\n', 647 | ' hostNetwork: true\n', 648 | ' containers:\n', 649 | ' - name: kube-controller-manager\n', 650 | ' image: quay.io/coreos/hyperkube:v1.2.2_coreos.0\n', 651 | ' command:\n', 652 | ' - /hyperkube\n', 653 | ' - controller-manager\n', 654 | ' - --master=http://127.0.0.1:8080\n', 655 | ' - --leader-elect=true\n', 656 | ' - --service-sync-period=10m\n', 657 | ' - --node-sync-period=5m\n', 658 | ' - --cloud-provider=aws\n', 659 | ' - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem\n', 660 | ' - --root-ca-file=/etc/kubernetes/ssl/ca.pem\n', 661 | ' livenessProbe:\n', 662 | ' httpGet:\n', 663 | ' host: 127.0.0.1\n', 664 | ' path: /healthz\n', 665 | ' port: 10252\n', 666 | ' initialDelaySeconds: 15\n', 667 | ' timeoutSeconds: 1\n', 668 | ' volumeMounts:\n', 669 | ' - mountPath: /etc/kubernetes/ssl\n', 670 | ' name: ssl-certs-generated\n', 671 | ' readOnly: true\n', 672 | ' - mountPath: /etc/ssl/certs\n', 673 | ' name: ssl-certs-host\n', 674 | ' readOnly: true\n', 675 | ' volumes:\n', 676 | ' - hostPath:\n', 677 | ' path: /etc/kubernetes/ssl\n', 678 | ' name: ssl-certs-generated\n', 679 | ' - hostPath:\n', 680 | ' path: /usr/share/ca-certificates\n', 681 | ' name: ssl-certs-host\n', 682 | ' - path: /etc/kubernetes/manifests/kube-scheduler.yaml\n', 683 | ' content: |\n', 684 | ' apiVersion: v1\n', 685 | ' kind: Pod\n', 686 | ' metadata:\n', 687 | ' name: kube-scheduler\n', 688 | ' namespace: kube-system\n', 689 | ' spec:\n', 690 | ' hostNetwork: true\n', 691 | ' containers:\n', 692 | ' - name: kube-scheduler\n', 693 | ' image: quay.io/coreos/hyperkube:v1.2.2_coreos.0\n', 694 | ' command:\n', 695 | ' - /hyperkube\n', 696 | ' - scheduler\n', 697 | ' - --master=http://127.0.0.1:8080\n', 698 | ' - --leader-elect=true\n', 699 | ' livenessProbe:\n', 700 | ' httpGet:\n', 701 | ' host: 127.0.0.1\n', 702 | ' path: /healthz\n', 703 | ' port: 10251\n', 704 | ' initialDelaySeconds: 15\n', 705 | ' timeoutSeconds: 1\n', 706 | ' - path: /srv/kubernetes/basic_auth.csv\n', 707 | ' content: |\n', 708 | ' ', Ref(PASSWORD), ',', Ref(USERNAME), ',admin\n', 709 | ' - path: /etc/kubernetes/manifests/kube2consul.yaml\n', 710 | ' content: |\n', 711 | ' apiVersion: v1\n', 712 | ' kind: Pod\n', 713 | ' metadata:\n', 714 | ' name: kube2consul\n', 715 | ' namespace: kube-system\n', 716 | ' spec:\n', 717 | ' hostNetwork: true\n', 718 | ' containers:\n', 719 | ' - name: consul-agent\n', 720 | ' image: gliderlabs/consul-agent:0.6\n', 721 | ' args:\n', 722 | ' - -advertise=$private_ipv4\n', 723 | ' - -dc=', Ref(CONSUL_DC), '\n', 724 | ' - -join=', Ref(CONSUL_JOIN), '\n', 725 | ' ports:\n', 726 | ' - hostPort: 8301\n', 727 | ' containerPort: 8301\n', 728 | ' protocol: TCP\n', 729 | ' hostIP: $private_ipv4\n', 730 | ' - hostPort: 8301\n', 731 | ' containerPort: 8301\n', 732 | ' protocol: UDP\n', 733 | ' hostIP: $private_ipv4\n', 734 | ' - hostPort: 8500\n', 735 | ' containerPort: 8500\n', 736 | ' protocol: TCP\n', 737 | ' hostIP: $private_ipv4\n', 738 | ' - name: kube2consul\n', 739 | ' image: jmccarty3/kube2consul:latest\n', 740 | ' command:\n', 741 | ' - /kube2consul\n', 742 | ' - -consul-agent=http://127.0.0.1:8500\n', 743 | ' - -kube_master_url=http://127.0.0.1:8080\n', 744 | ])), 745 | )) 746 | 747 | AUTO_SCALING_GROUP = TEMPLATE.add_resource(autoscaling.AutoScalingGroup( 748 | 'AutoScalingGroup', 749 | DesiredCapacity='1', 750 | Tags=[autoscaling.Tag('Name', 'Kubernetes Master', True)], 751 | LaunchConfigurationName=Ref(LAUNCH_CONFIGURATION), 752 | LoadBalancerNames=[ 753 | Ref(API_SERVER_LOAD_BALANCER), 754 | Ref(CONSUL_HTTP_API_LOAD_BALANCER) 755 | ], 756 | MinSize='1', 757 | MaxSize='3', 758 | VPCZoneIdentifier=[Ref(SUBNET)], 759 | UpdatePolicy=policies.UpdatePolicy( 760 | AutoScalingRollingUpdate=policies.AutoScalingRollingUpdate( 761 | MinInstancesInService='1', 762 | MaxBatchSize='1', 763 | ), 764 | ), 765 | )) 766 | 767 | TEMPLATE.add_output(Output( 768 | 'APIServer', 769 | Value=Join('', ['https://', GetAtt(API_SERVER_LOAD_BALANCER, 'DNSName')]), 770 | )) 771 | 772 | TEMPLATE.add_output(Output( 773 | 'ConsulHTTPAPI', 774 | Value=Join('', ['http://', GetAtt(CONSUL_HTTP_API_LOAD_BALANCER, 'DNSName'), ':8500']), 775 | )) 776 | 777 | TEMPLATE.add_output(Output( 778 | 'CAKey', 779 | Value=CA_KEY_PEM, 780 | )) 781 | 782 | TEMPLATE.add_output(Output( 783 | 'CACert', 784 | Value=CA_CERT_PEM, 785 | )) 786 | 787 | if __name__ == '__main__': 788 | print TEMPLATE.to_json() 789 | 790 | # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=100 791 | --------------------------------------------------------------------------------