├── circle.yml ├── examples ├── legostack │ ├── README.txt │ ├── templates │ │ ├── asg-scheduled-capacity-change.json │ │ ├── vpc-sg-access-bridge.json │ │ ├── vpc-subnet-dualaz.json │ │ ├── vpc-generic-instance-singleaz.json │ │ ├── vpc-generic-asg-dualaz.json │ │ ├── vpc-base-dualaz.json │ │ └── vpc-nat-dualaz.json │ └── legostack.yaml ├── cumulus_example_stack.yaml ├── instance_layer.json └── vpc_layer.json ├── .gitignore ├── LICENSE ├── bin └── cumulus ├── setup.py ├── cumulus ├── __init__.py ├── CFStack.py └── MegaStack.py └── README.rst /circle.yml: -------------------------------------------------------------------------------- 1 | test: 2 | override: 3 | - pip install flake8 4 | - flake8 --ignore=E402 bin/cumulus . 5 | -------------------------------------------------------------------------------- /examples/legostack/README.txt: -------------------------------------------------------------------------------- 1 | This examples demonstrates the modular approach to CloudFormation made possible by cumulus. 2 | 3 | Make sure to create a keypair in your target EC2 region (example uses us-west-1) and set the keypair name in legostack.yaml, look for ParamGlobalRSAKey. 4 | 5 | ~ msessa -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | var 14 | sdist 15 | develop-eggs 16 | .installed.cfg 17 | lib 18 | lib64 19 | 20 | # Installer logs 21 | pip-log.txt 22 | 23 | # Unit test / coverage reports 24 | .coverage 25 | .tox 26 | nosetests.xml 27 | 28 | # Translations 29 | *.mo 30 | 31 | # Mr Developer 32 | .mr.developer.cfg 33 | .project 34 | .pydevproject 35 | 36 | # IDEA IDE 37 | .idea 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013 CatchOfTheDay.com.au Pty Ltd 2 | 3 | Author: Peter Hall 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | -------------------------------------------------------------------------------- /bin/cumulus: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Copyright 2013 CatchOfTheDay.com.au Pty Ltd 3 | 4 | Author: Peter Hall 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License.""" 17 | 18 | import sys 19 | from os.path import dirname, abspath 20 | 21 | # Add one directory up to python path, so it can find cumulus package 22 | BIN_DIR = dirname(abspath(__file__)) 23 | ROOT_DIR = dirname(BIN_DIR) 24 | sys.path.insert(0, ROOT_DIR) 25 | 26 | from cumulus import main 27 | 28 | main() 29 | -------------------------------------------------------------------------------- /examples/legostack/templates/asg-scheduled-capacity-change.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Pluggable stack to change ASG desiredcapacity on two recurring schedules", 4 | "Parameters": { 5 | "ParamAutoScalingGroup": { 6 | "Type": "String", 7 | "Description": "ASG Identifier" 8 | }, 9 | "ParamFirstRecurrence": { 10 | "Type": "String", 11 | "Description": "Cron-Like expression for the first schedule" 12 | }, 13 | "ParamFirstCapacity": { 14 | "Type": "Number", 15 | "MinValue" : "0", 16 | "Description": "Capacity on first schedule" 17 | }, 18 | "ParamSecondRecurrence": { 19 | "Type": "String", 20 | "Description": "Cron-Like expression for the second schedule" 21 | }, 22 | "ParamSecondCapacity" : { 23 | "Type": "Number", 24 | "MinValue" : "0", 25 | "Description": "Capacity on second schedule" 26 | } 27 | }, 28 | "Resources": { 29 | "FirstSchedule" : { 30 | "Type" : "AWS::AutoScaling::ScheduledAction", 31 | "Properties" : { 32 | "AutoScalingGroupName" : { "Ref" : "ParamAutoScalingGroup" }, 33 | "DesiredCapacity" : { "Ref" : "ParamFirstCapacity" }, 34 | "Recurrence" : { "Ref" : "ParamFirstRecurrence" } 35 | } 36 | }, 37 | "SecondSchedule" : { 38 | "Type" : "AWS::AutoScaling::ScheduledAction", 39 | "Properties" : { 40 | "AutoScalingGroupName" : { "Ref" : "ParamAutoScalingGroup" }, 41 | "DesiredCapacity" : { "Ref" : "ParamSecondCapacity" }, 42 | "Recurrence" : { "Ref" : "ParamSecondRecurrence" } 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /examples/legostack/templates/vpc-sg-access-bridge.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Generic template for bridging network access between stacks", 4 | "Parameters": { 5 | "ParamSourceSG": { 6 | "Type": "String", 7 | "Default" : "", 8 | "Description": "Source security group. Can only be used if SourceCIDR is omitted" 9 | }, 10 | "ParamSourceCIDR": { 11 | "Type": "String", 12 | "Default": "", 13 | "Description": "Source CIDR network. Can only be used if SourceSG is omitted" 14 | }, 15 | "ParamDestinationSG": { 16 | "Type": "String", 17 | "Description": "Destination security group" 18 | }, 19 | "ParamFromPort": { 20 | "Type": "Number", 21 | "Description": "First port in the range" 22 | }, 23 | "ParamToPort": { 24 | "Type": "Number", 25 | "Description": "Last port in the range" 26 | }, 27 | "ParamIpProtocol": { 28 | "Type": "String", 29 | "Default": "tcp", 30 | "Description" : "Protocol (tcp/udp/icmp)" 31 | } 32 | }, 33 | "Conditions" : { 34 | "UseCIDR" : {"Fn::And": [ 35 | {"Fn::Not": [ {"Fn::Equals" : [{"Ref" : "ParamSourceCIDR"}, ""]} ] }, 36 | {"Fn::Equals" : [{"Ref" : "ParamSourceSG"}, ""]} 37 | ] }, 38 | "UseSG" : {"Fn::And": [ 39 | {"Fn::Not": [ {"Fn::Equals" : [{"Ref" : "ParamSourceSG"}, ""]} ] }, 40 | {"Fn::Equals" : [{"Ref" : "ParamSourceCIDR"}, ""]} 41 | ] } 42 | }, 43 | "Resources": { 44 | "BridgeAccessRuleFromSG": { 45 | "Condition" : "UseSG", 46 | "Type": "AWS::EC2::SecurityGroupIngress", 47 | "Properties": { 48 | "GroupId" : { "Ref": "ParamDestinationSG" }, 49 | "SourceSecurityGroupId" : { "Ref": "ParamSourceSG" }, 50 | "IpProtocol" : { "Ref" : "ParamIpProtocol" }, 51 | "FromPort" : { "Ref" : "ParamFromPort" }, 52 | "ToPort" : { "Ref" : "ParamToPort" } 53 | } 54 | }, 55 | "BridgeAccessRuleFromCIDR": { 56 | "Condition" : "UseCIDR", 57 | "Type": "AWS::EC2::SecurityGroupIngress", 58 | "Properties": { 59 | "GroupId" : { "Ref": "ParamDestinationSG" }, 60 | "CidrIp" : { "Ref": "ParamSourceCIDR" }, 61 | "IpProtocol" : { "Ref" : "ParamIpProtocol" }, 62 | "FromPort" : { "Ref" : "ParamFromPort" }, 63 | "ToPort" : { "Ref" : "ParamToPort" } 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /examples/cumulus_example_stack.yaml: -------------------------------------------------------------------------------- 1 | #Basic example of what cumulus can do. The following takes two simple cloudformation template files and creates a VPC with a instance running in it. 2 | 3 | #Overall stack name. All stacks in CF get prefixed with this name. 4 | example-stack: 5 | # Optional SNS ARN to receive notification about the sub-stacks 6 | # sns-topic-arn: arn:aws:sns:region:account:topic 7 | # Same as above but with multiple topics 8 | # sns-topic-arn: 9 | # - arn:aws:sns:region:account:topic1 10 | # - arn:aws:sns:region:account:topic2 11 | #The region Cumulus will create the stack in 12 | region: us-west-2 13 | #Turn on colour cloudformation event status output 14 | highlight-output: true 15 | #Limit the account in which this stack can be run 16 | account_id: 972549067366 17 | stacks: 18 | #Base stack, has the same name as the top level. Cumulus knows not to call it examplestack-example-stack. 19 | #This template has no parameters to pass in 20 | example-stack: 21 | cf_template: vpc_layer.json 22 | #No dependencies on this stack. Still include the empty depends variable 23 | depends: 24 | params: 25 | #A simple stack that launches a instance inside our VPC above 26 | #This stack will show up in CF as example-stack-instance. 27 | instance: 28 | cf_template: instance_layer.json 29 | # You can optionally override the global SNS topic by specifying it in the substack 30 | # sns-topic-arn: arn:aws:sns:region:account:topic3 31 | #One dependency for this stack, no reason a stack can't depend on multiple other stacks. Just be careful about loops 32 | #List dependencies for this stack, be careful when referring to multiple stacks to ensure you don't create a dependency loop 33 | depends: 34 | - example-stack 35 | params: 36 | #Statically setting the values for SSHLocation, InstanceType and KeyNmae 37 | SSHLocation: 38 | value: 0.0.0.0/0 39 | InstanceType: 40 | value: t1.micro 41 | KeyName: 42 | value: example 43 | #This is where cumulus adds real value, dynamically get the value of the vpcId from example-stack 44 | VPC: 45 | #source: which CF stack to look for the value 46 | #type: can me resource, parameter or output. Which type of variable are we looking for 47 | #variable: what is the logical ID of the resource or the name of the parmeter / output 48 | source : example-stack 49 | type : resource 50 | variable : VPC 51 | Subnet: 52 | source : example-stack 53 | type : resource 54 | variable : Subnet 55 | 56 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """A setuptools based setup module. 2 | 3 | See: 4 | https://packaging.python.org/en/latest/distributing.html 5 | https://github.com/pypa/sampleproject 6 | """ 7 | 8 | # Always prefer setuptools over distutils 9 | from setuptools import setup, find_packages 10 | 11 | setup( 12 | name='cumulus-aws', 13 | 14 | # Versions should comply with PEP440. For a discussion on single-sourcing 15 | # the version across setup.py and the project code, see 16 | # https://packaging.python.org/en/latest/single_source_version.html 17 | version='1.0.0', 18 | 19 | description='Manages AWS Cloudformation stacks across multiple CF' 20 | ' templates', 21 | 22 | # The project's main homepage. 23 | url='https://github.com/cotdsa/cumulus', 24 | 25 | # Author details 26 | author='Peter Hall', 27 | author_email='cumulus@peterkh.net', 28 | 29 | # Choose your license 30 | license='Apache Software License 2.0', 31 | 32 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 33 | classifiers=[ 34 | # How mature is this project? Common values are 35 | # 3 - Alpha 36 | # 4 - Beta 37 | # 5 - Production/Stable 38 | 'Development Status :: 5 - Production/Stable', 39 | 40 | # Indicate who your project is intended for 41 | 'Intended Audience :: System Administrators', 42 | 'Topic :: System :: Systems Administration', 43 | 44 | # Pick your license as you wish (should match "license" above) 45 | 'License :: OSI Approved :: MIT License', 46 | 47 | # Specify the Python versions you support here. In particular, ensure 48 | # that you indicate whether you support Python 2, Python 3 or both. 49 | 'Programming Language :: Python :: 2.7', 50 | ], 51 | 52 | # You can just specify the packages manually here if your project is 53 | # simple. Or you can use find_packages(). 54 | packages=find_packages(), 55 | 56 | # List run-time dependencies here. These will be installed by pip when 57 | # your project is installed. For an analysis of "install_requires" vs pip's 58 | # requirements files see: 59 | # https://packaging.python.org/en/latest/requirements.html 60 | install_requires=['PyYAML', 'argparse', 'boto', 'simplejson', 'pystache'], 61 | 62 | # List additional groups of dependencies here (e.g. development 63 | # dependencies). You can install these using the following syntax, 64 | # for example: 65 | # $ pip install -e .[dev,test] 66 | # extras_require={ 67 | # 'dev': ['check-manifest'], 68 | # 'test': ['coverage'], 69 | # }, 70 | 71 | include_package_data=True, 72 | 73 | # To provide executable scripts, use entry points in preference to the 74 | # "scripts" keyword. Entry points provide cross-platform support and allow 75 | # pip to create the appropriate form of executable for the target platform. 76 | entry_points={ 77 | 'console_scripts': [ 78 | 'cumulus=cumulus:main', 79 | ], 80 | }, 81 | ) 82 | -------------------------------------------------------------------------------- /examples/legostack/templates/vpc-subnet-dualaz.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Subnet across two availability zones", 4 | "Parameters": { 5 | "ParamVPC": { 6 | "Type": "AWS::EC2::VPC::Id", 7 | "Description": "VPC ID" 8 | }, 9 | "ParamCIDR1": { 10 | "Type": "String", 11 | "Description": "Subnet 1 CIDR", 12 | "AllowedPattern": "([0-9]{1,3}\\.){3}[0-9]{1,3}\/[0-9]{1,2}", 13 | "ConstraintDescription": "Parameter CIDRBlock must be a valid CIDR expression." 14 | }, 15 | "ParamAZ1": { 16 | "Type": "String", 17 | "Description": "Subnet 1 Availability zone", 18 | "AllowedPattern": "[a-z][a-z]-[a-z]+-[0-9][a-z]", 19 | "ConstraintDescription": "Must select a valid AZ name within the target region." 20 | }, 21 | "ParamRouteTable1": { 22 | "Type": "String", 23 | "Description": "Route table to attach the first subnet to (optional)", 24 | "Default" : "" 25 | }, 26 | "ParamCIDR2": { 27 | "Type": "String", 28 | "Description": "Subnet 2 CIDR", 29 | "AllowedPattern": "([0-9]{1,3}\\.){3}[0-9]{1,3}\/[0-9]{1,2}", 30 | "ConstraintDescription": "Parameter CIDRBlock must be a valid CIDR expression." 31 | }, 32 | "ParamAZ2": { 33 | "Type": "String", 34 | "Description": "Subnet 2 Availability zone", 35 | "AllowedPattern": "[a-z][a-z]-[a-z]+-[0-9][a-z]", 36 | "ConstraintDescription": "Must select a valid AZ name within the target region." 37 | }, 38 | "ParamRouteTable2": { 39 | "Type": "String", 40 | "Description": "Route table to attach the second subnet to (optional)", 41 | "Default" : "" 42 | } 43 | }, 44 | "Conditions" : { 45 | "HasRouteTable" : { "Fn::And" : [ 46 | { "Fn::Not" : [ { "Fn::Equals" : [{"Ref" : "ParamRouteTable1"}, ""] } ] }, 47 | { "Fn::Not" : [ { "Fn::Equals" : [{"Ref" : "ParamRouteTable2"}, ""] } ] } 48 | ] } 49 | }, 50 | "Resources": { 51 | "Subnet1" : { 52 | "Type" : "AWS::EC2::Subnet", 53 | "Properties" : { 54 | "VpcId" : { "Ref" : "ParamVPC" }, 55 | "CidrBlock" : { "Ref" : "ParamCIDR1" }, 56 | "AvailabilityZone" : { "Ref" : "ParamAZ1" } 57 | } 58 | }, 59 | "Subnet2" : { 60 | "Type" : "AWS::EC2::Subnet", 61 | "Properties" : { 62 | "VpcId" : { "Ref" : "ParamVPC" }, 63 | "CidrBlock" : { "Ref" : "ParamCIDR2" }, 64 | "AvailabilityZone" : { "Ref" : "ParamAZ2" } 65 | } 66 | }, 67 | "RouteTableAssociation1" : { 68 | "Type" : "AWS::EC2::SubnetRouteTableAssociation", 69 | "Condition": "HasRouteTable", 70 | "Properties" : { 71 | "SubnetId" : { "Ref" : "Subnet1" }, 72 | "RouteTableId" : { "Ref" : "ParamRouteTable1" } 73 | } 74 | }, 75 | "RouteTableAssociation2" : { 76 | "Type" : "AWS::EC2::SubnetRouteTableAssociation", 77 | "Condition": "HasRouteTable", 78 | "Properties" : { 79 | "SubnetId" : { "Ref" : "Subnet2" }, 80 | "RouteTableId" : { "Ref" : "ParamRouteTable2" } 81 | } 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /examples/legostack/templates/vpc-generic-instance-singleaz.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Single-AZ generic instance", 4 | "Parameters": { 5 | "ParamInstSubnet": { 6 | "Type": "AWS::EC2::Subnet::Id", 7 | "Description": "Subnet for instance" 8 | }, 9 | "ParamVPC": { 10 | "Type": "AWS::EC2::VPC::Id", 11 | "Description": "VPC ID" 12 | }, 13 | "ParamRSAKeyName" : { 14 | "Type": "AWS::EC2::KeyPair::KeyName", 15 | "Description": "RSA Key for instances" 16 | }, 17 | "ParamIamInstanceProfile" : { 18 | "Type": "String", 19 | "Description": "IAM Role Profile for instances" 20 | }, 21 | "ParamInstanceAMI" : { 22 | "Type": "String", 23 | "Description": "ID of instance AMI" 24 | }, 25 | "ParamInstanceDefaultSG": { 26 | "Type": "AWS::EC2::SecurityGroup::Id", 27 | "Description": "ID of Default Management SG" 28 | }, 29 | "ParamInstanceType" : { 30 | "Type": "String", 31 | "Default" : "c3.large", 32 | "Description": "Instance type for newly created instances" 33 | }, 34 | "ParamUseElasticIP": { 35 | "Default": "false", 36 | "Description": "Whether to create and assign an elastic ip to the instance", 37 | "Type": "String", 38 | "AllowedValues": [ 39 | "true", 40 | "false" 41 | ] 42 | } 43 | }, 44 | "Conditions" : { 45 | "UseElasticIP" : {"Fn::Equals" : [{"Ref" : "ParamUseElasticIP"}, "true"]} 46 | }, 47 | "Resources": { 48 | "SGInstance": { 49 | "Type": "AWS::EC2::SecurityGroup", 50 | "Properties": { 51 | "GroupDescription": "Security group for instance", 52 | "VpcId": { 53 | "Ref": "ParamVPC" 54 | }, 55 | "SecurityGroupIngress": [ 56 | ] 57 | } 58 | }, 59 | "Instance": { 60 | "Type": "AWS::EC2::Instance", 61 | "Properties": { 62 | "InstanceType": { "Ref" : "ParamInstanceType" }, 63 | "IamInstanceProfile": { 64 | "Ref": "ParamIamInstanceProfile" 65 | }, 66 | "UserData": { "Fn::Base64": { "Fn::Join": [ "", 67 | [ 68 | "#!/bin/bash\n" 69 | ] 70 | ] } }, 71 | "SubnetId": { 72 | "Ref": "ParamInstSubnet" 73 | }, 74 | "SourceDestCheck": "true", 75 | "ImageId": { 76 | "Ref" : "ParamInstanceAMI" 77 | }, 78 | "KeyName": { "Ref": "ParamRSAKeyName" }, 79 | "SecurityGroupIds": [ 80 | { "Ref": "ParamInstanceDefaultSG" }, 81 | { "Ref": "SGInstance"} 82 | ], 83 | "Tags": [ 84 | { 85 | "Key": "Name", 86 | "Value": { "Fn::Join": [ "@", [ "inst", { "Ref": "AWS::StackName" } ] ] } 87 | } 88 | ] 89 | } 90 | }, 91 | "InstIPAddress1": { 92 | "Type": "AWS::EC2::EIP", 93 | "DependsOn": "Instance", 94 | "Condition": "UseElasticIP", 95 | "Properties": { 96 | "Domain": "vpc", 97 | "InstanceId": { "Ref" : "Instance" } 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /cumulus/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Peter Hall 20/03/2013 3 | # 4 | # Take a yaml file that describes a full VPC made of multiple CF templates 5 | 6 | import argparse 7 | import logging 8 | from cumulus.MegaStack import MegaStack 9 | 10 | 11 | def main(): 12 | """ 13 | Entry function for cumulus. 14 | """ 15 | conf_parser = argparse.ArgumentParser() 16 | conf_parser.add_argument( 17 | "-y", "--yamlfile", 18 | dest="yamlfile", required=True, 19 | help="The yaml file to read the VPC mega stack configuration from") 20 | conf_parser.add_argument( 21 | "-a", "--action", 22 | dest="action", required=True, 23 | help="The action to preform: create, check, update, delete or watch") 24 | conf_parser.add_argument( 25 | "-l", "--log", 26 | dest="loglevel", required=False, default="info", 27 | help="Log Level for output messages," 28 | " CRITICAL, ERROR, WARNING, INFO or DEBUG") 29 | conf_parser.add_argument( 30 | "-L", "--botolog", 31 | dest="botologlevel", required=False, default="critical", 32 | help="Log Level for boto, CRITICAL, ERROR, WARNING, INFO or DEBUG") 33 | conf_parser.add_argument( 34 | "-s", "--stack", 35 | dest="stackname", required=False, 36 | help="The stack name, used with the watch action," 37 | " ignored for other actions") 38 | args = conf_parser.parse_args() 39 | 40 | # Validate that action is something we know what to do with 41 | valid_actions = ['create', 'check', 'update', 'delete', 'watch'] 42 | if args.action not in valid_actions: 43 | print ("Invalid action provided, must be one of: '%s'" 44 | % (", ".join(valid_actions))) 45 | exit(1) 46 | 47 | # Make sure we can read the yaml file provided 48 | try: 49 | open(args.yamlfile, 'r') 50 | except IOError as exception: 51 | print "Cannot read yaml file %s: %s" % (args.yamlfile, exception) 52 | exit(1) 53 | 54 | # Get and configure the log level 55 | numeric_level = getattr(logging, args.loglevel.upper(), None) 56 | boto_numeric_level = getattr(logging, args.botologlevel.upper(), None) 57 | if not isinstance(numeric_level, int): 58 | print 'Invalid log level: %s' % args.loglevel 59 | exit(1) 60 | logging.basicConfig(level=numeric_level) 61 | logger = logging.getLogger(__name__) 62 | 63 | # Get and configure the log level for boto 64 | if not isinstance(boto_numeric_level, int): 65 | logger.critical("Invalid boto log level: %s", args.botologlevel) 66 | exit(1) 67 | logging.getLogger('boto').setLevel(boto_numeric_level) 68 | 69 | # Create the mega_stack object and sort out dependencies 70 | the_mega_stack = MegaStack(args.yamlfile) 71 | the_mega_stack.sort_stacks_by_deps() 72 | 73 | # Print some info about what we found in the yaml and dependency order 74 | logger.info("Mega stack name: %s", the_mega_stack.name) 75 | logger.info("Found %s CF stacks in yaml.", len(the_mega_stack.cf_stacks)) 76 | logger.info("Processing stacks in the following order: %s", 77 | [x.name for x in the_mega_stack.stack_objs]) 78 | for stack in the_mega_stack.stack_objs: 79 | logger.debug("%s depends on %s", stack.name, stack.depends_on) 80 | 81 | # Run the method of the mega stack object for the action provided 82 | if args.action == 'create': 83 | the_mega_stack.create(args.stackname) 84 | 85 | if args.action == 'check': 86 | the_mega_stack.check(args.stackname) 87 | 88 | if args.action == 'delete': 89 | the_mega_stack.delete(args.stackname) 90 | 91 | if args.action == 'update': 92 | the_mega_stack.update(args.stackname) 93 | 94 | if args.action == 'watch': 95 | the_mega_stack.watch(args.stackname) 96 | 97 | 98 | if __name__ == '__main__': 99 | main() 100 | -------------------------------------------------------------------------------- /examples/instance_layer.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Cut down version of AWS CloudFormation Sample Template vpc_single_instance_in_subnet.template, which creates instances inside an existing VPC", 4 | "Parameters": { 5 | "SSHLocation": { 6 | "Description": " The IP address source range allowed SSH access to defined EC2 instances", 7 | "Type": "String", 8 | "MinLength": "9", 9 | "MaxLength": "18", 10 | "Default": "0.0.0.0/0", 11 | "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", 12 | "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." 13 | }, 14 | "InstanceType": { 15 | "Description": "WebServer EC2 instance type", 16 | "Type": "String", 17 | "Default": "m1.small", 18 | "AllowedValues": [ 19 | "t1.micro", 20 | "m1.small", 21 | "m1.medium", 22 | "m1.large", 23 | "m1.xlarge", 24 | "m2.xlarge", 25 | "m2.2xlarge", 26 | "m2.4xlarge", 27 | "m3.xlarge", 28 | "m3.2xlarge", 29 | "c1.medium", 30 | "c1.xlarge", 31 | "cc1.4xlarge", 32 | "cc2.8xlarge", 33 | "cg1.4xlarge" 34 | ], 35 | "ConstraintDescription": "must be a valid EC2 instance type." 36 | }, 37 | "KeyName": { 38 | "Description": "Name of an existing EC2 KeyPair to allow SSH access to defined instances", 39 | "Type": "String" 40 | }, 41 | "VPC": { 42 | "Description": "VpcId of the VPC to launch this instance in", 43 | "Type": "String" 44 | }, 45 | "Subnet": { 46 | "Description": "Subnet within the VPC to launch the instance", 47 | "Type": "String" 48 | } 49 | }, 50 | "Mappings": { 51 | "RegionMap": { 52 | "us-east-1": { 53 | "AMI": "ami-7f418316" 54 | }, 55 | "us-west-1": { 56 | "AMI": "ami-951945d0" 57 | }, 58 | "us-west-2": { 59 | "AMI": "ami-16fd7026" 60 | }, 61 | "eu-west-1": { 62 | "AMI": "ami-24506250" 63 | }, 64 | "sa-east-1": { 65 | "AMI": "ami-3e3be423" 66 | }, 67 | "ap-southeast-1": { 68 | "AMI": "ami-74dda626" 69 | }, 70 | "ap-southeast-2": { 71 | "AMI": "ami-b3990e89" 72 | }, 73 | "ap-northeast-1": { 74 | "AMI": "ami-dcfa4edd" 75 | } 76 | } 77 | }, 78 | "Resources": { 79 | "IPAddress": { 80 | "Type": "AWS::EC2::EIP", 81 | "Properties": { 82 | "Domain": "vpc", 83 | "InstanceId": { 84 | "Ref": "WebServerInstance" 85 | } 86 | } 87 | }, 88 | "InstanceSecurityGroup": { 89 | "Type": "AWS::EC2::SecurityGroup", 90 | "Properties": { 91 | "VpcId": { 92 | "Ref": "VPC" 93 | }, 94 | "GroupDescription": "Enable SSH access via port 22", 95 | "SecurityGroupIngress": [ 96 | { 97 | "IpProtocol": "tcp", 98 | "FromPort": "22", 99 | "ToPort": "22", 100 | "CidrIp": { 101 | "Ref": "SSHLocation" 102 | } 103 | }, 104 | { 105 | "IpProtocol": "tcp", 106 | "FromPort": "80", 107 | "ToPort": "80", 108 | "CidrIp": "0.0.0.0/0" 109 | } 110 | ] 111 | } 112 | }, 113 | "WebServerInstance": { 114 | "Type": "AWS::EC2::Instance", 115 | "Properties": { 116 | "ImageId": { 117 | "Fn::FindInMap": [ 118 | "RegionMap", 119 | { 120 | "Ref": "AWS::Region" 121 | }, 122 | "AMI" 123 | ] 124 | }, 125 | "SecurityGroupIds": [ 126 | { 127 | "Ref": "InstanceSecurityGroup" 128 | } 129 | ], 130 | "SubnetId": { 131 | "Ref": "Subnet" 132 | }, 133 | "InstanceType": { 134 | "Ref": "InstanceType" 135 | }, 136 | "KeyName": { 137 | "Ref": "KeyName" 138 | } 139 | } 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /examples/legostack/templates/vpc-generic-asg-dualaz.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Dual-AZ Auto-Scaling Generic Instances", 4 | "Parameters": { 5 | "ParamInstSubnet1": { 6 | "Type": "AWS::EC2::Subnet::Id", 7 | "Description": "Subnet 1 for auto-scaling-group" 8 | }, 9 | "ParamInstSubnet2": { 10 | "Type": "AWS::EC2::Subnet::Id", 11 | "Description": "Subnet 2 for auto-scaling-group" 12 | }, 13 | "ParamInstSubnet1AZ": { 14 | "Type": "String", 15 | "Description": "Subnet 1 Availability zone", 16 | "AllowedPattern": "[a-z][a-z]-[a-z]+-[0-9][a-z]", 17 | "ConstraintDescription": "Must select a valid AZ name within the target region" 18 | }, 19 | "ParamInstSubnet2AZ": { 20 | "Type": "String", 21 | "Description": "Subnet 2 Availability zone", 22 | "AllowedPattern": "[a-z][a-z]-[a-z]+-[0-9][a-z]", 23 | "ConstraintDescription": "Must select a valid AZ name within the target region" 24 | }, 25 | "ParamVPC": { 26 | "Type": "AWS::EC2::VPC::Id", 27 | "Description": "VPC ID" 28 | }, 29 | "ParamRSAKeyName" : { 30 | "Type": "AWS::EC2::KeyPair::KeyName", 31 | "Description": "RSA Key for instances" 32 | }, 33 | "ParamIamInstanceProfile" : { 34 | "Type": "String", 35 | "Description": "IAM Role Profile for instances" 36 | }, 37 | "ParamInstanceAMI" : { 38 | "Type": "String", 39 | "Description": "ID of instance AMI" 40 | }, 41 | "ParamInstanceDefaultSG": { 42 | "Type": "AWS::EC2::SecurityGroup::Id", 43 | "Description": "ID of Default Management SG" 44 | }, 45 | "ParamInstanceType" : { 46 | "Type": "String", 47 | "Default" : "c3.large", 48 | "Description": "Instance type for newly created instances" 49 | }, 50 | "ParamASGMinCapacity" : { 51 | "Type": "Number", 52 | "Default": "0", 53 | "Description": "Minimum capacity for the ASG" 54 | }, 55 | "ParamASGMaxCapacity" : { 56 | "Type": "Number", 57 | "Default": "10", 58 | "Description": "Maximum capacity for the ASG" 59 | }, 60 | "ParamASGDesiredCapacity" : { 61 | "Type": "Number", 62 | "Default": "2", 63 | "Description": "Desired capacity for the ASG" 64 | } 65 | }, 66 | "Resources": { 67 | 68 | "SGInstances": { 69 | "Type": "AWS::EC2::SecurityGroup", 70 | "Properties": { 71 | "GroupDescription": "Security Group for the ASG instances", 72 | "VpcId": { 73 | "Ref": "ParamVPC" 74 | }, 75 | "SecurityGroupIngress": [ 76 | 77 | ] 78 | } 79 | }, 80 | "InstLanuchConf": { 81 | "Type": "AWS::AutoScaling::LaunchConfiguration", 82 | "Properties": { 83 | "IamInstanceProfile": { "Ref": "ParamIamInstanceProfile" }, 84 | "ImageId": { "Ref" : "ParamInstanceAMI" }, 85 | "InstanceType": { "Ref" : "ParamInstanceType" }, 86 | "KeyName": { "Ref": "ParamRSAKeyName" }, 87 | "SecurityGroups": [ 88 | { 89 | "Ref": "ParamInstanceDefaultSG" 90 | }, 91 | { 92 | "Ref": "SGInstances" 93 | } 94 | ], 95 | "BlockDeviceMappings" : [ { 96 | "DeviceName" : "/dev/sda1", 97 | "Ebs" : {"VolumeSize" : "25"} 98 | } ], 99 | "UserData": { "Fn::Base64": { "Fn::Join": [ "", 100 | [ 101 | "#!/bin/bash\n" 102 | ] 103 | ] } } 104 | } 105 | }, 106 | "InstASG": { 107 | "Type": "AWS::AutoScaling::AutoScalingGroup", 108 | "Properties": { 109 | "LaunchConfigurationName": { 110 | "Ref": "InstLanuchConf" 111 | }, 112 | "AvailabilityZones": [ 113 | { "Ref" : "ParamInstSubnet1AZ" }, 114 | { "Ref" : "ParamInstSubnet2AZ" } 115 | ], 116 | "HealthCheckType": "EC2", 117 | "MaxSize": { "Ref" : "ParamASGMaxCapacity" }, 118 | "MinSize": { "Ref" : "ParamASGMinCapacity" }, 119 | "DesiredCapacity": { "Ref" : "ParamASGDesiredCapacity" }, 120 | "VPCZoneIdentifier": [ 121 | { 122 | "Ref": "ParamInstSubnet1" 123 | }, 124 | { 125 | "Ref": "ParamInstSubnet2" 126 | } 127 | ], 128 | "Tags": [ 129 | { 130 | "Key": "Name", 131 | "Value": { "Fn::Join": [ "@", [ "inst", { "Ref": "AWS::StackName" } ] ] }, 132 | "PropagateAtLaunch": "true" 133 | } 134 | ] 135 | } 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /examples/legostack/templates/vpc-base-dualaz.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Foundation VPC stack across two availability zones", 4 | "Parameters": { 5 | "ParamCIDRBlock": { 6 | "Description": "The CIDR block for the VPC ( format X.X.X.X/X )", 7 | "Type": "String", 8 | "AllowedPattern": "([0-9]{1,3}\\.){3}[0-9]{1,3}\/[0-9]{1,2}", 9 | "ConstraintDescription": "Parameter CIDRBlock must be a valid CIDR expression." 10 | }, 11 | "ParamAvailabilityZone1": { 12 | "Description": "Left-Side Availability Zone, must exist in your AWS region", 13 | "Type": "String", 14 | "AllowedPattern": "[a-z][a-z]-[a-z]+-[0-9][a-z]", 15 | "ConstraintDescription": "Must select a valid AZ name within the target region" 16 | }, 17 | "ParamAvailabilityZone2": { 18 | "Description": "Right-Side Availability Zone, must exist in your AWS region", 19 | "Type": "String", 20 | "AllowedPattern": "[a-z][a-z]-[a-z]+-[0-9][a-z]", 21 | "ConstraintDescription": "must select a valid AZ name within the target region" 22 | }, 23 | "ParamGlobalRSAKey": { 24 | "Description": "A global EC2 key pair", 25 | "Type": "AWS::EC2::KeyPair::KeyName" 26 | } 27 | }, 28 | "Resources": { 29 | "StandardInstanceRole": { 30 | "Type": "AWS::IAM::Role", 31 | "Properties": { 32 | "AssumeRolePolicyDocument": { 33 | "Statement": [ 34 | { 35 | "Effect": "Allow", 36 | "Principal": { 37 | "Service": [ 38 | "ec2.amazonaws.com" 39 | ] 40 | }, 41 | "Action": [ 42 | "sts:AssumeRole" 43 | ] 44 | } 45 | ] 46 | }, 47 | "Path": "/", 48 | "Policies": [ 49 | ] 50 | } 51 | }, 52 | "StandardInstanceRoleProfile": { 53 | "Type": "AWS::IAM::InstanceProfile", 54 | "Properties": { 55 | "Path": "/", 56 | "Roles": [ 57 | { "Ref": "StandardInstanceRole" } 58 | ] 59 | } 60 | }, 61 | "BaseAccessPolicy" : { 62 | "Type": "AWS::IAM::Policy", 63 | "Properties": { 64 | "PolicyName" : "Base_access_policy_for_all_instances", 65 | "Roles": [ { "Ref" : "StandardInstanceRole" } ], 66 | "PolicyDocument" : { 67 | "Statement": [ 68 | { 69 | "Effect": "Allow", 70 | "Action": [ 71 | "ec2:DescribeTags" 72 | ], 73 | "Resource": "*" 74 | } 75 | ] 76 | } 77 | } 78 | }, 79 | "VPC": { 80 | "Type": "AWS::EC2::VPC", 81 | "Properties": { 82 | "CidrBlock": { "Ref" : "ParamCIDRBlock" }, 83 | "Tags": [ 84 | { 85 | "Key": "left-az", 86 | "Value": { 87 | "Ref": "ParamAvailabilityZone1" 88 | } 89 | }, 90 | { 91 | "Key": "right-az", 92 | "Value": { 93 | "Ref": "ParamAvailabilityZone2" 94 | } 95 | } 96 | ] 97 | } 98 | }, 99 | "InternetGateway": { 100 | "Type": "AWS::EC2::InternetGateway" 101 | }, 102 | "AttachGateway": { 103 | "Type": "AWS::EC2::VPCGatewayAttachment", 104 | "DependsOn": "InternetGateway", 105 | "Properties": { 106 | "VpcId": { 107 | "Ref": "VPC" 108 | }, 109 | "InternetGatewayId": { 110 | "Ref": "InternetGateway" 111 | } 112 | } 113 | }, 114 | "PublicRouteTable": { 115 | "Type": "AWS::EC2::RouteTable", 116 | "DependsOn": "VPC", 117 | "Properties": { 118 | "VpcId": { 119 | "Ref": "VPC" 120 | } 121 | } 122 | }, 123 | "PublicRoute": { 124 | "Type": "AWS::EC2::Route", 125 | "DependsOn": "PublicRouteTable", 126 | "Properties": { 127 | "RouteTableId": { 128 | "Ref": "PublicRouteTable" 129 | }, 130 | "DestinationCidrBlock": "0.0.0.0/0", 131 | "GatewayId": { 132 | "Ref": "InternetGateway" 133 | } 134 | } 135 | }, 136 | "InternalRouteTable1": { 137 | "Type": "AWS::EC2::RouteTable", 138 | "DependsOn" : "VPC", 139 | "Properties": { 140 | "VpcId": { "Ref": "VPC" } 141 | } 142 | }, 143 | "InternalRouteTable2": { 144 | "Type": "AWS::EC2::RouteTable", 145 | "DependsOn" : "VPC", 146 | "Properties": { 147 | "VpcId": { "Ref": "VPC" } 148 | } 149 | }, 150 | "SGManagementSource": { 151 | "Type": "AWS::EC2::SecurityGroup", 152 | "DependsOn" : "VPC", 153 | "Properties": { 154 | "VpcId": { 155 | "Ref": "VPC" 156 | }, 157 | "GroupDescription": "Source Security Group for Management Activities" 158 | } 159 | }, 160 | "SGBaseManagement": { 161 | "Type": "AWS::EC2::SecurityGroup", 162 | "DependsOn" : "VPC", 163 | "Properties": { 164 | "GroupDescription": "Base Security Group for Management / Monitoring access", 165 | "VpcId": { 166 | "Ref": "VPC" 167 | }, 168 | "SecurityGroupIngress": [ 169 | { 170 | "IpProtocol": "-1", 171 | "SourceSecurityGroupId": { 172 | "Ref": "SGManagementSource" 173 | }, 174 | "FromPort": "0", 175 | "ToPort": "65535" 176 | } 177 | ] 178 | } 179 | } 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /examples/vpc_layer.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Cut down version of AWS CloudFormation Sample Template vpc_single_instance_in_subnet.template: Creates a VPC ready for instances", 4 | "Resources": { 5 | "VPC": { 6 | "Type": "AWS::EC2::VPC", 7 | "Properties": { 8 | "CidrBlock": "10.0.0.0/16" 9 | } 10 | }, 11 | "Subnet": { 12 | "Type": "AWS::EC2::Subnet", 13 | "Properties": { 14 | "VpcId": { 15 | "Ref": "VPC" 16 | }, 17 | "CidrBlock": "10.0.0.0/24" 18 | } 19 | }, 20 | "InternetGateway": { 21 | "Type": "AWS::EC2::InternetGateway", 22 | "Properties": {} 23 | }, 24 | "AttachGateway": { 25 | "Type": "AWS::EC2::VPCGatewayAttachment", 26 | "Properties": { 27 | "VpcId": { 28 | "Ref": "VPC" 29 | }, 30 | "InternetGatewayId": { 31 | "Ref": "InternetGateway" 32 | } 33 | } 34 | }, 35 | "RouteTable": { 36 | "Type": "AWS::EC2::RouteTable", 37 | "Properties": { 38 | "VpcId": { 39 | "Ref": "VPC" 40 | } 41 | }, 42 | "DependsOn": "AttachGateway" 43 | }, 44 | "Route": { 45 | "Type": "AWS::EC2::Route", 46 | "Properties": { 47 | "RouteTableId": { 48 | "Ref": "RouteTable" 49 | }, 50 | "DestinationCidrBlock": "0.0.0.0/0", 51 | "GatewayId": { 52 | "Ref": "InternetGateway" 53 | } 54 | } 55 | }, 56 | "SubnetRouteTableAssociation": { 57 | "Type": "AWS::EC2::SubnetRouteTableAssociation", 58 | "Properties": { 59 | "SubnetId": { 60 | "Ref": "Subnet" 61 | }, 62 | "RouteTableId": { 63 | "Ref": "RouteTable" 64 | } 65 | } 66 | }, 67 | "NetworkAcl": { 68 | "Type": "AWS::EC2::NetworkAcl", 69 | "Properties": { 70 | "VpcId": { 71 | "Ref": "VPC" 72 | }, 73 | "Tags": [ 74 | { 75 | "Key": "Application", 76 | "Value": { 77 | "Ref": "AWS::StackId" 78 | } 79 | } 80 | ] 81 | } 82 | }, 83 | "InboundHTTPNetworkAclEntry": { 84 | "Type": "AWS::EC2::NetworkAclEntry", 85 | "Properties": { 86 | "NetworkAclId": { 87 | "Ref": "NetworkAcl" 88 | }, 89 | "RuleNumber": "100", 90 | "Protocol": "6", 91 | "RuleAction": "allow", 92 | "Egress": "false", 93 | "CidrBlock": "0.0.0.0/0", 94 | "PortRange": { 95 | "From": "80", 96 | "To": "80" 97 | } 98 | } 99 | }, 100 | "InboundSSHNetworkAclEntry": { 101 | "Type": "AWS::EC2::NetworkAclEntry", 102 | "Properties": { 103 | "NetworkAclId": { 104 | "Ref": "NetworkAcl" 105 | }, 106 | "RuleNumber": "101", 107 | "Protocol": "6", 108 | "RuleAction": "allow", 109 | "Egress": "false", 110 | "CidrBlock": "0.0.0.0/0", 111 | "PortRange": { 112 | "From": "22", 113 | "To": "22" 114 | } 115 | } 116 | }, 117 | "InboundResponsePortsNetworkAclEntry": { 118 | "Type": "AWS::EC2::NetworkAclEntry", 119 | "Properties": { 120 | "NetworkAclId": { 121 | "Ref": "NetworkAcl" 122 | }, 123 | "RuleNumber": "102", 124 | "Protocol": "6", 125 | "RuleAction": "allow", 126 | "Egress": "false", 127 | "CidrBlock": "0.0.0.0/0", 128 | "PortRange": { 129 | "From": "1024", 130 | "To": "65535" 131 | } 132 | } 133 | }, 134 | "OutBoundHTTPNetworkAclEntry": { 135 | "Type": "AWS::EC2::NetworkAclEntry", 136 | "Properties": { 137 | "NetworkAclId": { 138 | "Ref": "NetworkAcl" 139 | }, 140 | "RuleNumber": "100", 141 | "Protocol": "6", 142 | "RuleAction": "allow", 143 | "Egress": "true", 144 | "CidrBlock": "0.0.0.0/0", 145 | "PortRange": { 146 | "From": "80", 147 | "To": "80" 148 | } 149 | } 150 | }, 151 | "OutBoundHTTPSNetworkAclEntry": { 152 | "Type": "AWS::EC2::NetworkAclEntry", 153 | "Properties": { 154 | "NetworkAclId": { 155 | "Ref": "NetworkAcl" 156 | }, 157 | "RuleNumber": "101", 158 | "Protocol": "6", 159 | "RuleAction": "allow", 160 | "Egress": "true", 161 | "CidrBlock": "0.0.0.0/0", 162 | "PortRange": { 163 | "From": "443", 164 | "To": "443" 165 | } 166 | } 167 | }, 168 | "OutBoundResponsePortsNetworkAclEntry": { 169 | "Type": "AWS::EC2::NetworkAclEntry", 170 | "Properties": { 171 | "NetworkAclId": { 172 | "Ref": "NetworkAcl" 173 | }, 174 | "RuleNumber": "102", 175 | "Protocol": "6", 176 | "RuleAction": "allow", 177 | "Egress": "true", 178 | "CidrBlock": "0.0.0.0/0", 179 | "PortRange": { 180 | "From": "1024", 181 | "To": "65535" 182 | } 183 | } 184 | }, 185 | "SubnetNetworkAclAssociation": { 186 | "Type": "AWS::EC2::SubnetNetworkAclAssociation", 187 | "Properties": { 188 | "SubnetId": { 189 | "Ref": "Subnet" 190 | }, 191 | "NetworkAclId": { 192 | "Ref": "NetworkAcl" 193 | } 194 | } 195 | } 196 | } 197 | } -------------------------------------------------------------------------------- /examples/legostack/templates/vpc-nat-dualaz.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "Dual-AZ Template providing NAT instances and routes", 4 | "Parameters": { 5 | "ParamVPC": { 6 | "Type": "AWS::EC2::VPC::Id", 7 | "Description": "VPC ID" 8 | }, 9 | "ParamVPCCDRBlock": { 10 | "Type": "String", 11 | "Description": "VPC CIDR Block", 12 | "AllowedPattern": "([0-9]{1,3}\\.){3}[0-9]{1,3}\/[0-9]{1,2}", 13 | "ConstraintDescription": "Parameter CIDRBlock must be a valid CIDR expression." 14 | }, 15 | "ParamInstSubnet1": { 16 | "Type": "AWS::EC2::Subnet::Id", 17 | "Description": "Subnet for NAT Instance 1" 18 | }, 19 | "ParamRouteTable1": { 20 | "Type": "String", 21 | "Description": "A route table to associate with the first NAT instance" 22 | }, 23 | "ParamInstSubnet2": { 24 | "Type": "AWS::EC2::Subnet::Id", 25 | "Description": "Subnet for NAT instance 2" 26 | }, 27 | "ParamRouteTable2": { 28 | "Type": "String", 29 | "Description": "A route table to associate with the second NAT instance" 30 | }, 31 | "ParamRSAKeyName" : { 32 | "Type": "String", 33 | "Description": "RSA Key for instances" 34 | }, 35 | "ParamIamInstanceProfile" : { 36 | "Type": "String", 37 | "Description": "IAM Instance Profile for instances" 38 | }, 39 | "ParamInstanceDefaultSG": { 40 | "Type": "AWS::EC2::SecurityGroup::Id", 41 | "Description": "ID of Default Management SG" 42 | } 43 | }, 44 | "Mappings": { 45 | "AWSNATAMI": { 46 | "us-east-1": { 47 | "AMI": "ami-c6699baf" 48 | }, 49 | "us-west-2": { 50 | "AMI": "ami-52ff7262" 51 | }, 52 | "us-west-1": { 53 | "AMI": "ami-3bcc9e7e" 54 | }, 55 | "eu-west-1": { 56 | "AMI": "ami-0b5b6c7f" 57 | }, 58 | "ap-southeast-1": { 59 | "AMI": "ami-02eb9350" 60 | }, 61 | "ap-northeast-1": { 62 | "AMI": "ami-14d86d15" 63 | }, 64 | "sa-east-1": { 65 | "AMI": "ami-0439e619" 66 | }, 67 | "ap-southeast-2": { 68 | "AMI": "ami-a1980f9b" 69 | } 70 | } 71 | }, 72 | "Resources": { 73 | "NATSecurityGroup": { 74 | "Type": "AWS::EC2::SecurityGroup", 75 | "Properties": { 76 | "GroupDescription": "Enable internal access to the NAT device", 77 | "VpcId": { "Ref": "ParamVPC" }, 78 | "SecurityGroupIngress": [ 79 | { 80 | "IpProtocol": "-1", 81 | "CidrIp": { "Ref" : "ParamVPCCDRBlock" } 82 | } 83 | ] 84 | } 85 | }, 86 | "NATDevice1": { 87 | "Type": "AWS::EC2::Instance", 88 | "Properties": { 89 | "InstanceType": "m1.small", 90 | "Monitoring": "true", 91 | "KeyName": { "Ref": "ParamRSAKeyName" }, 92 | "IamInstanceProfile" : { "Ref" : "ParamIamInstanceProfile" }, 93 | "SubnetId": { "Ref" : "ParamInstSubnet1" }, 94 | "SecurityGroupIds" : [ 95 | { "Ref" : "NATSecurityGroup" }, 96 | { "Ref" : "ParamInstanceDefaultSG" } 97 | ], 98 | "SourceDestCheck": "false", 99 | "ImageId": { "Fn::FindInMap": ["AWSNATAMI", {"Ref": "AWS::Region"}, "AMI" ] }, 100 | "Tags": [ 101 | { 102 | "Key": "Name", 103 | "Value": { 104 | "Fn::Join": [ 105 | "@", 106 | [ 107 | "NAT-L", 108 | { 109 | "Ref": "AWS::StackName" 110 | } 111 | ] 112 | ] 113 | } 114 | } 115 | ] 116 | } 117 | }, 118 | "NATIPAddress1": { 119 | "Type": "AWS::EC2::EIP", 120 | "DependsOn": "NATDevice1", 121 | "Properties": { 122 | "Domain": "vpc", 123 | "InstanceId": { "Ref" : "NATDevice1" } 124 | } 125 | }, 126 | "Route1": { 127 | "Type": "AWS::EC2::Route", 128 | "Properties": { 129 | "RouteTableId": { "Ref": "ParamRouteTable1" }, 130 | "DestinationCidrBlock": "0.0.0.0/0", 131 | "InstanceId" : { "Ref" : "NATDevice1" } 132 | } 133 | }, 134 | "NATDevice2": { 135 | "Type": "AWS::EC2::Instance", 136 | "Properties": { 137 | "InstanceType": "m1.small", 138 | "Monitoring": "true", 139 | "KeyName": { "Ref": "ParamRSAKeyName" }, 140 | "IamInstanceProfile" : { "Ref" : "ParamIamInstanceProfile" }, 141 | "SubnetId": { "Ref" : "ParamInstSubnet2" }, 142 | "SecurityGroupIds" : [ 143 | { "Ref" : "NATSecurityGroup" }, 144 | { "Ref" : "ParamInstanceDefaultSG" } 145 | ], 146 | "SourceDestCheck": "false", 147 | "ImageId": { "Fn::FindInMap": ["AWSNATAMI", {"Ref": "AWS::Region"}, "AMI" ] }, 148 | "Tags": [ 149 | { 150 | "Key": "Name", 151 | "Value": { 152 | "Fn::Join": [ 153 | "@", 154 | [ 155 | "NAT-R", 156 | { 157 | "Ref": "AWS::StackName" 158 | } 159 | ] 160 | ] 161 | } 162 | } 163 | ] 164 | } 165 | }, 166 | "NATIPAddress2": { 167 | "Type": "AWS::EC2::EIP", 168 | "DependsOn": "NATDevice2", 169 | "Properties": { 170 | "Domain": "vpc", 171 | "InstanceId": { "Ref" : "NATDevice2" } 172 | } 173 | }, 174 | "Route2": { 175 | "Type": "AWS::EC2::Route", 176 | "Properties": { 177 | "RouteTableId": { "Ref": "ParamRouteTable2" }, 178 | "DestinationCidrBlock": "0.0.0.0/0", 179 | "InstanceId" : { "Ref" : "NATDevice2" } 180 | } 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Cumulus 2 | ======= 3 | 4 | Helps manage AWS CloudFormation stacks 5 | 6 | Build status 7 | ------------ 8 | 9 | |Circle CI| 10 | 11 | News 12 | ---- 13 | 14 | 2014-07-01 15 | ~~~~~~~~~~ 16 | 17 | - Added colour cloudformation event status output via 18 | 'highlight-output' setting in YAML. 19 | 20 | :: 21 | 22 | highlight-output: true 23 | 24 | 2014-04-17 25 | ~~~~~~~~~~ 26 | 27 | - You can now insert PyStache {{}} style variables to import 28 | environment variables. 29 | 30 | :: 31 | 32 | stack: 33 | ami_id: {{AMIID}} 34 | 35 | AMID=ami-1q23123123 cumulus -y example_stack.yaml -a create 36 | 37 | would be seen by cloudformation with the ami id of ami-1q23123123 38 | 39 | 2013-09-06 40 | ~~~~~~~~~~ 41 | 42 | - You can now define stack level tags using the *tags* directive in the 43 | YaML file, like: 44 | 45 | :: 46 | 47 | tags: 48 | tag1: value 49 | tag2: value 50 | 51 | tags can be specified both 52 | at root level and sub-stack level. tags at root level are applied to all 53 | sub-stacks and duplicate sub-stack tags will override root level tags 54 | 55 | - You can use the directive ``disable: true`` in any sub-stack to prevent it from being created/updated/deleted 56 | 57 | The problem 58 | ----------- 59 | 60 | Amazon CloudFormation (CF) allows you to instantiate multiple AWS 61 | resources in a repeatable, ordered and structured method. As our 62 | infrastructure grew, so did our CF templates and soon they were 63 | monolothic and complex to maintain. We looked at spliting these 64 | templates into smaller chunks, which worked as a short term solution but 65 | created a new problem. With multiple templates dependant on other 66 | declared resources, we were forced to manually pass parameters for 67 | inter-stack operability. This greatly affected the repeatability of our 68 | stacks as we did not have an easy method to keep track of what 69 | parameters were used, especially those relating to physical resource 70 | IDs. 71 | 72 | The solution 73 | ------------ 74 | 75 | Cumulus attempts to solve the problem by introducing a layer above CF 76 | templates, a stack configuration YAML file. This allows multiple CF 77 | stacks to be created in order and maintained respecting their 78 | dependencies. The YAML file stores values for parameters to be passed 79 | into each of the stacks. Parameters can be assigned with static values 80 | or will source the value of a parameter, output or resource of another 81 | stack described in the YAML file. Cumulus actively translates reference 82 | values to physical resource values on creation of the stack. 83 | 84 | Current state / known issues 85 | ---------------------------- 86 | 87 | For our use, Cumulus can create, update and delete stacks reliably but 88 | is still very much in an Alpha state. We're looking forward to see how 89 | you use Cumulus, and please submit pull requests for any issues you may 90 | encounter or for feature requests :) 91 | 92 | This is my first real python project, so I'm sure the code can be, just 93 | generally better... 94 | 95 | Known issues: 96 | 97 | - Templates are passed in as a JSON string to CF, this will break large 98 | templates 99 | 100 | Roadmap: 101 | 102 | - Implement a way of displaying meaningful diffs during update runs 103 | - Add support for using S3/Externally hosted templates 104 | - Support larger templates 105 | 106 | How to get started 107 | ------------------ 108 | 109 | Clone the repo somewhere: 110 | 111 | :: 112 | 113 | $ git clone git://github.com/peterkh/cumulus.git 114 | 115 | Install Cumulus with setuptools: 116 | 117 | :: 118 | 119 | $ sudo python setup.py install 120 | 121 | Make sure you have AWS credentials set up for boto (the library used by 122 | Cumulus to interact with AWS). Set the following environment variables: 123 | 124 | **AWS\_ACCESS\_KEY\_ID** - Your AWS Access Key ID 125 | 126 | **AWS\_SECRET\_ACCESS\_KEY** - Your AWS Secret Access Key 127 | 128 | or create a boto config file as described 129 | `here `__, covering some 130 | other helpful boto-related settings. 131 | 132 | Creating the example stack 133 | -------------------------- 134 | 135 | **Common sense warning:** Running this example will create real 136 | resources in AWS and will cost you AWS credits / money / magic beans. 137 | 138 | I have included an example stack in the examples/ dir. It consists of 139 | three files: 140 | 141 | - cumulus\_example\_stack.yaml: The Cumulus yaml file for the stack. 142 | Creates a stack out of the following two templates in ap-southeast-2 143 | (Sydney region) 144 | - vpc\_layer.json: CF template to creates a VPC, base subnet and ACL 145 | - instance\_layer.json: CF template to create an instance inside a 146 | given VPC 147 | 148 | The template files are complete and work independently of Cumulus. 149 | Cumulus's purpose in life is just to make managing them easier. 150 | 151 | To create the example stack, change into the examples/ dir and run: 152 | 153 | :: 154 | 155 | $ cumulus -y cumulus_example_stack.yaml -a create 156 | 157 | Cumulus will print out CF messages as it builds. 158 | 159 | You can then try modifying the template and/or the values of the 160 | parameters and then update the stack: 161 | 162 | :: 163 | 164 | $ cumulus -y cumulus_example_stack.yaml -a update 165 | 166 | Once you have finished experimenting, you can delete as follows: 167 | 168 | :: 169 | 170 | $ cumulus -y cumulus_example_stack.yaml -a delete 171 | 172 | General usage 173 | ------------- 174 | 175 | :: 176 | 177 | cumulus -h 178 | usage: cumulus [-h] -y YAMLFILE -a ACTION [-l LOGLEVEL] [-L BOTOLOGLEVEL] 179 | [-s STACKNAME] 180 | 181 | optional arguments: 182 | -h, --help show this help message and exit 183 | -y YAMLFILE, --yamlfile YAMLFILE 184 | The yaml file to read the VPC mega stack configuration 185 | from 186 | -a ACTION, --action ACTION 187 | The action to perform: create, check, update, delete 188 | or watch 189 | -l LOGLEVEL, --log LOGLEVEL 190 | Log Level for output messages, CRITICAL, ERROR, 191 | WARNING, INFO or DEBUG 192 | -L BOTOLOGLEVEL, --botolog BOTOLOGLEVEL 193 | Log Level for boto, CRITICAL, ERROR, WARNING, INFO or 194 | DEBUG 195 | -s STACKNAME, --stack STACKNAME 196 | The stack name, used with the watch action, ignored 197 | for other actions 198 | 199 | YAML file format 200 | ---------------- 201 | 202 | Have a look at examples/cumulus\_example\_stack.yaml for a commented 203 | version of the yaml file. 204 | 205 | All sections are required at the moment, even if they are blank (i.e. 206 | depends, params). depends also needs to be empty or an array, even if 207 | the stack has only one dependency. 208 | 209 | .. |Circle CI| image:: https://circleci.com/gh/cotdsa/cumulus/tree/master.svg?style=svg 210 | :target: https://circleci.com/gh/cotdsa/cumulus/tree/master 211 | -------------------------------------------------------------------------------- /examples/legostack/legostack.yaml: -------------------------------------------------------------------------------- 1 | # In this example we leverage cumulus to build a two-tier DMZ/BACKEND infrastructure using only generic templates 2 | # 3 | # When all the stacks are created we'll be able to SSH to the jumpbox instance and from it SSH to a set of backend instances 4 | # 5 | 6 | legostack: 7 | # Working on N.California. You can change it just remember to also update availablilty zones below and AMI ids 8 | region: us-west-1 9 | highlight-output: true 10 | # Tags defined here are spread across every resource created by sub-stacks 11 | tags: 12 | globaltag: tagvalue 13 | stacks: 14 | # 15 | # This is the foundation stack built on a bare VPC template that provides basic internet gateway and routing tables 16 | # A "Management" security group is also created for convenience, altough not really used in this example 17 | # 18 | legostack: 19 | cf_template: templates/vpc-base-dualaz.json 20 | depends: 21 | params: 22 | # The VPC address space 23 | ParamCIDRBlock: 24 | value: 10.123.0.0/16 25 | ParamAvailabilityZone1: 26 | value: us-west-1b 27 | ParamAvailabilityZone2: 28 | value: us-west-1c 29 | # This is a global keypair, it's saved in the foundation stack for convenience later on. 30 | # You can always have different keys for different sub-stacks 31 | # This keypair HAS to exist in the region you're creating the stack in 32 | ParamGlobalRSAKey: 33 | value: nebula 34 | # 35 | # Create a DMZ subnet using the generic subnet template 36 | # The difference with an internal subnet is the routing table this subnet is associated with. 37 | # Using PublicRouteTable from the foundation stack ensures this subnet will have the IGW as 38 | # default gateway allowing instances in this subnet to use Elastic IPs 39 | # 40 | dmz-sub: 41 | cf_template: templates/vpc-subnet-dualaz.json 42 | depends: 43 | - legostack 44 | params: 45 | ParamCIDR1: 46 | value: 10.123.1.0/25 47 | ParamCIDR2: 48 | value: 10.123.1.128/25 49 | ParamVPC: 50 | type: resource 51 | source: legostack 52 | variable: VPC 53 | ParamAZ1: 54 | type: parameter 55 | source: legostack 56 | variable: ParamAvailabilityZone1 57 | ParamAZ2: 58 | type: parameter 59 | source: legostack 60 | variable: ParamAvailabilityZone2 61 | ParamRouteTable1: 62 | type: resource 63 | source: legostack 64 | variable: PublicRouteTable 65 | ParamRouteTable2: 66 | type: resource 67 | source: legostack 68 | variable: PublicRouteTable 69 | # 70 | # This stack creates a pair of NAT instances inside the DMZ subnet. 71 | # It also updates the internal routing tables created by the foundation stack setting the NAT instances as default gataway 72 | # After this stack is created, every subnet using the InternalRouteTables will have internet access via the NAT instances 73 | # 74 | nat: 75 | cf_template: templates/vpc-nat-dualaz.json 76 | depends: 77 | - legostack 78 | - dmz-sub 79 | params: 80 | ParamVPC: 81 | type: resource 82 | source: legostack 83 | variable: VPC 84 | ParamVPCCDRBlock: 85 | type: parameter 86 | source: legostack 87 | variable: ParamCIDRBlock 88 | ParamInstSubnet1: 89 | type: resource 90 | source: dmz-sub 91 | variable: Subnet1 92 | ParamInstSubnet2: 93 | type: resource 94 | source: dmz-sub 95 | variable: Subnet2 96 | ParamRouteTable1: 97 | type: resource 98 | source: legostack 99 | variable: InternalRouteTable1 100 | ParamRouteTable2: 101 | type: resource 102 | source: legostack 103 | variable: InternalRouteTable2 104 | ParamRSAKeyName: 105 | type: parameter 106 | source: legostack 107 | variable: ParamGlobalRSAKey 108 | ParamIamInstanceProfile: 109 | type: resource 110 | source: legostack 111 | variable: StandardInstanceRoleProfile 112 | ParamInstanceDefaultSG: 113 | type: resource 114 | source: legostack 115 | variable: SGBaseManagement 116 | 117 | # 118 | # Creates a single generic instance with an Elastic IP inside the DMZ subnet 119 | # The template used doesn't specifically allow any traffic to the instance, we'll use a special "bridge" stack later on 120 | # 121 | jumpbox: 122 | cf_template: templates/vpc-generic-instance-singleaz.json 123 | depends: 124 | - legostack 125 | - dmz-sub 126 | params: 127 | ParamInstanceType: 128 | value: 't2.small' 129 | ParamUseElasticIP: 130 | value: 'true' 131 | ParamInstanceAMI: 132 | # Ubuntu 12.04 Precise HVM EBS 133 | value: 'ami-0dad4949' 134 | 135 | ParamInstSubnet: 136 | type: resource 137 | source: dmz-sub 138 | variable: Subnet1 139 | ParamVPC: 140 | type: resource 141 | source: legostack 142 | variable: VPC 143 | ParamRSAKeyName: 144 | type: parameter 145 | source: legostack 146 | variable: ParamGlobalRSAKey 147 | ParamIamInstanceProfile: 148 | type: resource 149 | source: legostack 150 | variable: StandardInstanceRoleProfile 151 | ParamInstanceDefaultSG: 152 | type: resource 153 | source: legostack 154 | variable: SGBaseManagement 155 | 156 | # 157 | # Creates an internal subnet for backend nodes. Note the route tables used are the internal ones from the foundation stack 158 | # meaning instances placed in here are not directly reachable from the internet and all outbound traffic goes thrugh the NAT 159 | # 160 | backend-sub: 161 | cf_template: templates/vpc-subnet-dualaz.json 162 | depends: 163 | - legostack 164 | params: 165 | ParamCIDR1: 166 | value: 10.123.10.0/25 167 | ParamCIDR2: 168 | value: 10.123.10.128/25 169 | ParamVPC: 170 | type: resource 171 | source: legostack 172 | variable: VPC 173 | ParamAZ1: 174 | type: parameter 175 | source: legostack 176 | variable: ParamAvailabilityZone1 177 | ParamAZ2: 178 | type: parameter 179 | source: legostack 180 | variable: ParamAvailabilityZone2 181 | ParamRouteTable1: 182 | type: resource 183 | source: legostack 184 | variable: InternalRouteTable1 185 | ParamRouteTable2: 186 | type: resource 187 | source: legostack 188 | variable: InternalRouteTable2 189 | 190 | # 191 | # Creates an Auto-Scaling-Group inside the backend subnet. Again no traffic is allowed by default into the instances. We'll use a "bridge" 192 | # 193 | backend-cluster: 194 | cf_template: templates/vpc-generic-asg-dualaz.json 195 | depends: 196 | - legostack 197 | - backend-sub 198 | - nat 199 | params: 200 | ParamInstanceType: 201 | value: 't2.small' 202 | ParamInstanceAMI: 203 | # Ubuntu 12.04 Precise HVM EBS 204 | value: 'ami-0dad4949' 205 | ParamASGDesiredCapacity: 206 | value: 2 207 | 208 | ParamInstSubnet1: 209 | type: resource 210 | source: backend-sub 211 | variable: Subnet1 212 | ParamInstSubnet2: 213 | type: resource 214 | source: backend-sub 215 | variable: Subnet2 216 | ParamInstSubnet1AZ: 217 | type: parameter 218 | source: legostack 219 | variable: ParamAvailabilityZone1 220 | ParamInstSubnet2AZ: 221 | type: parameter 222 | source: legostack 223 | variable: ParamAvailabilityZone2 224 | ParamVPC: 225 | type: resource 226 | source: legostack 227 | variable: VPC 228 | ParamRSAKeyName: 229 | type: parameter 230 | source: legostack 231 | variable: ParamGlobalRSAKey 232 | ParamIamInstanceProfile: 233 | type: resource 234 | source: legostack 235 | variable: StandardInstanceRoleProfile 236 | ParamInstanceDefaultSG: 237 | type: resource 238 | source: legostack 239 | variable: SGBaseManagement 240 | 241 | # 242 | # This stack injects a rule into the backend nodes' SG allowing SSH access from the jumpbox node's SG 243 | # 244 | sgbr-jumpbox-becluster: 245 | cf_template: templates/vpc-sg-access-bridge.json 246 | depends: 247 | - jumpbox 248 | - backend-cluster 249 | params: 250 | ParamFromPort: 251 | value: 22 252 | ParamToPort: 253 | value: 22 254 | ParamIpProtocol: 255 | value: tcp 256 | 257 | ParamSourceSG: 258 | type: resource 259 | source: jumpbox 260 | variable: SGInstance 261 | ParamDestinationSG: 262 | type: resource 263 | source: backend-cluster 264 | variable: SGInstances 265 | 266 | # 267 | # This stack injects a rule into the jumpbox's SG allowing SSH access from the internet 268 | # 269 | sgbr-public-jumpbox: 270 | cf_template: templates/vpc-sg-access-bridge.json 271 | depends: 272 | - jumpbox 273 | params: 274 | ParamFromPort: 275 | value: 22 276 | ParamToPort: 277 | value: 22 278 | ParamIpProtocol: 279 | value: tcp 280 | ParamSourceCIDR: 281 | value: 0.0.0.0/0 282 | 283 | 284 | ParamDestinationSG: 285 | type: resource 286 | source: jumpbox 287 | variable: SGInstance 288 | 289 | # 290 | # Add a scheduled capacity change to the backend autoscaling-group. 291 | # 292 | scaleup-midday: 293 | cf_template: templates/asg-scheduled-capacity-change.json 294 | depends: 295 | - backend-cluster 296 | params: 297 | # Scale up to 4 instances every day at 11:30 AM Eastern Australia Time 298 | ParamFirstRecurrence: 299 | value: '30 0 * * *' 300 | ParamFirstCapacity: 301 | value: 4 302 | # Scale down to 2 instances every day at 12:30 PM Eastern Australia Time 303 | ParamSecondRecurrence: 304 | value: '30 1 * * *' 305 | ParamSecondCapacity: 306 | value: 2 307 | 308 | ParamAutoScalingGroup: 309 | type: resource 310 | source: backend-cluster 311 | variable: InstASG -------------------------------------------------------------------------------- /cumulus/CFStack.py: -------------------------------------------------------------------------------- 1 | """ 2 | CFStack module. Manages a single CloudFormation stack. 3 | """ 4 | import logging 5 | import simplejson 6 | import yaml 7 | import os 8 | 9 | 10 | class CFStack(object): 11 | """ 12 | CFstack object represents a CloudFormation stack including its parameters, 13 | template and what other stacks it depends on. 14 | """ 15 | def __init__(self, mega_stack_name, name, params, template_name, cfconn, 16 | sns_topic_arn, tags=None, depends_on=None): 17 | self.logger = logging.getLogger(__name__) 18 | if mega_stack_name == name: 19 | self.cf_stack_name = name 20 | else: 21 | self.cf_stack_name = "%s-%s" % (mega_stack_name, name) 22 | self.mega_stack_name = mega_stack_name 23 | self.name = name 24 | self.yaml_params = params 25 | self.params = {} 26 | self.template_name = template_name 27 | self.template_body = '' 28 | if depends_on is None: 29 | self.depends_on = None 30 | else: 31 | self.depends_on = [] 32 | for dep in depends_on: 33 | if dep == mega_stack_name: 34 | self.depends_on.append(dep) 35 | else: 36 | self.depends_on.append("%s-%s" % (mega_stack_name, dep)) 37 | self.sns_topic_arn = sns_topic_arn 38 | self.cfconn = cfconn 39 | 40 | # Safer than setting default value for tags = {} 41 | if tags is None: 42 | self.tags = {} 43 | else: 44 | self.tags = tags 45 | 46 | try: 47 | open(template_name, 'r') 48 | except: 49 | self.logger.critical("Failed to open template file %s for stack %s" 50 | % (self.template_name, self.name)) 51 | exit(1) 52 | 53 | # check params is a dict if set 54 | if self.yaml_params and type(self.yaml_params) is not dict: 55 | self.logger.critical( 56 | "Parameters for stack %s must be of type dict not %s", 57 | self.name, type(self.yaml_params)) 58 | exit(1) 59 | 60 | self.cf_stacks = {} 61 | self.cf_stacks_resources = {} 62 | 63 | def deps_met(self, current_cf_stacks): 64 | """ 65 | Check whether stacks we depend on exist in CloudFormation 66 | """ 67 | if self.depends_on is None: 68 | return True 69 | else: 70 | for dep in self.depends_on: 71 | dep_met = False 72 | # check CF if stack we depend on has been created successfully 73 | for stack in current_cf_stacks: 74 | if str(stack.stack_name) == dep: 75 | dep_met = True 76 | if not dep_met: 77 | return False 78 | return True 79 | 80 | def exists_in_cf(self, current_cf_stacks): 81 | """ 82 | Check if this stack exists in CloudFormation 83 | """ 84 | for stack in current_cf_stacks: 85 | if str(stack.stack_name) == self.cf_stack_name: 86 | return stack 87 | return False 88 | 89 | def populate_params(self, current_cf_stacks): 90 | """ 91 | Populate the parameter list for this stack 92 | """ 93 | # If we have no parameters in the yaml file, 94 | # set params to an empty dict and return true 95 | if self.yaml_params is None: 96 | self.params = {} 97 | return True 98 | if self.deps_met(current_cf_stacks): 99 | for param_name, param_val in self.yaml_params.iteritems(): 100 | if type(param_val) is dict: 101 | self.params[param_name] = self._parse_param( 102 | param_name, param_val) 103 | # If param_val is a list it means there is an array of vars 104 | # we need to turn into a comma sep list. 105 | elif type(param_val) is list: 106 | param_list = [] 107 | for item in param_val: 108 | if type(item) is dict: 109 | param_list.append(self._parse_param( 110 | param_name, str(item['value']))) 111 | self.params[param_name] = ','.join(param_list) 112 | return True 113 | else: 114 | return False 115 | 116 | def _parse_param(self, param_name, param_dict): 117 | """ 118 | Parse a param dict and return var value or false if not valid 119 | """ 120 | # Static value set, so use it 121 | if 'value' in param_dict: 122 | return str(param_dict['value']) 123 | elif 'value_env' in param_dict: 124 | var_name = str(param_dict['value_env']).strip().upper() 125 | if var_name in os.environ: 126 | return os.environ[var_name] 127 | else: 128 | raise KeyError("Cannot resolve environment variable " + 129 | var_name) 130 | # No static value set, but if we have a source, 131 | # type and variable can try getting from CF 132 | elif ('source' in param_dict and 133 | 'type' in param_dict and 134 | 'variable' in param_dict): 135 | if param_dict['source'] == self.mega_stack_name: 136 | source_stack = param_dict['source'] 137 | else: 138 | source_stack = ("%s-%s" % 139 | (self.mega_stack_name, param_dict['source'])) 140 | return self.get_value_from_cf( 141 | source_stack=source_stack, 142 | var_type=param_dict['type'], 143 | var_name=param_dict['variable']) 144 | else: 145 | error_message = ("Error in yaml file, can't parse parameter %s" + 146 | " for %s stack.") 147 | self.logger.critical(error_message, param_name, self.name) 148 | exit(1) 149 | 150 | def get_cf_stack(self, stack, resources=False): 151 | """ 152 | Get information on parameters, outputs and resources from a stack 153 | and cache it 154 | """ 155 | if not resources: 156 | if stack not in self.cf_stacks: 157 | # We don't have this stack in the cache already 158 | # so we need to pull it from CF 159 | self.cf_stacks[stack] = self.cfconn.describe_stacks(stack)[0] 160 | return self.cf_stacks[stack] 161 | else: 162 | if stack not in self.cf_stacks_resources: 163 | the_stack = self.get_cf_stack(stack=stack, resources=False) 164 | self.cf_stacks_resources[stack] = the_stack.list_resources() 165 | return self.cf_stacks_resources[stack] 166 | 167 | def get_value_from_cf(self, source_stack, var_type, var_name): 168 | """ 169 | Get a variable from a existing cloudformation stack, var_type should be 170 | parameter, resource or output. 171 | If using resource, provide the logical ID and this will return the 172 | Physical ID 173 | """ 174 | the_stack = self.get_cf_stack(stack=source_stack) 175 | if var_type == 'parameter': 176 | for param in the_stack.parameters: 177 | if str(param.key) == var_name: 178 | return str(param.value) 179 | elif var_type == 'output': 180 | for output in the_stack.outputs: 181 | if str(output.key) == var_name: 182 | return str(output.value) 183 | elif var_type == 'resource': 184 | for res in self.get_cf_stack(stack=source_stack, resources=True): 185 | if str(res.logical_resource_id) == var_name: 186 | return str(res.physical_resource_id) 187 | else: 188 | error_message = ("Error: invalid var_type passed to" + 189 | " get_value_from_cf, needs to be parameter, " + 190 | "resource or output. Not: %s") 191 | self.logger.critical(error_message, (var_type)) 192 | exit(1) 193 | 194 | def get_params_tuples(self): 195 | """ 196 | Convert param dict to array of tuples needed by boto 197 | """ 198 | tuple_list = [] 199 | if len(self.params) > 0: 200 | for param in self.params.keys(): 201 | tuple_list.append((param, self.params[param])) 202 | return tuple_list 203 | 204 | def read_template(self): 205 | """ 206 | Open and parse the yaml/json template for this stack 207 | """ 208 | try: 209 | template_file = open(self.template_name, 'r') 210 | template = yaml.load(template_file) 211 | except Exception as exception: 212 | self.logger.critical("Cannot parse %s template for stack %s." 213 | " Error: %s", self.template_name, self.name, 214 | exception) 215 | exit(1) 216 | self.template_body = simplejson.dumps( 217 | template, 218 | sort_keys=True, 219 | indent=2, 220 | separators=(',', ': '), 221 | ) 222 | return True 223 | 224 | def template_uptodate(self, current_cf_stacks): 225 | """ 226 | Check if stack is up to date with cloudformation. 227 | Returns true if template matches what's in cloudformation, 228 | false if not or stack not found. 229 | """ 230 | cf_stack = self.exists_in_cf(current_cf_stacks) 231 | if cf_stack: 232 | cf_temp_res = cf_stack.get_template()['GetTemplateResponse'] 233 | cf_temp_body = cf_temp_res['GetTemplateResult']['TemplateBody'] 234 | cf_temp_dict = yaml.load(cf_temp_body) 235 | if cf_temp_dict == yaml.load(self.template_body): 236 | return True 237 | return False 238 | 239 | def params_uptodate(self, current_cf_stacks): 240 | """ 241 | Check if parameters in stack are up to date with Cloudformation 242 | """ 243 | cf_stack = self.exists_in_cf(current_cf_stacks) 244 | if not cf_stack: 245 | return False 246 | 247 | # If number of params in CF and this stack obj dont match, 248 | # then it needs updating 249 | if len(cf_stack.parameters) != len(self.params): 250 | msg = "New and old parameter lists are different lengths for %s" 251 | self.logger.debug(msg, self.name) 252 | return False 253 | 254 | for param in cf_stack.parameters: 255 | # check if param in CF exists in our new parameter set, 256 | # if not they are differenet and need updating 257 | key = param.key 258 | value = param.value 259 | if key not in self.params: 260 | msg = ("New params are missing key %s that exists in CF " + 261 | "for %s stack already.") 262 | self.logger.debug(msg, key, self.name) 263 | return False 264 | # if the value of parameters are different, needs updating 265 | if self.params[key] != value: 266 | msg = "Param %s for stack %s has changed from %s to %s" 267 | self.logger.debug(msg, key, self.name, 268 | value, self.params[key]) 269 | return False 270 | 271 | # We got to the end without returning False, so must be fine. 272 | return True 273 | -------------------------------------------------------------------------------- /cumulus/MegaStack.py: -------------------------------------------------------------------------------- 1 | """ 2 | Megastack module represents a logical mega stack that contains CloudFormation 3 | stacks and the relationship between them 4 | """ 5 | import boto 6 | import logging 7 | import simplejson 8 | import time 9 | import yaml 10 | import pystache 11 | import os 12 | from cumulus.CFStack import CFStack 13 | from boto import cloudformation, iam, sts 14 | from boto.exception import BotoServerError 15 | 16 | 17 | class MegaStack(object): 18 | """ 19 | Main worker class for cumulus. Holds array of CFstack objects and does most 20 | of the calls to CloudFormation API 21 | """ 22 | def __init__(self, yamlFile): 23 | self.logger = logging.getLogger(__name__) 24 | 25 | # load the yaml file and turn it into a dict 26 | thefile = open(yamlFile, 'r') 27 | 28 | rendered_file = pystache.render(thefile.read(), dict(os.environ)) 29 | 30 | self.stackDict = yaml.safe_load(rendered_file) 31 | # Make sure there is only one top level element in the yaml file 32 | if len(self.stackDict.keys()) != 1: 33 | error_message = ("Need one and only one mega stack name at the" + 34 | " top level, found %s") 35 | self.logger.critical(error_message % len(self.stackDict.keys())) 36 | exit(1) 37 | 38 | # Now we know we only have one top element, 39 | # that must be the mega stack name 40 | self.name = self.stackDict.keys()[0] 41 | 42 | # Find and set the mega stacks region. Exit if we can't find it 43 | if 'region' in self.stackDict[self.name]: 44 | self.region = self.stackDict[self.name]['region'] 45 | else: 46 | self.logger.critical("No region specified for mega stack," + 47 | " don't know where to build it.") 48 | exit(1) 49 | 50 | # Find and set the mega stack's AWS profile 51 | if 'aws_profile' in self.stackDict[self.name]: 52 | self.aws_profile = self.stackDict[self.name]['aws_profile'] 53 | else: 54 | self.aws_profile = None 55 | 56 | # Find and set the mega stack's STS role ARN 57 | if 'sts_role' in self.stackDict[self.name]: 58 | self.sts_role = self.stackDict[self.name]['sts_role'] 59 | else: 60 | self.sts_role = None 61 | 62 | # Connect to STS and assume the provided role 63 | if self.sts_role is not None: 64 | try: 65 | stsconn = sts.connect_to_region(self.region, 66 | profile_name=self.aws_profile) 67 | role = stsconn.assume_role(role_arn=self.sts_role, 68 | role_session_name='cumulus') 69 | self.aws_access_key_id = role.credentials.access_key 70 | self.aws_secret_access_key = role.credentials.secret_key 71 | self.aws_session_token = role.credentials.session_token 72 | self.logger.info("Using STS credentials to set up stack, stack" 73 | + " creation may fail if it takes longer than" 74 | + " 1 hour") 75 | except BotoServerError as e: 76 | self.logger.critical("Could not assume STS role") 77 | self.logger.critical(e.message) 78 | exit(1) 79 | else: 80 | self.aws_access_key_id = None 81 | self.aws_secret_access_key = None 82 | self.aws_session_token = None 83 | 84 | # Connect to an AWS service using proper credentials 85 | def connect(service): 86 | kwargs = {} 87 | if self.aws_access_key_id is not None \ 88 | and self.aws_secret_access_key is not None: 89 | # Using an STS assumed role 90 | kwargs['aws_access_key_id'] = self.aws_access_key_id 91 | kwargs['aws_secret_access_key'] = self.aws_secret_access_key 92 | kwargs['security_token'] = self.aws_session_token 93 | elif self.aws_profile is not None: 94 | # Using an AWS profile 95 | kwargs['profile_name'] = self.aws_profile 96 | return service.connect_to_region(self.region, **kwargs) 97 | 98 | if 'account_id' in self.stackDict[self.name]: 99 | # Get the account ID for the current AWS credentials 100 | iamconn = connect(iam) 101 | user_response = iamconn.get_user()['get_user_response'] 102 | user_result = user_response['get_user_result'] 103 | account_id = user_result['user']['arn'].split(':')[4] 104 | 105 | # Check if the current account ID matches the stack's account ID 106 | if account_id != str(self.stackDict[self.name]['account_id']): 107 | self.logger.critical("Account ID of stack does not match the" + 108 | " account ID of your AWS credentials.") 109 | exit(1) 110 | 111 | self.sns_topic_arn = self.stackDict[self.name].get('sns-topic-arn', []) 112 | if isinstance(self.sns_topic_arn, str): 113 | self.sns_topic_arn = [self.sns_topic_arn] 114 | for topic in self.sns_topic_arn: 115 | if topic.split(':')[3] != self.region: 116 | self.logger.critical("SNS Topic %s is not in the %s region." 117 | % (topic, self.region)) 118 | exit(1) 119 | 120 | self.global_tags = self.stackDict[self.name].get('tags', {}) 121 | # Array for holding CFStack objects once we create them 122 | self.stack_objs = [] 123 | 124 | # Get the names of the sub stacks from the yaml file and sort in array 125 | self.cf_stacks = self.stackDict[self.name]['stacks'].keys() 126 | 127 | # Megastack holds the connection to CloudFormation and list of stacks 128 | # currently in our region stops us making lots of calls to 129 | # CloudFormation API for each stack 130 | try: 131 | self.cfconn = connect(cloudformation) 132 | self.cf_desc_stacks = self._describe_all_stacks() 133 | except boto.exception.NoAuthHandlerFound as exception: 134 | self.logger.critical( 135 | "No credentials found for connecting to CloudFormation: %s" 136 | % exception) 137 | exit(1) 138 | 139 | # iterate through the stacks in the yaml file and create CFstack 140 | # objects for them 141 | for stack_name in self.cf_stacks: 142 | the_stack = self.stackDict[self.name]['stacks'][stack_name] 143 | if type(the_stack) is dict: 144 | if the_stack.get('disable', False): 145 | warn_message = ("Stack %s is disabled by configuration" + 146 | " directive. Skipping") 147 | self.logger.warning(warn_message % stack_name) 148 | continue 149 | local_sns_arn = the_stack.get('sns-topic-arn', 150 | self.sns_topic_arn) 151 | if isinstance(local_sns_arn, str): 152 | local_sns_arn = [local_sns_arn] 153 | for topic in local_sns_arn: 154 | if topic.split(':')[3] != self.region: 155 | error_message = "SNS Topic %s is not in the %s region." 156 | self.logger.critical(error_message 157 | % (topic, self.region)) 158 | exit(1) 159 | local_tags = the_stack.get('tags', {}) 160 | merged_tags = dict(self.global_tags.items() + 161 | local_tags.items()) 162 | # Add static cumulus-stack tag 163 | merged_tags['cumulus-stack'] = self.name 164 | if 'cf_template' in the_stack: 165 | self.stack_objs.append( 166 | CFStack( 167 | mega_stack_name=self.name, 168 | name=stack_name, 169 | params=the_stack.get('params'), 170 | template_name=the_stack['cf_template'], 171 | cfconn=self.cfconn, 172 | sns_topic_arn=local_sns_arn, 173 | depends_on=the_stack.get('depends'), 174 | tags=merged_tags 175 | ) 176 | ) 177 | 178 | def sort_stacks_by_deps(self): 179 | """ 180 | Sort the array of stack_objs so they are in dependancy order 181 | """ 182 | sorted_stacks = [] 183 | dep_graph = {} 184 | no_deps = [] 185 | # Add all stacks without dependancies to no_deps 186 | for stack in self.stack_objs: 187 | if stack.depends_on is None: 188 | no_deps.append(stack) 189 | else: 190 | dep_graph[stack.name] = stack.depends_on[:] 191 | # Perform a topological sort on stacks in dep_graph 192 | while len(no_deps) > 0: 193 | stack = no_deps.pop() 194 | sorted_stacks.append(stack) 195 | for node in dep_graph.keys(): 196 | for deps in dep_graph[node]: 197 | if stack.cf_stack_name == deps: 198 | dep_graph[node].remove(stack.cf_stack_name) 199 | if len(dep_graph[node]) < 1: 200 | for stack_obj in self.stack_objs: 201 | if stack_obj.name == node: 202 | no_deps.append(stack_obj) 203 | del dep_graph[node] 204 | if len(dep_graph) > 0: 205 | self.logger.critical("Could not resolve dependency order." + 206 | " Either circular dependency or " + 207 | "dependency on stack not in yaml file.") 208 | exit(1) 209 | else: 210 | self.stack_objs = sorted_stacks 211 | return True 212 | 213 | def check(self, stack_name=None): 214 | """ 215 | Checks the status of the yaml file. 216 | Displays parameters for the stacks it can. 217 | """ 218 | for stack in self.stack_objs: 219 | if stack_name and stack.name != stack_name: 220 | continue 221 | self.logger.info("Starting check of stack %s" % stack.name) 222 | if not stack.populate_params(self.cf_desc_stacks): 223 | info_message = ("Could not determine correct parameters for" + 224 | "CloudFormation stack %s\n\tMost likely " + 225 | "because stacks it depends on haven't been " + 226 | "created yet.") 227 | self.logger.info(info_message, stack.name) 228 | else: 229 | self.logger.info("Stack %s would be created with following " 230 | "parameter values: %s" 231 | % (stack.cf_stack_name, 232 | stack.get_params_tuples())) 233 | self.logger.info("Stack %s already exists in CF: %s" 234 | % (stack.cf_stack_name, 235 | bool(stack.exists_in_cf( 236 | self.cf_desc_stacks)))) 237 | 238 | def create(self, stack_name=None): 239 | """ 240 | Create all stacks in the yaml file. 241 | Any that already exist are skipped (no attempt to update) 242 | """ 243 | for stack in self.stack_objs: 244 | if stack_name and stack.name != stack_name: 245 | continue 246 | self.logger.info("Starting checks for creation of stack: %s" 247 | % stack.name) 248 | if stack.exists_in_cf(self.cf_desc_stacks): 249 | self.logger.info("Stack %s already exists in CloudFormation," 250 | " skipping" % stack.name) 251 | else: 252 | if stack.deps_met(self.cf_desc_stacks) is False: 253 | self.logger.critical("Dependancies for stack %s not met" 254 | " and they should be, exiting..." 255 | % stack.name) 256 | exit(1) 257 | if not stack.populate_params(self.cf_desc_stacks): 258 | self.logger.critical("Could not determine correct " 259 | "parameters for stack %s" 260 | % stack.name) 261 | exit(1) 262 | 263 | stack.read_template() 264 | self.logger.info("Creating: %s, %s" % ( 265 | stack.cf_stack_name, stack.get_params_tuples())) 266 | try: 267 | self.cfconn.create_stack( 268 | stack_name=stack.cf_stack_name, 269 | template_body=stack.template_body, 270 | parameters=stack.get_params_tuples(), 271 | capabilities=[ 272 | 'CAPABILITY_IAM', 273 | 'CAPABILITY_NAMED_IAM'], 274 | notification_arns=stack.sns_topic_arn, 275 | tags=stack.tags 276 | ) 277 | except Exception as exception: 278 | self.logger.critical( 279 | "Creating stack %s failed. Error: %s" % ( 280 | stack.cf_stack_name, exception)) 281 | exit(1) 282 | 283 | create_result = self.watch_events( 284 | stack.cf_stack_name, "CREATE_IN_PROGRESS") 285 | if create_result != "CREATE_COMPLETE": 286 | self.logger.critical( 287 | "Stack didn't create correctly, status is now %s" 288 | % create_result) 289 | exit(1) 290 | 291 | # CF told us stack completed ok. 292 | # Log message to that effect and refresh the list of stack 293 | # objects in CF 294 | self.logger.info("Finished creating stack: %s" 295 | % stack.cf_stack_name) 296 | self.cf_desc_stacks = self._describe_all_stacks() 297 | 298 | def delete(self, stack_name=None): 299 | """ 300 | Delete all the stacks from CloudFormation. 301 | Does this in reverse dependency order. 302 | Prompts for confirmation before deleting each stack 303 | """ 304 | # Removing stacks so need to do it in reverse dependency order 305 | for stack in reversed(self.stack_objs): 306 | if stack_name and stack.name != stack_name: 307 | continue 308 | self.logger.info("Starting checks for deletion of stack: %s" 309 | % stack.name) 310 | if not stack.exists_in_cf(self.cf_desc_stacks): 311 | self.logger.info( 312 | "Stack %s doesn't exist in CloudFormation, skipping" 313 | % stack.name) 314 | else: 315 | confirm = raw_input( 316 | "Confirm you wish to delete stack %s (Name in CF: %s)" 317 | " (type 'yes' if so): " 318 | % (stack.name, stack.cf_stack_name)) 319 | if not confirm == "yes": 320 | self.logger.info("Not confirmed, skipping...") 321 | continue 322 | self.logger.info("Starting delete of stack %s" % stack.name) 323 | self.cfconn.delete_stack(stack.cf_stack_name) 324 | delete_result = self.watch_events( 325 | stack.cf_stack_name, "DELETE_IN_PROGRESS") 326 | if (delete_result != "DELETE_COMPLETE" and 327 | delete_result != "STACK_GONE"): 328 | self.logger.critical( 329 | "Stack didn't delete correctly, status is now %s" 330 | % delete_result) 331 | exit(1) 332 | 333 | # CF told us stack completed ok. Log message to that effect and 334 | # refresh the list of stack objects in CF 335 | self.logger.info("Finished deleting stack: %s" 336 | % stack.cf_stack_name) 337 | self.cf_desc_stacks = self._describe_all_stacks() 338 | 339 | def update(self, stack_name=None): 340 | """ 341 | Attempts to update each of the stacks if template or parameters are 342 | different to what's currently in CloudFormation. 343 | If a stack doesn't already exist. Logs critical error and exits. 344 | """ 345 | for stack in self.stack_objs: 346 | if stack_name and stack.name != stack_name: 347 | continue 348 | self.logger.info("Starting checks for update of stack: %s" 349 | % stack.name) 350 | if not stack.exists_in_cf(self.cf_desc_stacks): 351 | self.logger.critical( 352 | "Stack %s doesn't exist in cloudformation, can't update" 353 | " something that doesn't exist." % stack.name) 354 | exit(1) 355 | if not stack.deps_met(self.cf_desc_stacks): 356 | self.logger.critical( 357 | "Dependencies for stack %s not met and they should be," 358 | " exiting..." % stack.name) 359 | exit(1) 360 | if not stack.populate_params(self.cf_desc_stacks): 361 | self.logger.critical("Could not determine correct parameters" 362 | " for stack %s" % stack.name) 363 | exit(1) 364 | stack.read_template() 365 | template_up_to_date = stack.template_uptodate(self.cf_desc_stacks) 366 | params_up_to_date = stack.params_uptodate(self.cf_desc_stacks) 367 | self.logger.debug("Stack is up to date: %s" 368 | % (template_up_to_date and params_up_to_date)) 369 | if template_up_to_date and params_up_to_date: 370 | self.logger.info( 371 | "Stack %s is already up to date with CloudFormation," 372 | " skipping..." % stack.name) 373 | else: 374 | if not template_up_to_date: 375 | self.logger.info( 376 | "Template for stack %s has changed." % stack.name) 377 | # Would like to get this working. Tried datadiff but can't 378 | # stop it from printing whole template 379 | # stack.print_template_diff(self.cf_desc_stacks) 380 | self.logger.info( 381 | "Starting update of stack %s with parameters: %s" 382 | % (stack.name, stack.get_params_tuples())) 383 | self.cfconn.validate_template( 384 | template_body=stack.template_body) 385 | 386 | try: 387 | self.cfconn.update_stack( 388 | stack_name=stack.cf_stack_name, 389 | template_body=stack.template_body, 390 | parameters=stack.get_params_tuples(), 391 | capabilities=[ 392 | 'CAPABILITY_IAM', 393 | 'CAPABILITY_NAMED_IAM'], 394 | tags=stack.tags, 395 | ) 396 | except boto.exception.BotoServerError as exception: 397 | try: 398 | e_message_dict = simplejson.loads(exception[2]) 399 | if (str(e_message_dict["Error"]["Message"]) == 400 | "No updates are to be performed."): 401 | self.logger.error( 402 | "CloudFormation has no updates to perform on" 403 | " %s, this might be because there is a " 404 | "parameter with NoEcho set" % stack.name) 405 | continue 406 | else: 407 | self.logger.error( 408 | "Got error message: %s" 409 | % e_message_dict["Error"]["Message"]) 410 | raise exception 411 | except simplejson.decoder.JSONDecodeError: 412 | self.logger.critical( 413 | "Unknown error updating stack: %s", exception) 414 | exit(1) 415 | update_result = self.watch_events( 416 | stack.cf_stack_name, [ 417 | "UPDATE_IN_PROGRESS", 418 | "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS"]) 419 | if update_result != "UPDATE_COMPLETE": 420 | self.logger.critical( 421 | "Stack didn't update correctly, status is now %s" 422 | % update_result) 423 | exit(1) 424 | 425 | self.logger.info( 426 | "Finished updating stack: %s" % stack.cf_stack_name) 427 | 428 | # avoid getting rate limited 429 | time.sleep(2) 430 | 431 | def watch(self, stack_name): 432 | """ 433 | Watch events for a given CloudFormation stack. 434 | It will keep watching until its state changes 435 | """ 436 | if not stack_name: 437 | self.logger.critical( 438 | "No stack name passed in, nothing to watch... use -s to " 439 | "provide stack name.") 440 | exit(1) 441 | the_stack = False 442 | for stack in self.stack_objs: 443 | if stack_name == stack.name: 444 | the_stack = stack 445 | if not the_stack: 446 | self.logger.error("Cannot find stack %s to watch" % stack_name) 447 | return False 448 | the_cf_stack = the_stack.exists_in_cf(self.cf_desc_stacks) 449 | if not the_cf_stack: 450 | self.logger.error( 451 | "Stack %s doesn't exist in CloudFormation, can't watch " 452 | "something that doesn't exist." % stack_name) 453 | return False 454 | 455 | self.logger.info( 456 | "Watching stack %s, while in state %s." 457 | % (the_stack.cf_stack_name, str(the_cf_stack.stack_status))) 458 | self.watch_events( 459 | the_stack.cf_stack_name, str(the_cf_stack.stack_status)) 460 | 461 | def watch_events(self, stack_name, while_status): 462 | """ 463 | Used by the various actions to watch CloudFormation events 464 | while a stacks in a given state 465 | """ 466 | try: 467 | cfstack_obj = self.cfconn.describe_stacks(stack_name)[0] 468 | events = list(self.cfconn.describe_stack_events(stack_name)) 469 | except boto.exception.BotoServerError as exception: 470 | if (str(exception.error_message) == 471 | "Stack:%s does not exist" % (stack_name)): 472 | return "STACK_GONE" 473 | 474 | colors = { 475 | 'blue': '\033[0;34m', 476 | 'red': '\033[0;31m', 477 | 'bred': '\033[1;31m', 478 | 'green': '\033[0;32m', 479 | 'bgreen': '\033[1;32m', 480 | 'yellow': '\033[0;33m', 481 | } 482 | 483 | status_color_map = { 484 | 'CREATE_IN_PROGRESS': colors['blue'], 485 | 'CREATE_FAILED': colors['bred'], 486 | 'CREATE_COMPLETE': colors['green'], 487 | 'ROLLBACK_IN_PROGRESS': colors['red'], 488 | 'ROLLBACK_FAILED': colors['bred'], 489 | 'ROLLBACK_COMPLETE': colors['yellow'], 490 | 'DELETE_IN_PROGRESS': colors['red'], 491 | 'DELETE_FAILED': colors['bred'], 492 | 'DELETE_COMPLETE': colors['yellow'], 493 | 'UPDATE_IN_PROGRESS': colors['blue'], 494 | 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS': colors['blue'], 495 | 'UPDATE_COMPLETE': colors['bgreen'], 496 | 'UPDATE_ROLLBACK_IN_PROGRESS': colors['red'], 497 | 'UPDATE_ROLLBACK_FAILED': colors['bred'], 498 | 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS': colors['red'], 499 | 'UPDATE_ROLLBACK_COMPLETE': colors['yellow'], 500 | 'UPDATE_FAILED': colors['bred'], 501 | } 502 | # print the last 5 events, so we get to see the start of the action we 503 | # are performing 504 | self.logger.info("Last 5 events for this stack:") 505 | for event in reversed(events[:5]): 506 | if self.stackDict[self.name].get('highlight-output', True): 507 | self.logger.info("%s %s%s\033[0m %s %s %s %s" % ( 508 | event.timestamp.isoformat(), 509 | status_color_map.get(event.resource_status, ''), 510 | event.resource_status, 511 | event.resource_type, 512 | event.logical_resource_id, 513 | event.physical_resource_id, 514 | event.resource_status_reason, 515 | )) 516 | else: 517 | self.logger.info("%s %s %s %s %s %s" % ( 518 | event.timestamp.isoformat(), 519 | event.resource_status, 520 | event.resource_type, 521 | event.logical_resource_id, 522 | event.physical_resource_id, 523 | event.resource_status_reason, 524 | )) 525 | status = str(cfstack_obj.stack_status) 526 | self.logger.info("New events:") 527 | while status in while_status: 528 | try: 529 | new_events = list( 530 | self.cfconn.describe_stack_events(stack_name)) 531 | except boto.exception.BotoServerError as exception: 532 | if (str(exception.error_message) == 533 | "Stack:%s does not exist" % (stack_name)): 534 | return "STACK_GONE" 535 | count = 0 536 | events_to_log = [] 537 | while (events[0].timestamp != new_events[count].timestamp or 538 | events[0].logical_resource_id != 539 | new_events[count].logical_resource_id): 540 | events_to_log.insert(0, new_events[count]) 541 | count += 1 542 | for event in events_to_log: 543 | if self.stackDict[self.name].get('highlight-output', True): 544 | self.logger.info("%s %s%s\033[0m %s %s %s %s" % ( 545 | event.timestamp.isoformat(), 546 | status_color_map.get(event.resource_status, ''), 547 | event.resource_status, 548 | event.resource_type, 549 | event.logical_resource_id, 550 | event.physical_resource_id, 551 | event.resource_status_reason, 552 | )) 553 | else: 554 | self.logger.info("%s %s %s %s %s %s" % ( 555 | event.timestamp.isoformat(), 556 | event.resource_status, 557 | event.resource_type, 558 | event.logical_resource_id, 559 | event.physical_resource_id, 560 | event.resource_status_reason, 561 | )) 562 | if count > 0: 563 | events = new_events[:] 564 | cfstack_obj.update() 565 | status = str(cfstack_obj.stack_status) 566 | time.sleep(5) 567 | return status 568 | 569 | def _describe_all_stacks(self): 570 | """ 571 | Get all pages of stacks from describe_stacks API call. 572 | """ 573 | result = [] 574 | resp = self.cfconn.describe_stacks() 575 | result.extend(resp) 576 | while resp.next_token: 577 | resp = self.cfconn.describe_stacks(next_token=resp.next_token) 578 | result.extend(resp) 579 | return result 580 | --------------------------------------------------------------------------------