├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── examples ├── stack1 │ ├── README.md │ ├── files │ │ ├── images.json │ │ └── userdata.txt │ ├── parameters.yaml │ ├── run.sh │ └── template.yaml ├── stack2 │ ├── README.md │ ├── files │ │ ├── images.json │ │ └── userdata.txt │ ├── parameters.yaml │ ├── run.sh │ └── templates │ │ ├── AutoscalingGroup.yaml │ │ └── SecurityGroup.yaml └── stack3 │ ├── README.md │ ├── files │ ├── images.json │ └── userdata.txt │ ├── parameters │ ├── base.yaml │ ├── database.yaml │ ├── loadbalancer.yaml │ └── web.yaml │ ├── run.sh │ └── templates │ ├── GenericRole.yaml │ └── SecurityGroups │ ├── All.yaml │ ├── Database.yaml │ ├── LoadBalancer.yaml │ └── WebServer.yaml ├── rainbow ├── __init__.py ├── cloudformation.py ├── datasources │ ├── __init__.py │ ├── base.py │ ├── cfn_datasource.py │ ├── datasource_exceptions.py │ ├── file_datasource.py │ └── yaml_datasource.py ├── main.py ├── preprocessor │ ├── __init__.py │ ├── base.py │ ├── instance_chooser.py │ └── preprocessor_exceptions.py ├── templates.py └── yaml_loader.py ├── setup.py └── tests ├── __init__.py ├── cfn_deep_merge ├── a.yaml ├── b.yaml └── c.yaml ├── datasources ├── a.yaml ├── b.yaml ├── c.yaml ├── d.file ├── e.file64 └── nested.yaml ├── preprocessor ├── instance_chooser1.yaml ├── instance_chooser2.yaml ├── instance_chooser3.yaml └── instance_chooser4.yaml ├── templates ├── default_parameters_template.yaml └── simpletemplate.yaml ├── test_cfn_datasource.py ├── test_cfn_deep_merge.py ├── test_cloudformation.py ├── test_datasources.py ├── test_preprocessor.py ├── test_yamlfile.py └── yamlfile ├── base.yaml ├── includeme.file ├── includeme.file64 └── includeme.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # Installer logs 27 | pip-log.txt 28 | pip-delete-this-directory.txt 29 | 30 | # Unit test / coverage reports 31 | htmlcov/ 32 | .tox/ 33 | .coverage 34 | .cache 35 | nosetests.xml 36 | coverage.xml 37 | 38 | # Translations 39 | *.mo 40 | *.pot 41 | 42 | # Django stuff: 43 | *.log 44 | 45 | # Sphinx documentation 46 | docs/_build/ 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | 5 | install: pip install -e .[test] 6 | script: cd tests; nosetests --with-coverage --cover-package=rainbow --cover-erase 7 | 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v0.4 - 20150120 2 | * Fixed a bug in handling of comma separated parameters 3 | * Added --update-stack-if-exists command line option 4 | * When updating a stack with no updates, exit with 0 5 | * Comments in example bash scripts fixed 6 | * New datasource: cfn_parameters 7 | 8 | # v0.3 - 20140713 9 | * Fixed Rb::InstanceChooser not to accept a string parameter, only lists 10 | * Fixed a bug where Rb::InstanceChooser didn't handle pointers to lists properly 11 | * Fixed handling of default values in Cloudformation parameters 12 | * Template !yaml magic can now specify a root key 13 | * Fixed preprocessing bug (Rb:: function handling) with int keys in templates 14 | 15 | # v0.2 - 20140610 16 | * setup.py changes for PIP 17 | * README updated on usage/installation 18 | 19 | # v0.1 - 20140610 20 | * Initial release 21 | 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2014 DoAT. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY DoAT ``AS IS'' AND ANY EXPRESS OR IMPLIED 14 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 15 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 16 | EVENT SHALL DoAT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 17 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 18 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, 19 | OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 20 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 21 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 22 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are 25 | those of the authors and should not be interpreted as representing official 26 | policies, either expressed or implied, of DoAT 27 | 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/EverythingMe/rainbow.svg?branch=master)](https://travis-ci.org/EverythingMe/rainbow) 2 | 3 | # Rainbow - AWS Cloudformation on steroids 4 | 5 | Working with Cloudformation inhouse, we discovered the following needs: 6 | * Break a big stack into several smaller ones 7 | * Reference resources *between* stacks and regions 8 | * Add dynamic template preprocessing logic 9 | * Compose a stack from reusable 'building blocks' 10 | * Improve readability by coding templates and parameters in YAML (and have comments!) 11 | 12 | # Installation 13 | `pip install rainbow-cfn` 14 | 15 | Usage 16 | ===== 17 | * [Configure boto](http://boto.readthedocs.org/en/latest/boto_config_tut.html) 18 | * Run `rainbow` 19 | 20 | Datasources 21 | =========== 22 | 23 | What is a datasource 24 | -------------------- 25 | Datasource is a key-value mapping which Rainbow uses to fill [Cloudformation templates parameters](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/concept-parameters.html) 26 | Datasources can be YAML files, other Cloudformation stacks (resources, outputs and even parameters), files, etc. 27 | The system is fully extensible for you to add any other datasources you might want. 28 | 29 | Every parameter the template requires is being looked up on the list of datasources given to the rainbow CLI tool. A parameter can exist on multiple datasources, having the first match returned. 30 | For example, say you run rainbow with `--data-source yaml:a.yaml --data-source yaml:b.yaml --data-source:c.yaml`, the files are: 31 | ```yaml 32 | # a.yaml 33 | InstanceType: m1.small 34 | KeyName: omrib-laptop 35 | AvailabilityZones: 36 | - us-east-1a 37 | 38 | # b.yaml 39 | AutoScalingGroupMinSize: 0 40 | AutoScalingGroupMaxSize: 1 41 | Image: PreciseAmd64PvInstanceStore 42 | 43 | # c.yaml 44 | InstanceType: m1.large 45 | ``` 46 | 47 | Lets say your template requires the following parameters: 48 | ```yaml 49 | Parameters: 50 | AvailabilityZones: {Type: CommaDelimitedList} 51 | KeyName: {Type: String} 52 | AutoScalingGroupMinSize: {Type: String} 53 | AutoScalingGroupMaxSize: {Type: String} 54 | Image: {Type: String} 55 | InstanceType: {Type: String} 56 | ``` 57 | 58 | The lookup order is `a.yaml`, `b.yaml` and `c.yaml`. Rainbow will fill all the parameters as follows: 59 | ``` 60 | AvailabilityZones: us-east-1a (from a.yaml) 61 | KeyName: omrib-laptop (from a.yaml) 62 | AutoScalingGroupMinSize: 0 (from b.yaml) 63 | AutoScalingGroupMaxSize: 1 (from b.yaml) 64 | Image: PreciseAmd64PvInstanceStore (from b.yaml) 65 | InstanceType: m1.small (from a.yaml. Even though the value exists in c.yaml as well, because the yaml:a.yaml datasource appears before yaml:c.yaml, the value will be m1.small and not m1.large) 66 | ``` 67 | 68 | Another cool feature of the YAML datasource is pointers. You can make the lookup begin all over again when a key has a value that begins with `$`. For example, when running rainbow with `--data-source yaml:a.yaml --data-source yaml:b.yaml`: 69 | ```yaml 70 | # a.yaml 71 | AdminKeyName: omrib-laptop 72 | 73 | # b.yaml 74 | KeyName: $AdminKeyName 75 | AdminKeyName: oren-laptop 76 | ``` 77 | 78 | The value of `KeyName` (`b.yaml`) points to `AdminKeyName`, which exists both on `a.yaml` and `b.yaml`, but the `a.yaml` value will be used, meaning `KeyName` equals `omrib-laptop`. 79 | Pointers can be used to reference a key from a different type of datasource. 80 | 81 | 82 | ## Available datasources 83 | 84 | ### YAML 85 | `yaml[:rootkey]:path/to/file` - stores all keys (starting from rootkey, if given) with their values 86 | 87 | ### cfn_resources 88 | `cfn_resources[:region]:stackname` - stores a logical resource to physical resource mapping. [Read more about Cloudformation resources](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/concept-resources.html) 89 | 90 | ### cfn_outputs 91 | `cfn_outputs[:region]:stackname` - stores a output to value mapping. [Read more about Cloudformation outputs](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/concept-outputs.html) 92 | 93 | ### cfn_parameters 94 | `cfn_parameters[:region]:stackname` - input parameters to value mapping. You can use this as the only datasource when you only want to modify the template without touching the parameters. 95 | 96 | ### file 97 | `file:name:path/to/file` - stores a single key `name` with the value of the file content 98 | 99 | ### file64 100 | `file64:name:path/to/file` - same as `file`, but returns a BASE64 string instead of plaintext 101 | 102 | # Rainbow functions 103 | 104 | ## Rb::InstanceChooser 105 | 106 | From a given list of possible instance types, choose the first one that exists on that region. 107 | Suppose you have a CFN stack that should be using a `c3.large` instance, but in a particular region that instance family is not yet supported. In that case, you want it to fallback to `c1.medium`. 108 | A code of `{'Rb::InstanceChooser': ['c3.large', 'c1.medium']}` will evaluate to `c3.large` on regions that supports it and `c1.medium` on regions that don't. 109 | -------------------------------------------------------------------------------- /examples/stack1/README.md: -------------------------------------------------------------------------------- 1 | EXAMPLE 1 - Basic stack 2 | ======================= 3 | 4 | Basic example using the following features: 5 | * Using `!yaml` to include a JSON file for the mappings 6 | * Using `!file64` to include the userdata.txt file 7 | 8 | Note that the stack parameters are coming from the parameters.yaml file 9 | 10 | -------------------------------------------------------------------------------- /examples/stack1/files/images.json: -------------------------------------------------------------------------------- 1 | {"us-east-1": {"PreciseAmd64PvEbs": "ami-59a4a230", "PreciseAmd64PvInstanceStore": "ami-c9d7d1a0", "PreciseAmd64HvmEbs": "ami-5fa4a236", "PreciseI386PvInstanceStore": "ami-13cdcb7a", "PreciseAmd64HvmInstanceStore": "ami-a9dddbc0", "PreciseI386PvEbs": "ami-45a4a22c"}, "ap-northeast-1": {"PreciseAmd64PvEbs": "ami-f381f5f2", "PreciseAmd64PvInstanceStore": "ami-31a1d530", "PreciseAmd64HvmEbs": "ami-f581f5f4", "PreciseI386PvInstanceStore": "ami-e1a7d3e0", "PreciseAmd64HvmInstanceStore": "ami-6586f264", "PreciseI386PvEbs": "ami-ef81f5ee"}, "eu-west-1": {"PreciseAmd64PvEbs": "ami-808675f7", "PreciseAmd64PvInstanceStore": "ami-6aab581d", "PreciseAmd64HvmEbs": "ami-8c8675fb", "PreciseI386PvInstanceStore": "ami-3aa1524d", "PreciseAmd64HvmInstanceStore": "ami-0caf5c7b", "PreciseI386PvEbs": "ami-828675f5"}, "ap-southeast-1": {"PreciseAmd64PvEbs": "ami-3c39686e", "PreciseAmd64PvInstanceStore": "ami-8e3061dc", "PreciseAmd64HvmEbs": "ami-22396870", "PreciseI386PvInstanceStore": "ami-14306146", "PreciseAmd64HvmInstanceStore": "ami-56396804", "PreciseI386PvEbs": "ami-3839686a"}, "ap-southeast-2": {"PreciseAmd64PvEbs": "ami-09f26b33", "PreciseAmd64PvInstanceStore": "ami-47ff667d", "PreciseAmd64HvmEbs": "ami-0ff26b35", "PreciseI386PvInstanceStore": "ami-f7fc65cd", "PreciseAmd64HvmInstanceStore": "ami-15f26b2f", "PreciseI386PvEbs": "ami-0bf26b31"}, "us-west-2": {"PreciseAmd64PvEbs": "ami-fa9cf1ca", "PreciseAmd64PvInstanceStore": "ami-84b6dbb4", "PreciseAmd64HvmEbs": "ami-fc9cf1cc", "PreciseI386PvInstanceStore": "ami-30b7da00", "PreciseAmd64HvmInstanceStore": "ami-88b0ddb8", "PreciseI386PvEbs": "ami-f89cf1c8"}, "us-west-1": {"PreciseAmd64PvEbs": "ami-660c3023", "PreciseAmd64PvInstanceStore": "ami-b21f23f7", "PreciseAmd64HvmEbs": "ami-620c3027", "PreciseI386PvInstanceStore": "ami-cc1e2289", "PreciseAmd64HvmInstanceStore": "ami-c0192585", "PreciseI386PvEbs": "ami-640c3021"}, "sa-east-1": {"PreciseAmd64PvEbs": "ami-1da90a00", "PreciseAmd64PvInstanceStore": "ami-3da60520", "PreciseAmd64HvmEbs": "ami-1fa90a02", "PreciseI386PvInstanceStore": "ami-c9a003d4", "PreciseAmd64HvmInstanceStore": "ami-dbaa09c6", "PreciseI386PvEbs": "ami-e3aa09fe"}} 2 | -------------------------------------------------------------------------------- /examples/stack1/files/userdata.txt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | touch /root/rainbow 3 | 4 | -------------------------------------------------------------------------------- /examples/stack1/parameters.yaml: -------------------------------------------------------------------------------- 1 | AutoScalingGroupMaxSize: 1 2 | AutoScalingGroupMinSize: 0 3 | AvailabilityZones: 4 | - us-east-1a 5 | Image: PreciseAmd64PvInstanceStore 6 | InstanceType: m1.small 7 | KeyName: omrib-laptop 8 | 9 | -------------------------------------------------------------------------------- /examples/stack1/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | rainbow \ 3 | --block \ 4 | --data-source yaml:parameters.yaml `# the only data source we have for this stack is our parameters.yaml file` \ 5 | rainbow-stack1 `# stack name` \ 6 | template.yaml `# stack template` 7 | 8 | -------------------------------------------------------------------------------- /examples/stack1/template.yaml: -------------------------------------------------------------------------------- 1 | Mappings: 2 | Images: !yaml files/images.json # because JSON is a valid YAML, we can include it with the !yaml directive 3 | # this file contains the mapping from region and image name to AMI id 4 | Parameters: # all the parameters will be loaded from the parameters.yaml data source 5 | AvailabilityZones: {Type: CommaDelimitedList} 6 | KeyName: {Type: String} 7 | AutoScalingGroupMinSize: {Type: String} 8 | AutoScalingGroupMaxSize: {Type: String} 9 | Image: {Type: String} 10 | InstanceType: {Type: String} 11 | Resources: 12 | AutoScalingGroup: 13 | Type: AWS::AutoScaling::AutoScalingGroup 14 | Properties: 15 | AvailabilityZones: {Ref: AvailabilityZones} 16 | Cooldown: 1200 17 | HealthCheckGracePeriod: 1200 18 | HealthCheckType: EC2 19 | LaunchConfigurationName: {Ref: LaunchConfig} 20 | MinSize: {Ref: AutoScalingGroupMinSize} 21 | MaxSize: {Ref: AutoScalingGroupMaxSize} 22 | LaunchConfig: 23 | Type: AWS::AutoScaling::LaunchConfiguration 24 | Properties: 25 | ImageId: {'Fn::FindInMap': [Images, {Ref: 'AWS::Region'}, {Ref: Image}]} 26 | InstanceType: {Ref: InstanceType} 27 | KeyName: {Ref: KeyName} 28 | SecurityGroups: [{Ref: SecurityGroup}] 29 | UserData: !file64 files/userdata.txt # the file/userdata.txt content will be base64 encoded 30 | SecurityGroup: 31 | Type: AWS::EC2::SecurityGroup 32 | Properties: 33 | GroupDescription: Awesome security group 34 | SecurityGroupIngress: 35 | # Allow SSH from everywhere 36 | - IpProtocol: tcp 37 | FromPort: 22 38 | ToPort: 22 39 | CidrIp: 0.0.0.0/0 40 | 41 | # Allow ICMP destination unreachable for TCP PMTU discovery to work properly 42 | - IpProtocol: icmp 43 | FromPort: 3 44 | ToPort: 4 45 | CidrIp: 0.0.0.0/0 46 | 47 | -------------------------------------------------------------------------------- /examples/stack2/README.md: -------------------------------------------------------------------------------- 1 | EXAMPLE 2 - Split stack 2 | ======================= 3 | 4 | This example is based on example1, but I broke the stack into two stacks: 5 | * `rainbow-stack2-sg` containing the `SecurityGroup` resource 6 | * `rainbow-stack2-asg` containing the `AutoscalingGroup` and the `LaunchConfig` resources 7 | 8 | The order of the creation is important because the `rainbow-stack2-asg` stack references the `SecurityGroup` resource from `rainbow-stack2-sg`. It does that by utilizing the `cfn_resources` datasource. You can read more about datasources by reading the main `README.md` file. 9 | 10 | -------------------------------------------------------------------------------- /examples/stack2/files/images.json: -------------------------------------------------------------------------------- 1 | {"us-east-1": {"PreciseAmd64PvEbs": "ami-59a4a230", "PreciseAmd64PvInstanceStore": "ami-c9d7d1a0", "PreciseAmd64HvmEbs": "ami-5fa4a236", "PreciseI386PvInstanceStore": "ami-13cdcb7a", "PreciseAmd64HvmInstanceStore": "ami-a9dddbc0", "PreciseI386PvEbs": "ami-45a4a22c"}, "ap-northeast-1": {"PreciseAmd64PvEbs": "ami-f381f5f2", "PreciseAmd64PvInstanceStore": "ami-31a1d530", "PreciseAmd64HvmEbs": "ami-f581f5f4", "PreciseI386PvInstanceStore": "ami-e1a7d3e0", "PreciseAmd64HvmInstanceStore": "ami-6586f264", "PreciseI386PvEbs": "ami-ef81f5ee"}, "eu-west-1": {"PreciseAmd64PvEbs": "ami-808675f7", "PreciseAmd64PvInstanceStore": "ami-6aab581d", "PreciseAmd64HvmEbs": "ami-8c8675fb", "PreciseI386PvInstanceStore": "ami-3aa1524d", "PreciseAmd64HvmInstanceStore": "ami-0caf5c7b", "PreciseI386PvEbs": "ami-828675f5"}, "ap-southeast-1": {"PreciseAmd64PvEbs": "ami-3c39686e", "PreciseAmd64PvInstanceStore": "ami-8e3061dc", "PreciseAmd64HvmEbs": "ami-22396870", "PreciseI386PvInstanceStore": "ami-14306146", "PreciseAmd64HvmInstanceStore": "ami-56396804", "PreciseI386PvEbs": "ami-3839686a"}, "ap-southeast-2": {"PreciseAmd64PvEbs": "ami-09f26b33", "PreciseAmd64PvInstanceStore": "ami-47ff667d", "PreciseAmd64HvmEbs": "ami-0ff26b35", "PreciseI386PvInstanceStore": "ami-f7fc65cd", "PreciseAmd64HvmInstanceStore": "ami-15f26b2f", "PreciseI386PvEbs": "ami-0bf26b31"}, "us-west-2": {"PreciseAmd64PvEbs": "ami-fa9cf1ca", "PreciseAmd64PvInstanceStore": "ami-84b6dbb4", "PreciseAmd64HvmEbs": "ami-fc9cf1cc", "PreciseI386PvInstanceStore": "ami-30b7da00", "PreciseAmd64HvmInstanceStore": "ami-88b0ddb8", "PreciseI386PvEbs": "ami-f89cf1c8"}, "us-west-1": {"PreciseAmd64PvEbs": "ami-660c3023", "PreciseAmd64PvInstanceStore": "ami-b21f23f7", "PreciseAmd64HvmEbs": "ami-620c3027", "PreciseI386PvInstanceStore": "ami-cc1e2289", "PreciseAmd64HvmInstanceStore": "ami-c0192585", "PreciseI386PvEbs": "ami-640c3021"}, "sa-east-1": {"PreciseAmd64PvEbs": "ami-1da90a00", "PreciseAmd64PvInstanceStore": "ami-3da60520", "PreciseAmd64HvmEbs": "ami-1fa90a02", "PreciseI386PvInstanceStore": "ami-c9a003d4", "PreciseAmd64HvmInstanceStore": "ami-dbaa09c6", "PreciseI386PvEbs": "ami-e3aa09fe"}} 2 | -------------------------------------------------------------------------------- /examples/stack2/files/userdata.txt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | touch /root/rainbow 3 | 4 | -------------------------------------------------------------------------------- /examples/stack2/parameters.yaml: -------------------------------------------------------------------------------- 1 | AutoScalingGroupMaxSize: 1 2 | AutoScalingGroupMinSize: 0 3 | AvailabilityZones: 4 | - us-east-1a 5 | Image: PreciseAmd64PvInstanceStore 6 | InstanceType: m1.small 7 | KeyName: omrib-laptop 8 | 9 | -------------------------------------------------------------------------------- /examples/stack2/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | rainbow \ 3 | --block `# block until the stack creation is finished. exits with a non-zero exit code upon failure` \ 4 | --data-source yaml:parameters.yaml `# the only data source is our parameters.yaml` \ 5 | rainbow-stack2-sg `# stack name` \ 6 | templates/SecurityGroup.yaml `# stack template` 7 | 8 | rainbow `# note that this script is run with bash -e` \ 9 | --block `# block until the stack creation is finished (and show the events)` \ 10 | --data-source yaml:parameters.yaml `# first priority data source is parameters.yaml, like the previous stack` \ 11 | --data-source cfn_resources:rainbow-stack2-sg `# second priority data source is the CFN resources created on previous stack` \ 12 | rainbow-stack2-asg `# stack name` \ 13 | templates/AutoscalingGroup.yaml `# stack template` \ 14 | 15 | -------------------------------------------------------------------------------- /examples/stack2/templates/AutoscalingGroup.yaml: -------------------------------------------------------------------------------- 1 | Mappings: 2 | Images: !yaml files/images.json 3 | Parameters: 4 | AvailabilityZones: {Type: CommaDelimitedList} 5 | KeyName: {Type: String} 6 | AutoScalingGroupMinSize: {Type: String} 7 | AutoScalingGroupMaxSize: {Type: String} 8 | Image: {Type: String} 9 | InstanceType: {Type: String} 10 | SecurityGroup: {Type: String} # this parameter comes from the cfn_resources datasource. 11 | # that's the logical name of the SecurityGroup resource. 12 | Resources: 13 | AutoScalingGroup: 14 | Type: AWS::AutoScaling::AutoScalingGroup 15 | Properties: 16 | AvailabilityZones: {Ref: AvailabilityZones} 17 | Cooldown: 1200 18 | HealthCheckGracePeriod: 1200 19 | HealthCheckType: EC2 20 | LaunchConfigurationName: {Ref: LaunchConfig} 21 | MinSize: {Ref: AutoScalingGroupMinSize} 22 | MaxSize: {Ref: AutoScalingGroupMaxSize} 23 | LaunchConfig: 24 | Type: AWS::AutoScaling::LaunchConfiguration 25 | Properties: 26 | ImageId: {'Fn::FindInMap': [Images, {Ref: 'AWS::Region'}, {Ref: Image}]} 27 | InstanceType: {Ref: InstanceType} 28 | KeyName: {Ref: KeyName} 29 | SecurityGroups: [{Ref: SecurityGroup}] 30 | UserData: !file64 files/userdata.txt 31 | 32 | -------------------------------------------------------------------------------- /examples/stack2/templates/SecurityGroup.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | SecurityGroup: 3 | Type: AWS::EC2::SecurityGroup 4 | Properties: 5 | GroupDescription: Awesome security group 6 | SecurityGroupIngress: 7 | # Allow SSH from everywhere 8 | - IpProtocol: tcp 9 | FromPort: 22 10 | ToPort: 22 11 | CidrIp: 0.0.0.0/0 12 | 13 | # Allow ICMP destination unreachable for TCP PMTU discovery to work properly 14 | - IpProtocol: icmp 15 | FromPort: 3 16 | ToPort: 4 17 | CidrIp: 0.0.0.0/0 18 | 19 | -------------------------------------------------------------------------------- /examples/stack3/README.md: -------------------------------------------------------------------------------- 1 | EXAMPLE 3 - Big cluster 2 | ======================= 3 | 4 | * `rainbow-stack3-sgs` containing the `SecurityGroup` resources `AllSecurityGroup`, `LoadBalancerSecurityGroup`, `WebSecurityGroup` and `DatabaseSecurityGroup`. Stack is generated from multiple templates being merged (`templates/SecurityGroups/*.yaml`) 5 | * `rainbow-stack3-loadbalancer`, `rainbow-stack3-web`, `rainbow-stack3-database` containing the `AutoscalingGroup` and the `LaunchConfig` resources for each role. All stacks are generated from a single template (`templates/GenericRole.yaml`) with datasource overrides per-role (`parameters/role.yaml`). 6 | 7 | -------------------------------------------------------------------------------- /examples/stack3/files/images.json: -------------------------------------------------------------------------------- 1 | {"us-east-1": {"PreciseAmd64PvEbs": "ami-59a4a230", "PreciseAmd64PvInstanceStore": "ami-c9d7d1a0", "PreciseAmd64HvmEbs": "ami-5fa4a236", "PreciseI386PvInstanceStore": "ami-13cdcb7a", "PreciseAmd64HvmInstanceStore": "ami-a9dddbc0", "PreciseI386PvEbs": "ami-45a4a22c"}, "ap-northeast-1": {"PreciseAmd64PvEbs": "ami-f381f5f2", "PreciseAmd64PvInstanceStore": "ami-31a1d530", "PreciseAmd64HvmEbs": "ami-f581f5f4", "PreciseI386PvInstanceStore": "ami-e1a7d3e0", "PreciseAmd64HvmInstanceStore": "ami-6586f264", "PreciseI386PvEbs": "ami-ef81f5ee"}, "eu-west-1": {"PreciseAmd64PvEbs": "ami-808675f7", "PreciseAmd64PvInstanceStore": "ami-6aab581d", "PreciseAmd64HvmEbs": "ami-8c8675fb", "PreciseI386PvInstanceStore": "ami-3aa1524d", "PreciseAmd64HvmInstanceStore": "ami-0caf5c7b", "PreciseI386PvEbs": "ami-828675f5"}, "ap-southeast-1": {"PreciseAmd64PvEbs": "ami-3c39686e", "PreciseAmd64PvInstanceStore": "ami-8e3061dc", "PreciseAmd64HvmEbs": "ami-22396870", "PreciseI386PvInstanceStore": "ami-14306146", "PreciseAmd64HvmInstanceStore": "ami-56396804", "PreciseI386PvEbs": "ami-3839686a"}, "ap-southeast-2": {"PreciseAmd64PvEbs": "ami-09f26b33", "PreciseAmd64PvInstanceStore": "ami-47ff667d", "PreciseAmd64HvmEbs": "ami-0ff26b35", "PreciseI386PvInstanceStore": "ami-f7fc65cd", "PreciseAmd64HvmInstanceStore": "ami-15f26b2f", "PreciseI386PvEbs": "ami-0bf26b31"}, "us-west-2": {"PreciseAmd64PvEbs": "ami-fa9cf1ca", "PreciseAmd64PvInstanceStore": "ami-84b6dbb4", "PreciseAmd64HvmEbs": "ami-fc9cf1cc", "PreciseI386PvInstanceStore": "ami-30b7da00", "PreciseAmd64HvmInstanceStore": "ami-88b0ddb8", "PreciseI386PvEbs": "ami-f89cf1c8"}, "us-west-1": {"PreciseAmd64PvEbs": "ami-660c3023", "PreciseAmd64PvInstanceStore": "ami-b21f23f7", "PreciseAmd64HvmEbs": "ami-620c3027", "PreciseI386PvInstanceStore": "ami-cc1e2289", "PreciseAmd64HvmInstanceStore": "ami-c0192585", "PreciseI386PvEbs": "ami-640c3021"}, "sa-east-1": {"PreciseAmd64PvEbs": "ami-1da90a00", "PreciseAmd64PvInstanceStore": "ami-3da60520", "PreciseAmd64HvmEbs": "ami-1fa90a02", "PreciseI386PvInstanceStore": "ami-c9a003d4", "PreciseAmd64HvmInstanceStore": "ami-dbaa09c6", "PreciseI386PvEbs": "ami-e3aa09fe"}} 2 | -------------------------------------------------------------------------------- /examples/stack3/files/userdata.txt: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | touch /root/rainbow 3 | 4 | -------------------------------------------------------------------------------- /examples/stack3/parameters/base.yaml: -------------------------------------------------------------------------------- 1 | AvailabilityZones: 2 | - us-east-1a 3 | Image: PreciseAmd64PvInstanceStore # the web and database roles are using c3 instances, which support HVM 4 | # they also override this parameter 5 | KeyName: omrib-laptop 6 | SecurityGroups: 7 | - $AllSecurityGroup # pointer to another key 8 | # AllSecurityGroup is a logical resource created on our security groups stack 9 | - $SecurityGroup # pointer to another key. this one is overriden by the per-role YAML file 10 | 11 | -------------------------------------------------------------------------------- /examples/stack3/parameters/database.yaml: -------------------------------------------------------------------------------- 1 | AutoScalingGroupMaxSize: 1 2 | AutoScalingGroupMinSize: 0 3 | InstanceType: c3.xlarge 4 | SecurityGroup: $DatabaseSecurityGroup # a pointer to a logical resource name on cfn_resources:rainbow-stack3-sgs 5 | Image: PreciseAmd64HvmInstanceStore # HVM instance. use HVM AMI 6 | 7 | -------------------------------------------------------------------------------- /examples/stack3/parameters/loadbalancer.yaml: -------------------------------------------------------------------------------- 1 | AutoScalingGroupMaxSize: 2 2 | AutoScalingGroupMinSize: 0 3 | InstanceType: c3.large 4 | SecurityGroup: $LoadBalancerSecurityGroup # a pointer to a logical resource name on cfn_resources:rainbow-stack3-sgs 5 | Image: PreciseAmd64HvmInstanceStore # HVM instance. use HVM AMI 6 | 7 | -------------------------------------------------------------------------------- /examples/stack3/parameters/web.yaml: -------------------------------------------------------------------------------- 1 | AutoScalingGroupMaxSize: 9 2 | AutoScalingGroupMinSize: 0 3 | InstanceType: m1.small 4 | SecurityGroup: $WebSecurityGroup # a pointer to a logical resource name on cfn_resources:rainbow-stack3-sgs 5 | 6 | -------------------------------------------------------------------------------- /examples/stack3/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | rainbow \ 3 | --block `# block until the stack creation is finished. exits with a non-zero exit code upon failure` \ 4 | rainbow-stack3-sgs `# stack name` \ 5 | templates/SecurityGroups/*.yaml `# stack templates. those are being merged into a single template` 6 | 7 | for ROLE in loadbalancer web database; do `# create a stack for each role: loadbalancer, web, database` 8 | rainbow \ 9 | --block \ 10 | --data-source yaml:parameters/${ROLE}.yaml `# first priority data source, includes overrides for base.yaml` \ 11 | --data-source yaml:parameters/base.yaml `# default values` \ 12 | --data-source cfn_resources:rainbow-stack3-sgs `# security groups` \ 13 | rainbow-stack3-${ROLE} `# stack name is rainbow-stack3-` \ 14 | templates/GenericRole.yaml `# use the same template for all 3 roles` 15 | done 16 | 17 | -------------------------------------------------------------------------------- /examples/stack3/templates/GenericRole.yaml: -------------------------------------------------------------------------------- 1 | Mappings: 2 | Images: !yaml files/images.json 3 | Parameters: 4 | AvailabilityZones: {Type: CommaDelimitedList} 5 | KeyName: {Type: String} 6 | AutoScalingGroupMinSize: {Type: String} 7 | AutoScalingGroupMaxSize: {Type: String} 8 | Image: {Type: String} 9 | InstanceType: {Type: String} 10 | SecurityGroups: {Type: CommaDelimitedList} 11 | Resources: 12 | AutoScalingGroup: 13 | Type: AWS::AutoScaling::AutoScalingGroup 14 | Properties: 15 | AvailabilityZones: {Ref: AvailabilityZones} 16 | Cooldown: 1200 17 | HealthCheckGracePeriod: 1200 18 | HealthCheckType: EC2 19 | LaunchConfigurationName: {Ref: LaunchConfig} 20 | MinSize: {Ref: AutoScalingGroupMinSize} 21 | MaxSize: {Ref: AutoScalingGroupMaxSize} 22 | LaunchConfig: 23 | Type: AWS::AutoScaling::LaunchConfiguration 24 | Properties: 25 | ImageId: {'Fn::FindInMap': [Images, {Ref: 'AWS::Region'}, {Ref: Image}]} 26 | InstanceType: {Ref: InstanceType} 27 | KeyName: {Ref: KeyName} 28 | SecurityGroups: {Ref: SecurityGroups} 29 | UserData: !file64 files/userdata.txt 30 | 31 | -------------------------------------------------------------------------------- /examples/stack3/templates/SecurityGroups/All.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | AllSecurityGroup: 3 | Type: AWS::EC2::SecurityGroup 4 | Properties: 5 | GroupDescription: All security group 6 | AllIcmpEchoIngress: 7 | Type: AWS::EC2::SecurityGroupIngress 8 | Properties: 9 | GroupName: {Ref: AllSecurityGroup} 10 | IpProtocol: icmp 11 | CidrIp: 0.0.0.0/0 12 | FromPort: 8 13 | ToPort: 0 14 | AllIcmpDestinationUnreachableFragmentationNeededIngress: 15 | Type: AWS::EC2::SecurityGroupIngress 16 | Properties: 17 | GroupName: {Ref: AllSecurityGroup} 18 | IpProtocol: icmp 19 | CidrIp: 0.0.0.0/0 20 | FromPort: 3 21 | ToPort: 4 22 | AllSshIngress: 23 | Type: AWS::EC2::SecurityGroupIngress 24 | Properties: 25 | GroupName: {Ref: AllSecurityGroup} 26 | IpProtocol: tcp 27 | CidrIp: 0.0.0.0/0 28 | FromPort: 22 29 | ToPort: 22 30 | 31 | -------------------------------------------------------------------------------- /examples/stack3/templates/SecurityGroups/Database.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | DatabaseSecurityGroup: 3 | Type: AWS::EC2::SecurityGroup 4 | Properties: 5 | GroupDescription: Database security group 6 | DatabaseWebIngress: 7 | Type: AWS::EC2::SecurityGroupIngress 8 | Properties: 9 | GroupName: {Ref: DatabaseSecurityGroup} 10 | IpProtocol: tcp 11 | SourceSecurityGroupName: {Ref: WebSecurityGroup} 12 | FromPort: 3306 13 | ToPort: 3306 14 | 15 | -------------------------------------------------------------------------------- /examples/stack3/templates/SecurityGroups/LoadBalancer.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | LoadBalancerSecurityGroup: 3 | Type: AWS::EC2::SecurityGroup 4 | Properties: 5 | GroupDescription: LoadBalancer security group 6 | LoadBalancerHttpIngress: 7 | Type: AWS::EC2::SecurityGroupIngress 8 | Properties: 9 | GroupName: {Ref: LoadBalancerSecurityGroup} 10 | IpProtocol: tcp 11 | CidrIp: 0.0.0.0/0 12 | FromPort: 80 13 | ToPort: 80 14 | LoadBalancerHttpsIngress: 15 | Type: AWS::EC2::SecurityGroupIngress 16 | Properties: 17 | GroupName: {Ref: LoadBalancerSecurityGroup} 18 | IpProtocol: tcp 19 | CidrIp: 0.0.0.0/0 20 | FromPort: 443 21 | ToPort: 443 22 | 23 | -------------------------------------------------------------------------------- /examples/stack3/templates/SecurityGroups/WebServer.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | WebSecurityGroup: 3 | Type: AWS::EC2::SecurityGroup 4 | Properties: 5 | GroupDescription: Web security group 6 | WebLoadBalancerIngress: 7 | Type: AWS::EC2::SecurityGroupIngress 8 | Properties: 9 | GroupName: {Ref: WebSecurityGroup} 10 | IpProtocol: tcp 11 | SourceSecurityGroupName: {Ref: LoadBalancerSecurityGroup} 12 | FromPort: 80 13 | ToPort: 80 14 | 15 | -------------------------------------------------------------------------------- /rainbow/__init__.py: -------------------------------------------------------------------------------- 1 | pass 2 | -------------------------------------------------------------------------------- /rainbow/cloudformation.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import itertools 4 | import boto.cloudformation 5 | import boto.exception 6 | 7 | 8 | def boto_all(func, *args, **kwargs): 9 | """ 10 | Iterate through all boto next_token's 11 | """ 12 | 13 | ret = [func(*args, **kwargs)] 14 | 15 | while ret[-1].next_token: 16 | kwargs['next_token'] = ret[-1].next_token 17 | ret.append(func(*args, **kwargs)) 18 | 19 | # flatten it by 1 level 20 | return list(reduce(itertools.chain, ret)) 21 | 22 | 23 | class StackStatus(str): 24 | pass 25 | 26 | 27 | class StackSuccessStatus(StackStatus): 28 | pass 29 | 30 | 31 | class StackFailStatus(StackStatus): 32 | pass 33 | 34 | 35 | class CloudformationException(Exception): 36 | pass 37 | 38 | 39 | class Cloudformation(object): 40 | # this is from http://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_Stack.html 41 | # boto.cloudformation.stack.StackEvent.valid_states doesn't have the full list. 42 | VALID_STACK_STATUSES = ['CREATE_IN_PROGRESS', 'CREATE_FAILED', 'CREATE_COMPLETE', 'ROLLBACK_IN_PROGRESS', 43 | 'ROLLBACK_FAILED', 'ROLLBACK_COMPLETE', 'DELETE_IN_PROGRESS', 'DELETE_FAILED', 44 | 'DELETE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS', 45 | 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_IN_PROGRESS', 'UPDATE_ROLLBACK_FAILED', 46 | 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS', 'UPDATE_ROLLBACK_COMPLETE'] 47 | 48 | default_region = 'us-east-1' 49 | 50 | def __init__(self, region=None): 51 | """ 52 | :param region: AWS region 53 | :type region: str 54 | """ 55 | 56 | self.connection = boto.cloudformation.connect_to_region(region or Cloudformation.default_region) 57 | 58 | if not self.connection: 59 | raise CloudformationException('Invalid region %s' % (region or Cloudformation.default_region,)) 60 | 61 | @staticmethod 62 | def resolve_template_parameters(template, datasource_collection): 63 | """ 64 | Resolve all template parameters from datasource_collection, return a dictionary of parameters 65 | to pass to update_stack() or create_stack() methods 66 | 67 | :type template: dict 68 | :type datasource_collection: DataSourceCollection 69 | :rtype: dict 70 | :return: parameters parameter for update_stack() or create_stack() 71 | """ 72 | 73 | parameters = {} 74 | for parameter, parameter_definition in template.get('Parameters', {}).iteritems(): 75 | if 'Default' in parameter_definition and not parameter in datasource_collection: 76 | parameter_value = parameter_definition['Default'] 77 | else: 78 | parameter_value = datasource_collection.get_parameter_recursive(parameter) 79 | 80 | if hasattr(parameter_value, '__iter__'): 81 | parameter_value = ','.join(map(str, parameter_value)) 82 | 83 | parameters[parameter] = parameter_value 84 | 85 | return parameters 86 | 87 | def stack_exists(self, name): 88 | """ 89 | Check if a CFN stack exists 90 | 91 | :param name: stack name 92 | :return: True/False 93 | :rtype: bool 94 | """ 95 | 96 | # conserve bandwidth (and API calls) by not listing any stacks in DELETE_COMPLETE state 97 | active_stacks = boto_all(self.connection.list_stacks, [state for state in Cloudformation.VALID_STACK_STATUSES 98 | if state != 'DELETE_COMPLETE']) 99 | return name in [stack.stack_name for stack in active_stacks if stack.stack_status] 100 | 101 | def update_stack(self, name, template, parameters): 102 | """ 103 | Update CFN stack 104 | 105 | :param name: stack name 106 | :type name: str 107 | :param template: JSON encodeable object 108 | :type template: str 109 | :param parameters: dictionary containing key value pairs as CFN parameters 110 | :type parameters: dict 111 | :rtype: bool 112 | :return: False if there aren't any updates to be performed, True if no exception has been thrown. 113 | """ 114 | 115 | try: 116 | self.connection.update_stack(name, json.dumps(template), disable_rollback=True, 117 | parameters=parameters.items(), capabilities=['CAPABILITY_IAM']) 118 | except boto.exception.BotoServerError, ex: 119 | if ex.message == 'No updates are to be performed.': 120 | # this is not really an error, but there aren't any updates. 121 | return False 122 | else: 123 | raise CloudformationException('error occured while updating stack %s: %s' % (name, ex.message)) 124 | else: 125 | return True 126 | 127 | def create_stack(self, name, template, parameters): 128 | """ 129 | Create CFN stack 130 | 131 | :param name: stack name 132 | :type name: str 133 | :param template: JSON encodeable object 134 | :type template: str 135 | :param parameters: dictionary containing key value pairs as CFN parameters 136 | :type parameters: dict 137 | """ 138 | 139 | try: 140 | self.connection.create_stack(name, json.dumps(template), disable_rollback=True, 141 | parameters=parameters.items(), capabilities=['CAPABILITY_IAM']) 142 | except boto.exception.BotoServerError, ex: 143 | raise CloudformationException('error occured while creating stack %s: %s' % (name, ex.message)) 144 | 145 | def describe_stack_events(self, name): 146 | """ 147 | Describe CFN stack events 148 | 149 | :param name: stack name 150 | :type name: str 151 | :return: stack events 152 | :rtype: list of boto.cloudformation.stack.StackEvent 153 | """ 154 | 155 | return boto_all(self.connection.describe_stack_events, name) 156 | 157 | def describe_stack(self, name): 158 | """ 159 | Describe CFN stack 160 | 161 | :param name: stack name 162 | :return: stack object 163 | :rtype: boto.cloudformation.stack.Stack 164 | """ 165 | 166 | return self.connection.describe_stacks(name)[0] 167 | 168 | def tail_stack_events(self, name, initial_entry=None): 169 | """ 170 | This function is a wrapper around _tail_stack_events(), because a generator function doesn't run any code 171 | before the first iterator item is accessed (aka .next() is called). 172 | This function can be called without an `inital_entry` and tail the stack events from the bottom. 173 | 174 | Each iteration returns either: 175 | 1. StackFailStatus object which indicates the stack creation/update failed (last iteration) 176 | 2. StackSuccessStatus object which indicates the stack creation/update succeeded (last iteration) 177 | 3. dictionary describing the stack event, containing the following keys: resource_type, logical_resource_id, 178 | physical_resource_id, resource_status, resource_status_reason 179 | 180 | A common usage pattern would be to call tail_stack_events('stack') prior to running update_stack() on it, 181 | thus creating the iterator prior to the actual beginning of the update. Then, after initiating the update 182 | process, for loop through the iterator receiving the generated events and status updates. 183 | 184 | :param name: stack name 185 | :type name: str 186 | :param initial_entry: where to start tailing from. None means to start from the last item (exclusive) 187 | :type initial_entry: None or int 188 | :return: generator object yielding stack events 189 | :rtype: generator 190 | """ 191 | 192 | if initial_entry is None: 193 | return self._tail_stack_events(name, len(self.describe_stack_events(name))) 194 | elif initial_entry < 0: 195 | return self._tail_stack_events(name, len(self.describe_stack_events(name)) + initial_entry) 196 | else: 197 | return self._tail_stack_events(name, initial_entry) 198 | 199 | def _tail_stack_events(self, name, initial_entry): 200 | """ 201 | See tail_stack_events() 202 | """ 203 | 204 | previous_stack_events = initial_entry 205 | 206 | while True: 207 | stack = self.describe_stack(name) 208 | stack_events = self.describe_stack_events(name) 209 | 210 | if len(stack_events) > previous_stack_events: 211 | # iterate on all new events, at reversed order (the list is sorted from newest to oldest) 212 | for event in stack_events[:-previous_stack_events or None][::-1]: 213 | yield {'resource_type': event.resource_type, 214 | 'logical_resource_id': event.logical_resource_id, 215 | 'physical_resource_id': event.physical_resource_id, 216 | 'resource_status': event.resource_status, 217 | 'resource_status_reason': event.resource_status_reason, 218 | 'timestamp': event.timestamp} 219 | 220 | previous_stack_events = len(stack_events) 221 | 222 | if stack.stack_status.endswith('_FAILED') or \ 223 | stack.stack_status in ('ROLLBACK_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE'): 224 | yield StackFailStatus(stack.stack_status) 225 | break 226 | elif stack.stack_status.endswith('_COMPLETE'): 227 | yield StackSuccessStatus(stack.stack_status) 228 | break 229 | 230 | time.sleep(2) 231 | -------------------------------------------------------------------------------- /rainbow/datasources/__init__.py: -------------------------------------------------------------------------------- 1 | from base import DataSourceCollection, DataSourceBase 2 | # we have to import these modules for them to register as valid data sources 3 | import cfn_datasource 4 | import yaml_datasource 5 | import file_datasource 6 | 7 | __all__ = ['DataSourceBase', 'DataSourceCollection'] 8 | -------------------------------------------------------------------------------- /rainbow/datasources/base.py: -------------------------------------------------------------------------------- 1 | from datasource_exceptions import * 2 | 3 | 4 | class DataCollectionPointer(str): 5 | def __repr__(self): 6 | return '<%s %s>' % (self.__class__.__name__, str.__repr__(self)) 7 | 8 | 9 | class DataSourceBaseMeta(type): 10 | datasources = {} 11 | 12 | @classmethod 13 | def __new__(mcs, cls, name, bases, d): 14 | ret = type.__new__(cls, name, bases, d) 15 | if hasattr(ret, 'datasource_name'): 16 | mcs.datasources[ret.datasource_name] = ret 17 | return ret 18 | 19 | 20 | class DataSourceBase(object): 21 | __metaclass__ = DataSourceBaseMeta 22 | 23 | # will be initialized to argparse.ArgumentParser command line args 24 | region = 'us-east-1' 25 | 26 | def __init__(self, data_source): 27 | self.data_source = data_source 28 | self.data = None 29 | 30 | def __getitem__(self, item): 31 | return self.data[item] 32 | 33 | def __contains__(self, item): 34 | return item in self.data 35 | 36 | def __repr__(self): 37 | return '<%s data_source=%r data=%r>' % (self.__class__.__name__, self.data_source, self.data) 38 | 39 | 40 | class DataSourceCollection(list): 41 | def __init__(self, datasources): 42 | """ 43 | :param datasources: list of strings containing data sources. i.e.: yaml:path/to/yaml.yaml 44 | :type datasources: list 45 | """ 46 | 47 | l = [] 48 | 49 | for datasource in datasources: 50 | try: 51 | source, data = datasource.split(":", 1) 52 | except: 53 | raise InvalidDataSourceFormatException( 54 | "Invalid data source format %r. Data source should be \":\"" % (datasource,)) 55 | 56 | if not source in DataSourceBaseMeta.datasources: 57 | raise UnknownDataSourceException( 58 | "Unknown data source %s, valid data sources are %s" % 59 | (source, ", ".join(DataSourceBaseMeta.datasources.keys()))) 60 | 61 | l.append(DataSourceBaseMeta.datasources[source](data)) 62 | 63 | super(DataSourceCollection, self).__init__(l) 64 | 65 | def get_parameter_recursive(self, parameter): 66 | """ 67 | See `get_parameter()` doc. 68 | The difference between the two functions is that this function follows pointers. 69 | 70 | :param parameter: parameter to look up 71 | :type parameter: str 72 | :return: `parameter` resolved (recursively) 73 | """ 74 | 75 | parameter = self.get_parameter(parameter) 76 | 77 | if isinstance(parameter, DataCollectionPointer): 78 | # pointer, resolve it 79 | 80 | return self.get_parameter_recursive(parameter) 81 | elif hasattr(parameter, '__iter__'): 82 | # resolve iterables and convert to a list 83 | 84 | return [self.get_parameter_recursive(i) if isinstance(i, DataCollectionPointer) else i for i in parameter] 85 | else: 86 | # regular parameter, return as is. 87 | 88 | return parameter 89 | 90 | def get_parameter(self, parameter): 91 | """ 92 | Look up `parameter` in all data sources available to the collection, returning the first match. 93 | This function doesn't follow pointers. You probably want `get_parameter_recursive()` 94 | 95 | :param parameter: parameter to look up 96 | :type parameter: str 97 | :return: `parameter` resolved 98 | """ 99 | 100 | for data_source in self: 101 | if parameter in data_source: 102 | return data_source[parameter] 103 | else: 104 | raise InvalidParameterException( 105 | "Unable to find parameter %s in any of the data sources %r" % (parameter, self)) 106 | 107 | def __contains__(self, item): 108 | try: 109 | self.get_parameter_recursive(item) 110 | return True 111 | except InvalidParameterException: 112 | return False 113 | -------------------------------------------------------------------------------- /rainbow/datasources/cfn_datasource.py: -------------------------------------------------------------------------------- 1 | from rainbow.cloudformation import Cloudformation 2 | from base import DataSourceBase 3 | 4 | __all__ = ['CfnOutputsDataSource', 'CfnResourcesDataSource', 'CfnParametersDataSource'] 5 | 6 | 7 | class CfnDataSourceBase(DataSourceBase): 8 | def __init__(self, data_source): 9 | super(CfnDataSourceBase, self).__init__(data_source) 10 | 11 | stack_name = data_source 12 | region = Cloudformation.default_region 13 | 14 | if ':' in data_source: 15 | region, stack_name = data_source.split(':', 1) 16 | 17 | cfn_connection = Cloudformation(region) 18 | if not cfn_connection: 19 | raise Exception('Invalid region %r' % (region,)) 20 | 21 | self.stack = cfn_connection.describe_stack(stack_name) 22 | 23 | 24 | class CfnOutputsDataSource(CfnDataSourceBase): 25 | datasource_name = 'cfn_outputs' 26 | 27 | def __init__(self, data_source): 28 | super(CfnOutputsDataSource, self).__init__(data_source) 29 | 30 | self.data = {i.key: i.value for i in self.stack.outputs} 31 | 32 | 33 | class CfnResourcesDataSource(CfnDataSourceBase): 34 | datasource_name = 'cfn_resources' 35 | 36 | def __init__(self, data_source): 37 | super(CfnResourcesDataSource, self).__init__(data_source) 38 | 39 | self.data = {r.logical_resource_id: r.physical_resource_id for r in self.stack.describe_resources()} 40 | 41 | 42 | class CfnParametersDataSource(CfnDataSourceBase): 43 | datasource_name = 'cfn_parameters' 44 | 45 | def __init__(self, data_source): 46 | super(CfnParametersDataSource, self).__init__(data_source) 47 | 48 | self.data = {p.key: p.value for p in self.stack.parameters} 49 | 50 | -------------------------------------------------------------------------------- /rainbow/datasources/datasource_exceptions.py: -------------------------------------------------------------------------------- 1 | class DataSourceBaseException(Exception): 2 | pass 3 | 4 | 5 | class InvalidDataSourceFormatException(DataSourceBaseException): 6 | pass 7 | 8 | 9 | class UnknownDataSourceException(DataSourceBaseException): 10 | pass 11 | 12 | 13 | class InvalidParameterException(DataSourceBaseException): 14 | pass 15 | -------------------------------------------------------------------------------- /rainbow/datasources/file_datasource.py: -------------------------------------------------------------------------------- 1 | from base import DataSourceBase 2 | from datasource_exceptions import InvalidDataSourceFormatException 3 | 4 | __all__ = ['FileDataSource', 'File64DataSource'] 5 | 6 | 7 | class FileDataSource(DataSourceBase): 8 | datasource_name = 'file' 9 | 10 | def __init__(self, data_source): 11 | super(FileDataSource, self).__init__(data_source) 12 | 13 | if not ':' in data_source: 14 | raise InvalidDataSourceFormatException("FileDataSource must be in name:path_to_file format") 15 | 16 | name, path = data_source.split(':', 1) 17 | 18 | with open(path) as f: 19 | self.data = {name: f.read(-1)} 20 | 21 | 22 | class File64DataSource(DataSourceBase): 23 | datasource_name = 'file64' 24 | 25 | def __init__(self, data_source): 26 | super(File64DataSource, self).__init__(data_source) 27 | 28 | if not ':' in data_source: 29 | raise InvalidDataSourceFormatException("File64DataSource must be in name:path_to_file format") 30 | 31 | name, path = data_source.split(':', 1) 32 | 33 | with open(path) as f: 34 | self.data = {name: f.read(-1).encode('base64')} 35 | -------------------------------------------------------------------------------- /rainbow/datasources/yaml_datasource.py: -------------------------------------------------------------------------------- 1 | from base import DataSourceBase 2 | from rainbow.yaml_loader import RainbowYamlLoader 3 | 4 | __all__ = ['YamlDataSource'] 5 | 6 | 7 | class YamlDataSource(DataSourceBase): 8 | datasource_name = 'yaml' 9 | 10 | def __init__(self, data_source): 11 | # data_source can be path/to/file.yaml or Key:path/to/file.yaml 12 | # this lets you use one yaml file for multiple purposes 13 | if ':' in data_source: 14 | key, yaml_file = data_source.split(':', 1) 15 | else: 16 | yaml_file = data_source 17 | key = None 18 | 19 | super(YamlDataSource, self).__init__(data_source) 20 | 21 | with open(yaml_file) as f: 22 | self.data = RainbowYamlLoader(f).get_data() 23 | 24 | if key: 25 | self.data = self.data[key] 26 | -------------------------------------------------------------------------------- /rainbow/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import pprint 4 | import sys 5 | import yaml 6 | import logging 7 | from rainbow.datasources import DataSourceCollection 8 | from rainbow.preprocessor import Preprocessor 9 | from rainbow.templates import TemplateLoader 10 | from rainbow.cloudformation import Cloudformation, StackFailStatus, StackSuccessStatus 11 | 12 | 13 | def main(): # pragma: no cover 14 | logging.basicConfig(level=logging.INFO) 15 | 16 | # boto logs errors in addition to throwing exceptions. on rainbow.cloudformation.Cloudformation.update_stack() 17 | # I'm ignoring the 'No updates are to be performed.' exception, so I don't want it to be logged. 18 | logging.getLogger('boto').setLevel(logging.CRITICAL) 19 | 20 | logger = logging.getLogger('rainbow') 21 | 22 | parser = argparse.ArgumentParser(description='Load cloudformation templates with cool data sources as arguments') 23 | parser.add_argument('-d', '--data-source', metavar='DATASOURCE', dest='datasources', action='append', default=[], 24 | help='Data source. Format is data_sourcetype:data_sourceargument. For example, ' + 25 | 'cfn_outputs:[region:]stackname, cfn_resources:[region:]stackname, or ' + 26 | 'yaml:yamlfile. First match is used') 27 | parser.add_argument('-r', '--region', default='us-east-1', help='AWS region') 28 | parser.add_argument('-n', '--noop', action='store_true', 29 | help="Don't actually call aws; just show what would be done.") 30 | parser.add_argument('-v', '--verbose', action='store_true') 31 | parser.add_argument('--dump-datasources', action='store_true', 32 | help='Simply output all datasources and their values') 33 | parser.add_argument('--update-stack', action='store_true', 34 | help='Update a pre-existing stack rather than create a new one') 35 | parser.add_argument('--update-stack-if-exists', action='store_true', 36 | help='Create a new stack if it doesn\'t exist, update if it does') 37 | parser.add_argument('--block', action='store_true', 38 | help='Track stack creation, if the stack creation failed, exits with a non-zero exit code') 39 | 40 | parser.add_argument('stack_name') 41 | parser.add_argument('templates', metavar='template', type=str, nargs='+') 42 | 43 | args = parser.parse_args() 44 | if args.verbose: 45 | logger.setLevel(logging.DEBUG) 46 | 47 | Cloudformation.default_region = args.region 48 | datasource_collection = DataSourceCollection(args.datasources) 49 | 50 | # load and merge templates 51 | template = TemplateLoader.load_templates(args.templates) 52 | 53 | # preprocess computed values 54 | preprocessor = Preprocessor(datasource_collection=datasource_collection, region=args.region) 55 | template = preprocessor.process(template) 56 | 57 | # build list of parameters for stack creation/update from datasources 58 | parameters = Cloudformation.resolve_template_parameters(template, datasource_collection) 59 | 60 | if args.dump_datasources: 61 | pprint.pprint(datasource_collection) 62 | return 63 | 64 | logger.debug('Will create stack "%s" with parameters: %r', args.stack_name, parameters) 65 | logger.debug('Template:\n%s', yaml.dump(template)) 66 | 67 | if args.noop: 68 | logger.info('NOOP mode. exiting') 69 | return 70 | 71 | cloudformation = Cloudformation(args.region) 72 | 73 | if args.update_stack_if_exists: 74 | if cloudformation.stack_exists(args.stack_name): 75 | args.update_stack = True 76 | else: 77 | args.update_stack = False 78 | 79 | if args.block: 80 | # set the iterator prior to updating the stack, so it'll begin from the current bottom 81 | stack_events_iterator = cloudformation.tail_stack_events(args.stack_name, None if args.update_stack else 0) 82 | 83 | if args.update_stack: 84 | stack_modified = cloudformation.update_stack(args.stack_name, template, parameters) 85 | if not stack_modified: 86 | logger.info('No updates to be performed') 87 | else: 88 | cloudformation.create_stack(args.stack_name, template, parameters) 89 | stack_modified = True 90 | 91 | if args.block and stack_modified: 92 | for event in stack_events_iterator: 93 | if isinstance(event, StackFailStatus): 94 | logger.warn('Stack creation failed: %s', event) 95 | sys.exit(1) 96 | elif isinstance(event, StackSuccessStatus): 97 | logger.info('Stack creation succeeded: %s', event) 98 | else: 99 | logger.info('%(resource_type)s %(logical_resource_id)s %(physical_resource_id)s %(resource_status)s ' 100 | '%(resource_status_reason)s', event) 101 | 102 | if __name__ == '__main__': # pragma: no cover 103 | main() 104 | -------------------------------------------------------------------------------- /rainbow/preprocessor/__init__.py: -------------------------------------------------------------------------------- 1 | from base import Preprocessor 2 | import instance_chooser 3 | 4 | __all__ = ['Preprocessor'] 5 | 6 | -------------------------------------------------------------------------------- /rainbow/preprocessor/base.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from preprocessor_exceptions import * 3 | 4 | 5 | class PreprocessorBase(object): 6 | functions = {} 7 | 8 | @classmethod 9 | def expose(cls, name): 10 | """ 11 | Decorator function to expose a function as a preprocessor function 12 | 13 | Example usage: 14 | @PreprocessorBase.expose('MyAwesomeFunction') 15 | def myfunc(preprocessor, parameter): 16 | return '[%s]' % (parameter,) 17 | 18 | 19 | Given the following template (YAML): 20 | key: value 21 | key2: {'Rb::MyAwesomeFunction': 'a great value'} 22 | 23 | The preprocessor will return: 24 | {'key': 'value', 25 | 'key2': '[a great value]'} 26 | """ 27 | 28 | def decorator(f): 29 | cls.functions[name] = f 30 | return f 31 | 32 | return decorator 33 | 34 | 35 | class Preprocessor(object): 36 | def __init__(self, datasource_collection, region): 37 | self.datasource_collection = datasource_collection 38 | self.region = region 39 | 40 | def process(self, template): 41 | """ 42 | Go through template, look for {'Rb::FunctionName': } dictionaries, calling the Rainbow function 43 | FunctionName to process them. 44 | 45 | :type template: dict 46 | :param template: input dictionary 47 | :return: a copy of the template dictionary with all the Rb:: function calls processed 48 | """ 49 | 50 | template = copy.deepcopy(template) 51 | 52 | if isinstance(template, dict): 53 | if len(template) == 1 and type(template.keys()[0]) is str and template.keys()[0].startswith('Rb::'): 54 | k, v = template.items()[0] 55 | if k.startswith('Rb::'): 56 | function = k[4:] 57 | if not function in PreprocessorBase.functions: 58 | raise InvalidPreprocessorFunctionException( 59 | 'Rainbow Function (Rb::) %s not found in %r' % (function, 60 | PreprocessorBase.functions.keys(),)) 61 | else: 62 | return PreprocessorBase.functions[function](self, v) 63 | else: 64 | for k, v in template.iteritems(): 65 | template[k] = self.process(v) 66 | 67 | return template 68 | -------------------------------------------------------------------------------- /rainbow/preprocessor/instance_chooser.py: -------------------------------------------------------------------------------- 1 | from base import PreprocessorBase 2 | from preprocessor_exceptions import PreprocessorBaseException 3 | from rainbow.datasources.base import DataCollectionPointer 4 | 5 | 6 | regions_instances = { 7 | 'us-east-1': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 'cg1.4xlarge', 't1.micro', 'cr1.8xlarge', 8 | 'c3.2xlarge', 'c3.xlarge', 'm1.large', 'hs1.8xlarge', 'c3.8xlarge', 'c3.4xlarge', 'hi1.4xlarge', 9 | 'i2.8xlarge', 'c1.xlarge', 'm2.2xlarge', 'g2.2xlarge', 'm2.xlarge', 'm1.medium', 'i2.xlarge', 10 | 'm3.medium', 'cc2.8xlarge', 'i2.2xlarge', 'c3.large', 'i2.4xlarge', 'm1.xlarge', 'm2.4xlarge', 11 | 'm3.xlarge'], 12 | 'ap-northeast-1': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 't1.micro', 'cr1.8xlarge', 'c3.2xlarge', 13 | 'c3.xlarge', 'm1.large', 'hs1.8xlarge', 'c3.8xlarge', 'c3.4xlarge', 'hi1.4xlarge', 'i2.8xlarge', 14 | 'c1.xlarge', 'm2.2xlarge', 'g2.2xlarge', 'm2.xlarge', 'm1.medium', 'i2.xlarge', 'm3.medium', 15 | 'cc2.8xlarge', 'i2.2xlarge', 'c3.large', 'i2.4xlarge', 'm1.xlarge', 'm2.4xlarge', 'm3.xlarge'], 16 | 'eu-west-1': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 'cg1.4xlarge', 't1.micro', 'cr1.8xlarge', 17 | 'c3.2xlarge', 'c3.xlarge', 'm1.large', 'hs1.8xlarge', 'c3.8xlarge', 'c3.4xlarge', 'hi1.4xlarge', 18 | 'i2.8xlarge', 'c1.xlarge', 'm2.2xlarge', 'g2.2xlarge', 'm2.xlarge', 'm1.medium', 'i2.xlarge', 19 | 'm3.medium', 'cc2.8xlarge', 'i2.2xlarge', 'c3.large', 'i2.4xlarge', 'm1.xlarge', 'm2.4xlarge', 20 | 'm3.xlarge'], 21 | 'ap-southeast-1': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 't1.micro', 'c3.2xlarge', 'c3.xlarge', 22 | 'm1.large', 'hs1.8xlarge', 'c3.8xlarge', 'c3.4xlarge', 'i2.8xlarge', 'c1.xlarge', 'm2.2xlarge', 23 | 'm2.xlarge', 'm1.medium', 'i2.xlarge', 'm3.medium', 'i2.2xlarge', 'c3.large', 'i2.4xlarge', 24 | 'm1.xlarge', 'm2.4xlarge', 'm3.xlarge'], 25 | 'ap-southeast-2': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 't1.micro', 'c3.2xlarge', 'c3.xlarge', 26 | 'm1.large', 'hs1.8xlarge', 'c3.8xlarge', 'c3.4xlarge', 'i2.8xlarge', 'c1.xlarge', 'm2.2xlarge', 27 | 'm2.xlarge', 'm1.medium', 'i2.xlarge', 'm3.medium', 'i2.2xlarge', 'c3.large', 'i2.4xlarge', 28 | 'm1.xlarge', 'm2.4xlarge', 'm3.xlarge'], 29 | 'us-west-2': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 't1.micro', 'cr1.8xlarge', 'c3.2xlarge', 30 | 'c3.xlarge', 'm1.large', 'hs1.8xlarge', 'c3.8xlarge', 'c3.4xlarge', 'hi1.4xlarge', 'i2.8xlarge', 31 | 'c1.xlarge', 'm2.2xlarge', 'g2.2xlarge', 'm2.xlarge', 'm1.medium', 'i2.xlarge', 'm3.medium', 32 | 'cc2.8xlarge', 'i2.2xlarge', 'c3.large', 'i2.4xlarge', 'm1.xlarge', 'm2.4xlarge', 'm3.xlarge'], 33 | 'us-west-1': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 't1.micro', 'c3.2xlarge', 'c3.xlarge', 'm1.large', 34 | 'c3.8xlarge', 'c3.4xlarge', 'i2.8xlarge', 'c1.xlarge', 'm2.2xlarge', 'g2.2xlarge', 'm2.xlarge', 35 | 'm1.medium', 'i2.xlarge', 'm3.medium', 'i2.2xlarge', 'c3.large', 'i2.4xlarge', 'm1.xlarge', 36 | 'm2.4xlarge', 'm3.xlarge'], 37 | 'sa-east-1': ['m3.large', 'm3.2xlarge', 'm1.small', 'c1.medium', 't1.micro', 'm1.large', 'c1.xlarge', 'm2.2xlarge', 38 | 'm2.xlarge', 'm1.medium', 'm3.medium', 'm1.xlarge', 'm2.4xlarge', 'm3.xlarge'] 39 | } 40 | 41 | 42 | class InvalidInstanceException(PreprocessorBaseException): 43 | pass 44 | 45 | 46 | @PreprocessorBase.expose('InstanceChooser') 47 | def instance_chooser(preprocessor, instance_types): 48 | """ 49 | Rb::InstanceChooser 50 | Choose the first valid instance (for the current region). 51 | For example, if a certain region doesn't support the c3 instances family, you can specify 'c3.large' 52 | with a fallback to 'c1.medium'. The function returns the first instance type that's available on that region. 53 | Example usage: 54 | {'Rb::InstanceChooser': ['c3.large', 'c1.medium']} 55 | On a region that supports c3.large, 'c3.large' will be returned 56 | On a region that doesn't, 'c1.medium' will be returned 57 | 58 | :param preprocessor: Preprocessor instance processing the function 59 | :type preprocessor: Preprocessor 60 | :param instance_types: list of instance types to choose from 61 | :type instance_types: list 62 | :rtype: str 63 | """ 64 | 65 | if isinstance(instance_types, DataCollectionPointer): 66 | instance_types = preprocessor.datasource_collection.get_parameter_recursive(instance_types) 67 | 68 | if not hasattr(instance_types, '__iter__'): 69 | raise InvalidInstanceException('Instance types should be an iterable (a list)') 70 | 71 | # resolve pointers if relevant 72 | for i, v in enumerate(instance_types): 73 | if isinstance(v, DataCollectionPointer): 74 | instance_types[i] = preprocessor.datasource_collection.get_parameter_recursive(str(v)) 75 | 76 | available_instance_types = [instance_type for instance_type in instance_types 77 | if instance_type in regions_instances[preprocessor.region]] 78 | 79 | if not available_instance_types: 80 | raise InvalidInstanceException( 81 | "Unable to find a suitable instance type for region %s out of %r. Available instances: %r" % 82 | (preprocessor.region, instance_types, regions_instances[preprocessor.region])) 83 | else: 84 | return available_instance_types[0] 85 | -------------------------------------------------------------------------------- /rainbow/preprocessor/preprocessor_exceptions.py: -------------------------------------------------------------------------------- 1 | class PreprocessorBaseException(Exception): 2 | pass 3 | 4 | 5 | class InvalidPreprocessorFunctionException(PreprocessorBaseException): 6 | pass 7 | -------------------------------------------------------------------------------- /rainbow/templates.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from rainbow.yaml_loader import RainbowYamlLoader 3 | 4 | 5 | def is_cfn_magic(d): 6 | """ 7 | Given dict `d`, determine if it uses CFN magic (Fn:: functions or Ref) 8 | This function is used for deep merging CFN templates, we don't treat CFN magic as a regular dictionary 9 | for merging purposes. 10 | 11 | :rtype: bool 12 | :param d: dictionary to check 13 | :return: true if the dictionary uses CFN magic, false if not 14 | """ 15 | 16 | if len(d) != 1: 17 | return False 18 | 19 | k = d.keys()[0] 20 | 21 | if k == 'Ref' or k.startswith('Fn::') or k.startswith('Rb::'): 22 | return True 23 | 24 | return False 25 | 26 | 27 | def cfn_deep_merge(a, b): 28 | """ 29 | Deep merge two CFN templates, treating CFN magics (see `is_cfn_magic` for more information) as non-mergeable 30 | Prefers b over a 31 | 32 | :rtype: dict 33 | :param a: first dictionary 34 | :param b: second dictionary (overrides a) 35 | :return: a new dictionary which is a merge of a and b 36 | """ 37 | 38 | # if a and b are dictionaries and both of them aren't cfn magic, merge them 39 | if isinstance(a, dict) and isinstance(b, dict) and not (is_cfn_magic(a) or is_cfn_magic(b)): 40 | # we're modifying and returning a, so start off with a copy 41 | a = copy.deepcopy(a) 42 | 43 | # merge two dictionaries 44 | for k in b: 45 | if k in a: 46 | a[k] = cfn_deep_merge(a[k], b[k]) 47 | else: 48 | a[k] = copy.deepcopy(b[k]) 49 | 50 | return a 51 | else: 52 | return copy.deepcopy(b) 53 | 54 | 55 | class TemplateLoader(object): 56 | @staticmethod 57 | def load_templates(templates): 58 | """ 59 | Load & merge templates 60 | 61 | :param templates: list of template paths (strings) 62 | :type templates: list 63 | :return: merged template 64 | :rtype: dict 65 | """ 66 | 67 | template = {} 68 | for template_path in templates: 69 | with open(template_path) as f: 70 | template = cfn_deep_merge(template, RainbowYamlLoader(f).get_data()) 71 | 72 | return template 73 | -------------------------------------------------------------------------------- /rainbow/yaml_loader.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import re 3 | from rainbow.datasources.base import DataCollectionPointer 4 | 5 | 6 | class RainbowYamlLoader(yaml.Loader): 7 | @staticmethod 8 | def yaml_pointer(loader, node): 9 | """ 10 | DataSource pointer, wraps the string in a DataCollectionPointer() object 11 | 12 | :param loader: loader.add_constructor parameter 13 | :param node: loader.add_constructor parameter 14 | :rtype: DataCollectionPointer 15 | :return: File content as string 16 | """ 17 | 18 | value = loader.construct_scalar(node) 19 | 20 | # remove implicit resolver character 21 | if value[0] == '$': 22 | value = value[1:] 23 | 24 | return DataCollectionPointer(value) 25 | 26 | @staticmethod 27 | def yaml_file(loader, node): 28 | """ 29 | Load file content as string to YAML 30 | 31 | :param loader: loader.add_constructor parameter 32 | :param node: loader.add_constructor parameter 33 | :rtype: str 34 | :return: File content as string 35 | """ 36 | 37 | value = loader.construct_scalar(node) 38 | with open(value) as f: 39 | return f.read() 40 | 41 | @classmethod 42 | def yaml_file64(cls, loader, node): 43 | """ 44 | Same as yaml_file, but returns base64 encoded data 45 | 46 | :param loader: loader.add_constructor parameter 47 | :param node: loader.add_constructor parameter 48 | :rtype: str 49 | :return: File content as base64 encoded string 50 | """ 51 | 52 | return cls.yaml_file(loader, node).encode('base64') 53 | 54 | @staticmethod 55 | def yaml_yaml(loader, node): 56 | """ 57 | Yo dawg, we heard you like YAML... 58 | Same as yaml_file, but returns a yaml decoded data 59 | 60 | :param loader: loader.add_constructor parameter 61 | :param node: loader.add_constructor parameter 62 | :return: File content as base64 encoded string 63 | """ 64 | template_path = loader.construct_scalar(node) 65 | 66 | if ':' in template_path: 67 | key, yaml_file = template_path.split(':', 1) 68 | else: 69 | yaml_file = template_path 70 | key = None 71 | 72 | with open(yaml_file) as f: 73 | template = RainbowYamlLoader(f).get_data() 74 | 75 | if key: 76 | template = template[key] 77 | return template 78 | 79 | 80 | def __init__(self, *args, **kwargs): 81 | self.add_constructor('!file', self.__class__.yaml_file) 82 | self.add_constructor('!file64', self.__class__.yaml_file64) 83 | self.add_constructor('!yaml', self.__class__.yaml_yaml) 84 | self.add_constructor('!pointer', self.__class__.yaml_pointer) 85 | self.add_implicit_resolver('!pointer', re.compile(r'^\$\S+'), None) 86 | 87 | super(RainbowYamlLoader, self).__init__(*args, **kwargs) 88 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='rainbow-cfn', 5 | version='0.5', 6 | description='Rainbow is AWS Cloudformation on steroids', 7 | author='EverythingMe', 8 | author_email='omrib@everything.me', 9 | url='http://github.com/EverythingMe/rainbow', 10 | packages=find_packages(), 11 | install_requires=['boto', 'PyYAML'], 12 | 13 | extras_require={ 14 | 'test': ['nose', 'coverage', 'mock'] 15 | }, 16 | 17 | entry_points={ 18 | 'console_scripts': [ 19 | 'rainbow = rainbow.main:main' 20 | ] 21 | }, 22 | 23 | classifiers=[ 24 | 'Development Status :: 5 - Production/Stable', 25 | 'Environment :: Console', 26 | 'Intended Audience :: Developers', 27 | 'Intended Audience :: System Administrators', 28 | 'Operating System :: OS Independent', 29 | 'Programming Language :: Python :: 2.7', 30 | 'Topic :: System :: Clustering', 31 | 'Topic :: System :: Systems Administration', 32 | 'Topic :: Utilities' 33 | ] 34 | ) 35 | 36 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EverythingMe/rainbow/17aad61231b1f1b9d0dca43979e2fa4c8a1603f3/tests/__init__.py -------------------------------------------------------------------------------- /tests/cfn_deep_merge/a.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | AutoScalingGroup: 3 | Properties: 4 | AvailabilityZones: [{Ref: AvailabilityZone}] 5 | Cooldown: 1200 6 | DesiredCapacity: 0 7 | HealthCheckGracePeriod: 1200 8 | HealthCheckType: EC2 9 | LaunchConfigurationName: {Ref: LaunchConfig} 10 | MaxSize: 0 11 | MinSize: 0 12 | NotificationConfiguration: 13 | NotificationTypes: 14 | - autoscaling:EC2_INSTANCE_LAUNCH 15 | - autoscaling:EC2_INSTANCE_LAUNCH_ERROR 16 | - autoscaling:EC2_INSTANCE_TERMINATE 17 | - autoscaling:EC2_INSTANCE_TERMINATE_ERROR 18 | TopicARN: {Ref: AutoscalingNotificationsTopic} 19 | Tags: 20 | - Key: chef:environment 21 | PropagateAtLaunch: 'true' 22 | Value: {Ref: ChefEnvironment} 23 | - Key: chef:role 24 | PropagateAtLaunch: 'true' 25 | Value: {Ref: ChefRole} 26 | - Key: Name 27 | PropagateAtLaunch: 'true' 28 | Value: {'Fn::Join': ['-', [{Ref: ChefEnvironment}, {Ref: ChefRole}]]} 29 | Type: AWS::AutoScaling::AutoScalingGroup 30 | LaunchConfig: 31 | Properties: 32 | ImageId: {'Fn::FindInMap': [Images, {Ref: 'AWS::Region'}, {Ref: 'AmiType'}]} 33 | InstanceType: {Ref: InstanceType} 34 | KeyName: {Ref: BootstrapKeyName} 35 | SecurityGroups: {Ref: SecurityGroups} 36 | UserData: {'Fn::Base64': {'Fn::Join': ['', [ 37 | {'Fn::FindInMap': [UserData, CloudInit, Head]}, "\n", 38 | ' CFN_STACK_ID=', {Ref: 'AWS::StackId'}, "\n", 39 | " CFN_RESOURCE=AutoScalingGroup\n", 40 | ' ENVIRONMENT=', {Ref: ChefEnvironment}, "\n", 41 | ' ROLE=', {Ref: ChefRole}, "\n", 42 | ' DOMAIN=', {Ref: DomainSuffix}, "\n", 43 | ' CHEF_SERVER_URL=', {Ref: ChefServerUrl}, "\n", 44 | ' CHEF_SERVER_VALIDATION_PEM=', {'Fn::Base64': {Ref: ChefValidationPem}}, "\n", 45 | {'Fn::FindInMap': [UserData, CloudInit, Body]}, "\n"]]}} 46 | Type: AWS::AutoScaling::LaunchConfiguration 47 | 48 | -------------------------------------------------------------------------------- /tests/cfn_deep_merge/b.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | AutoScalingGroup: 3 | Properties: 4 | NewPropertyScalar: 100 5 | NewPropertyList: [1,2,3] 6 | NewPropertyDict: 7 | a: b 8 | c: d 9 | e: f 10 | AvailabilityZones: [{Ref: NewAvailabilityZone}] 11 | Cooldown: 300 12 | DesiredCapacity: 10 13 | HealthCheckGracePeriod: 300 14 | HealthCheckType: DifferentHealthCheck 15 | LaunchConfigurationName: {Ref: LaunchConfigush} 16 | NotificationConfiguration: 17 | NotificationTypes: 18 | - autoscaling:EC2_INSTANCE_LAUNCH 19 | - autoscaling:EC2_INSTANCE_TERMINATE 20 | Tags: 21 | - Key: chef:role 22 | PropagateAtLaunch: 'true' 23 | Value: {Ref: ChefRole} 24 | - Key: Name 25 | PropagateAtLaunch: 'true' 26 | Value: {'Fn::Join': ['-', [{Ref: ChefEnvironment}, {Ref: ChefRole}]]} 27 | LaunchConfig: 28 | Properties: 29 | UserData: {'Fn::Base64': {'Fn::Join': ['', [ 30 | {'Fn::FindInMap': [UserData, CloudInit, Head]}, "\n", 31 | ' ENVIRONMENT=', {Ref: ChefEnvironment}, "\n", 32 | ' ROLE=', {Ref: ChefRole}, "\n", 33 | ' DOMAIN=', {Ref: DomainSuffix}, "\n", 34 | ' CHEF_SERVER_URL=', {Ref: ChefServerUrl}, "\n", 35 | ' CHEF_SERVER_VALIDATION_PEM=', {'Fn::Base64': {Ref: ChefValidationPem}}, "\n", 36 | {'Fn::FindInMap': [UserData, CloudInit, Body]}, "\n"]]}} 37 | 38 | -------------------------------------------------------------------------------- /tests/cfn_deep_merge/c.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | AutoScalingGroup: 3 | Properties: 4 | NewPropertyScalar: 100 5 | NewPropertyList: [1,2,3] 6 | NewPropertyDict: 7 | a: b 8 | c: d 9 | e: f 10 | AvailabilityZones: [{Ref: NewAvailabilityZone}] 11 | Cooldown: 300 12 | DesiredCapacity: 10 13 | HealthCheckGracePeriod: 300 14 | HealthCheckType: DifferentHealthCheck 15 | LaunchConfigurationName: {Ref: LaunchConfigush} 16 | MaxSize: 0 17 | MinSize: 0 18 | NotificationConfiguration: 19 | NotificationTypes: 20 | - autoscaling:EC2_INSTANCE_LAUNCH 21 | - autoscaling:EC2_INSTANCE_TERMINATE 22 | TopicARN: {Ref: AutoscalingNotificationsTopic} 23 | Tags: 24 | - Key: chef:role 25 | PropagateAtLaunch: 'true' 26 | Value: {Ref: ChefRole} 27 | - Key: Name 28 | PropagateAtLaunch: 'true' 29 | Value: {'Fn::Join': ['-', [{Ref: ChefEnvironment}, {Ref: ChefRole}]]} 30 | Type: AWS::AutoScaling::AutoScalingGroup 31 | LaunchConfig: 32 | Properties: 33 | ImageId: {'Fn::FindInMap': [Images, {Ref: 'AWS::Region'}, {Ref: 'AmiType'}]} 34 | InstanceType: {Ref: InstanceType} 35 | KeyName: {Ref: BootstrapKeyName} 36 | SecurityGroups: {Ref: SecurityGroups} 37 | UserData: {'Fn::Base64': {'Fn::Join': ['', [ 38 | {'Fn::FindInMap': [UserData, CloudInit, Head]}, "\n", 39 | ' ENVIRONMENT=', {Ref: ChefEnvironment}, "\n", 40 | ' ROLE=', {Ref: ChefRole}, "\n", 41 | ' DOMAIN=', {Ref: DomainSuffix}, "\n", 42 | ' CHEF_SERVER_URL=', {Ref: ChefServerUrl}, "\n", 43 | ' CHEF_SERVER_VALIDATION_PEM=', {'Fn::Base64': {Ref: ChefValidationPem}}, "\n", 44 | {'Fn::FindInMap': [UserData, CloudInit, Body]}, "\n"]]}} 45 | Type: AWS::AutoScaling::LaunchConfiguration 46 | 47 | -------------------------------------------------------------------------------- /tests/datasources/a.yaml: -------------------------------------------------------------------------------- 1 | a_ptr: $b_list 2 | a_list: 3 | - item1 4 | - item2 5 | - item3 6 | - $item4 7 | item4: $item5 8 | item5: $item6 9 | item6: $item7 10 | item7: item4 11 | a_str: foobar 12 | shared: from a 13 | c1large: c1.large 14 | PossibleInstances: 15 | - c3.large 16 | - c1.medium -------------------------------------------------------------------------------- /tests/datasources/b.yaml: -------------------------------------------------------------------------------- 1 | b_ptr: $a_list 2 | b_str: a B string 3 | b_list: [1,2,3] 4 | shared: from b 5 | -------------------------------------------------------------------------------- /tests/datasources/c.yaml: -------------------------------------------------------------------------------- 1 | c_ptr: $b_ptr 2 | shared: from c -------------------------------------------------------------------------------- /tests/datasources/d.file: -------------------------------------------------------------------------------- 1 | d file contents -------------------------------------------------------------------------------- /tests/datasources/e.file64: -------------------------------------------------------------------------------- 1 | e file contents -------------------------------------------------------------------------------- /tests/datasources/nested.yaml: -------------------------------------------------------------------------------- 1 | a: !yaml datasources/a.yaml 2 | b: !yaml datasources/b.yaml 3 | c: !yaml datasources/c.yaml 4 | -------------------------------------------------------------------------------- /tests/preprocessor/instance_chooser1.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | Type: AWS::EC2::Instance 3 | Properties: 4 | InstanceType: {'Rb::InstanceChooser': [$c1large, 'c1.xlarge']} 5 | -------------------------------------------------------------------------------- /tests/preprocessor/instance_chooser2.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | Type: AWS::EC2::Instance 3 | Properties: 4 | InstanceType: {'Rb::InstanceChooser': ['c3.large']} 5 | -------------------------------------------------------------------------------- /tests/preprocessor/instance_chooser3.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | Type: AWS::EC2::Instance 3 | Properties: 4 | InstanceType: {'Rb::InstanceChooser': ['nosuchinstance']} 5 | -------------------------------------------------------------------------------- /tests/preprocessor/instance_chooser4.yaml: -------------------------------------------------------------------------------- 1 | Resources: 2 | Type: AWS::EC2::Instance 3 | Properties: 4 | InstanceType: {'Rb::InstanceChooser': $PossibleInstances} 5 | -------------------------------------------------------------------------------- /tests/templates/default_parameters_template.yaml: -------------------------------------------------------------------------------- 1 | Parameters: 2 | DefaultString: {Type: String, Default: "default string value"} 3 | DefaultCommaDelimitedList: {Type: CommaDelimitedList, Default: "default, comma, delimited, list"} 4 | -------------------------------------------------------------------------------- /tests/templates/simpletemplate.yaml: -------------------------------------------------------------------------------- 1 | Parameters: 2 | a_list: {Type: CommaDelimitedList} 3 | b_list: {Type: CommaDelimitedList} 4 | Resources: 5 | MyLovelyResource: 6 | Properties: 7 | MyProperty: {'Ref': 'a_list'} 8 | AnotherProperty: value2 9 | Type: AWS::Dummy::DummyResource 10 | -------------------------------------------------------------------------------- /tests/test_cfn_datasource.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import functools 3 | from unittest import TestCase 4 | from rainbow.datasources import DataSourceCollection 5 | from rainbow.cloudformation import Cloudformation 6 | 7 | 8 | __author__ = 'omrib' 9 | 10 | 11 | def mock_boto_cloudformation_connect_to_region(region, stacks): 12 | return MockCloudformationConnection(region, stacks) 13 | 14 | 15 | class DotDict(dict): 16 | def __getattr__(self, item): 17 | return self[item] 18 | 19 | def __setattr__(self, key, value): 20 | self[key] = value 21 | 22 | 23 | class MockCloudformationConnection(object): 24 | def __init__(self, region, stacks): 25 | """ 26 | :type region: str 27 | :type stacks: dict 28 | """ 29 | 30 | self.region = region 31 | self.stacks = stacks 32 | 33 | # noinspection PyUnusedLocal 34 | def describe_stacks(self, stack_name_or_id=None, next_token=None): 35 | if stack_name_or_id: 36 | return [self.stacks[self.region][stack_name_or_id]] 37 | else: 38 | return self.stacks[self.region].values() 39 | 40 | 41 | class MockCloudformationStack(object): 42 | def __init__(self, resources={}, outputs={}, parameters={}): 43 | self._resources = resources 44 | self._outputs = outputs 45 | self._parameters = parameters 46 | 47 | def describe_resources(self): 48 | return [DotDict(logical_resource_id=key, physical_resource_id=value) 49 | for key, value in self._resources.iteritems()] 50 | 51 | @property 52 | def outputs(self): 53 | return [DotDict(key=key, value=value) 54 | for key, value in self._outputs.iteritems()] 55 | 56 | @property 57 | def parameters(self): 58 | return [DotDict(key=key, value=value) 59 | for key, value in self._parameters.iteritems()] 60 | 61 | class TestDataSources(TestCase): 62 | def setUp(self): 63 | Cloudformation.default_region = 'us-east-1' 64 | 65 | # c can override b, which can override a 66 | data_sources = ['cfn_resources:us-stack1', 67 | 'cfn_outputs:us-stack2', 68 | 'cfn_parameters:us-stack3', 69 | 'cfn_resources:eu-west-1:eu-stack1', 70 | 'cfn_outputs:eu-west-1:eu-stack2', 71 | 'cfn_parameters:eu-west-1:eu-stack3'] 72 | 73 | stacks = { 74 | 'us-east-1': { 75 | 'us-stack1': MockCloudformationStack(resources={'UsResource1': 'us-stack1-UsResource1-ABCDEFGH'}), 76 | 'us-stack2': MockCloudformationStack(outputs={'UsOutput1': 'US output'}), 77 | 'us-stack3': MockCloudformationStack(parameters={'UsParameter1': 'US parameter'}) 78 | }, 79 | 'eu-west-1': { 80 | 'eu-stack1': MockCloudformationStack(resources={'EuResource1': 'eu-stack1-EuResource1-ABCDEFGH'}), 81 | 'eu-stack2': MockCloudformationStack(outputs={'EuOutput1': 'EU output'}), 82 | 'eu-stack3': MockCloudformationStack(parameters={'EuParameter1': 'EU parameter'}) 83 | } 84 | } 85 | 86 | with mock.patch('boto.cloudformation.connect_to_region', 87 | functools.partial(mock_boto_cloudformation_connect_to_region, stacks=stacks)): 88 | self.datasource_collection = DataSourceCollection(data_sources) 89 | 90 | def test_cfn_resources_implicit_region(self): 91 | self.assertEqual(self.datasource_collection.get_parameter_recursive('UsResource1'), 'us-stack1-UsResource1-ABCDEFGH') 92 | 93 | def test_cfn_outputs_implicit_region(self): 94 | self.assertEqual(self.datasource_collection.get_parameter_recursive('UsOutput1'), 'US output') 95 | 96 | def test_cfn_parameters_implicit_region(self): 97 | self.assertEqual(self.datasource_collection.get_parameter_recursive('UsParameter1'), 'US parameter') 98 | 99 | def test_cfn_resources_ecplicit_region(self): 100 | self.assertEqual(self.datasource_collection.get_parameter_recursive('EuResource1'), 'eu-stack1-EuResource1-ABCDEFGH') 101 | 102 | def test_cfn_outputs_explicit_region(self): 103 | self.assertEqual(self.datasource_collection.get_parameter_recursive('EuOutput1'), 'EU output') 104 | 105 | def test_cfn_parameters_explicit_region(self): 106 | self.assertEqual(self.datasource_collection.get_parameter_recursive('EuParameter1'), 'EU parameter') 107 | -------------------------------------------------------------------------------- /tests/test_cfn_deep_merge.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from rainbow.yaml_loader import RainbowYamlLoader 3 | from rainbow.templates import cfn_deep_merge 4 | 5 | __author__ = 'omrib' 6 | 7 | 8 | class TestCfnDeepMerge(TestCase): 9 | def setUp(self): 10 | with open('cfn_deep_merge/a.yaml') as a, \ 11 | open('cfn_deep_merge/b.yaml') as b, \ 12 | open('cfn_deep_merge/c.yaml') as c: 13 | self.a = RainbowYamlLoader(a).get_data() 14 | self.b = RainbowYamlLoader(b).get_data() 15 | self.c = RainbowYamlLoader(c).get_data() 16 | 17 | def test_cfn_deep_merge(self): 18 | a_b_merged = cfn_deep_merge(self.a, self.b) 19 | self.assertDictEqual(self.c, a_b_merged) 20 | -------------------------------------------------------------------------------- /tests/test_cloudformation.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from rainbow.datasources import DataSourceCollection 3 | from rainbow.preprocessor import Preprocessor 4 | from rainbow.preprocessor.preprocessor_exceptions import InvalidPreprocessorFunctionException 5 | from rainbow.preprocessor.instance_chooser import InvalidInstanceException 6 | from rainbow.yaml_loader import RainbowYamlLoader 7 | from rainbow.cloudformation import Cloudformation 8 | from rainbow.templates import TemplateLoader 9 | 10 | __author__ = 'omrib' 11 | 12 | 13 | class TestCloudformation(TestCase): 14 | def setUp(self): 15 | data_sources = ['yaml:c:datasources/nested.yaml', 16 | 'yaml:datasources/b.yaml', 17 | 'yaml:datasources/a.yaml'] 18 | 19 | self.datasource_collection = DataSourceCollection(data_sources) 20 | self.preprocessor = Preprocessor(self.datasource_collection, 'us-east-1') 21 | 22 | def test_resolve_template_parameters(self): 23 | template = TemplateLoader.load_templates(['templates/simpletemplate.yaml']) 24 | parameters = Cloudformation.resolve_template_parameters(template, self.datasource_collection) 25 | 26 | self.assertEqual(parameters['a_list'], 'item1,item2,item3,item4') 27 | self.assertEqual(parameters['b_list'], '1,2,3') 28 | 29 | def test_invalid_function(self): 30 | self.assertRaises(InvalidPreprocessorFunctionException, self.preprocessor.process, {'Rb::NoSuchFunction': ''}) 31 | 32 | def test_resolve_template_default_parameter(self): 33 | template = TemplateLoader.load_templates(['templates/default_parameters_template.yaml']) 34 | parameters = Cloudformation.resolve_template_parameters(template, self.datasource_collection) 35 | 36 | self.assertEqual(parameters['DefaultString'], 'default string value') 37 | self.assertEqual(parameters['DefaultCommaDelimitedList'], 'default, comma, delimited, list') 38 | -------------------------------------------------------------------------------- /tests/test_datasources.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from rainbow.datasources import * 3 | from rainbow.datasources.datasource_exceptions import * 4 | 5 | __author__ = 'omrib' 6 | 7 | 8 | class TestDataSources(TestCase): 9 | def setUp(self): 10 | # c can override b, which can override a 11 | data_sources = ['file64:e_str:datasources/e.file64', 12 | 'file:d_str:datasources/d.file', 13 | 'yaml:c:datasources/nested.yaml', 14 | 'yaml:datasources/b.yaml', 15 | 'yaml:datasources/a.yaml'] 16 | 17 | self.datasource_collection = DataSourceCollection(data_sources) 18 | 19 | def test_non_exist_parameter(self): 20 | self.assertRaises(InvalidParameterException, self.datasource_collection.get_parameter_recursive, 'test') 21 | 22 | def test_pointer(self): 23 | self.assertEqual( 24 | self.datasource_collection.get_parameter_recursive('a_ptr'), 25 | self.datasource_collection.get_parameter_recursive('b_list') 26 | ) 27 | 28 | self.assertEqual( 29 | self.datasource_collection.get_parameter_recursive('b_ptr'), 30 | self.datasource_collection.get_parameter_recursive('a_list') 31 | ) 32 | 33 | self.assertEqual( 34 | self.datasource_collection.get_parameter_recursive('c_ptr'), 35 | self.datasource_collection.get_parameter_recursive('a_list') 36 | ) 37 | 38 | def test_string(self): 39 | self.assertEqual( 40 | self.datasource_collection.get_parameter_recursive('a_str'), 41 | 'foobar' 42 | ) 43 | 44 | self.assertEqual( 45 | self.datasource_collection.get_parameter_recursive('b_str'), 46 | 'a B string' 47 | ) 48 | 49 | with open('datasources/d.file') as f: 50 | self.assertEqual( 51 | self.datasource_collection.get_parameter_recursive('d_str'), 52 | f.read() 53 | ) 54 | 55 | with open('datasources/e.file64') as f: 56 | self.assertEqual( 57 | self.datasource_collection.get_parameter_recursive('e_str'), 58 | f.read().encode('base64') 59 | ) 60 | 61 | def test_list(self): 62 | self.assertListEqual( 63 | self.datasource_collection.get_parameter_recursive('a_list'), 64 | ['item1', 'item2', 'item3', 'item4'] 65 | ) 66 | 67 | self.assertListEqual( 68 | self.datasource_collection.get_parameter_recursive('b_list'), 69 | [1, 2, 3] 70 | ) 71 | 72 | def test_override(self): 73 | self.assertEqual( 74 | self.datasource_collection.get_parameter_recursive('shared'), 75 | 'from c' 76 | ) 77 | -------------------------------------------------------------------------------- /tests/test_preprocessor.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from rainbow.datasources import DataSourceCollection 3 | from rainbow.preprocessor import Preprocessor 4 | from rainbow.preprocessor.preprocessor_exceptions import InvalidPreprocessorFunctionException 5 | from rainbow.preprocessor.instance_chooser import InvalidInstanceException 6 | from rainbow.yaml_loader import RainbowYamlLoader 7 | 8 | __author__ = 'omrib' 9 | 10 | 11 | class TestPreprocessor(TestCase): 12 | def setUp(self): 13 | # c can override b, which can override a 14 | data_sources = ['yaml:c:datasources/nested.yaml', 15 | 'yaml:datasources/b.yaml', 16 | 'yaml:datasources/a.yaml'] 17 | 18 | self.datasource_collection = DataSourceCollection(data_sources) 19 | self.preprocessor = Preprocessor(self.datasource_collection, 'us-east-1') 20 | 21 | def test_instance_chooser(self): 22 | with open('preprocessor/instance_chooser1.yaml') as instance_chooser1, \ 23 | open('preprocessor/instance_chooser2.yaml') as instance_chooser2, \ 24 | open('preprocessor/instance_chooser3.yaml') as instance_chooser3, \ 25 | open('preprocessor/instance_chooser4.yaml') as instance_chooser4: 26 | template = RainbowYamlLoader(instance_chooser1).get_data() 27 | template2 = RainbowYamlLoader(instance_chooser2).get_data() 28 | template3 = RainbowYamlLoader(instance_chooser3).get_data() 29 | template4 = RainbowYamlLoader(instance_chooser4).get_data() 30 | 31 | processed = self.preprocessor.process(template) 32 | self.assertEqual(processed['Resources']['Properties']['InstanceType'], 'c1.xlarge') 33 | 34 | processed2 = self.preprocessor.process(template2) 35 | self.assertEqual(processed2['Resources']['Properties']['InstanceType'], 'c3.large') 36 | 37 | self.assertRaises(InvalidInstanceException, self.preprocessor.process, template3) 38 | 39 | processed4 = self.preprocessor.process(template4) 40 | self.assertEqual(processed4['Resources']['Properties']['InstanceType'], 'c3.large') 41 | 42 | def test_invalid_function(self): 43 | self.assertRaises(InvalidPreprocessorFunctionException, self.preprocessor.process, {'Rb::NoSuchFunction': ''}) 44 | -------------------------------------------------------------------------------- /tests/test_yamlfile.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from rainbow.yaml_loader import RainbowYamlLoader 3 | 4 | __author__ = 'omrib' 5 | 6 | 7 | class TestPreprocessor(TestCase): 8 | def setUp(self): 9 | with open('yamlfile/base.yaml') as f: 10 | self.yamlfile = RainbowYamlLoader(f).get_data() 11 | 12 | def test_yaml_file(self): 13 | with open('yamlfile/includeme.file') as f: 14 | self.assertEqual( 15 | self.yamlfile['included_file'], 16 | f.read() 17 | ) 18 | 19 | def test_yaml_file64(self): 20 | with open('yamlfile/includeme.file64') as f: 21 | self.assertEqual( 22 | self.yamlfile['included_file64'], 23 | f.read().encode('base64') 24 | ) 25 | 26 | def test_yaml_yaml(self): 27 | with open('yamlfile/includeme.yaml') as f: 28 | self.assertEqual( 29 | self.yamlfile['included_yaml'], 30 | RainbowYamlLoader(f).get_data() 31 | ) 32 | -------------------------------------------------------------------------------- /tests/yamlfile/base.yaml: -------------------------------------------------------------------------------- 1 | included_file: !file yamlfile/includeme.file 2 | included_file64: !file64 yamlfile/includeme.file64 3 | included_yaml: !yaml yamlfile/includeme.yaml 4 | -------------------------------------------------------------------------------- /tests/yamlfile/includeme.file: -------------------------------------------------------------------------------- 1 | this is includeme.file content -------------------------------------------------------------------------------- /tests/yamlfile/includeme.file64: -------------------------------------------------------------------------------- 1 | this is includeme.file64 content -------------------------------------------------------------------------------- /tests/yamlfile/includeme.yaml: -------------------------------------------------------------------------------- 1 | included_yaml: 2 | key: value --------------------------------------------------------------------------------