├── .gitignore ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── ansible ├── index.html ├── playbook.yml └── roles │ ├── appserver │ └── tasks │ │ └── main.yml │ ├── common │ └── tasks │ │ └── main.yml │ ├── dbserver │ └── tasks │ │ └── main.yml │ └── webserver │ └── tasks │ └── main.yml ├── ci_ansible_inventory ├── generate_inventory_file.sh ├── lambda ├── bootstrap.py ├── main.py ├── runcommand_helper.py ├── test_bootstrap.py ├── test_main.py └── test_runcommand_helper.py ├── lambda_bootstrap_payload.zip ├── lambda_main_payload.zip ├── lambda_runcommand_helper_payload.zip ├── requirements.txt ├── solano.yml └── terraform ├── asg ├── main.tf └── variables.tf ├── iam └── main.tf ├── lambda ├── bootstrap.tf ├── main.tf └── runcommand_helper.tf ├── lc ├── main.tf └── variables.tf ├── main.tf ├── variables.tf └── vpc ├── main.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | # pytest 2 | **.pyc 3 | **__pycache__ 4 | **.cache 5 | .coverage 6 | terraform.tfstate 7 | terraform.tfstate.backup 8 | .terraform 9 | terraform.tfvars 10 | **.DS_STORE 11 | NOTES.md 12 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Lambda Runcommand Configuration Management 2 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What's That Smell? 2 | This project, affectionately referred to as GARLC, is a demonstration of combining multiple [AWS](https://aws.amazon.com/) services. 3 | 4 | Project GARLC is made up of: 5 | * [Git](https://git-scm.com/) – for configuration storage and version control. GitHub is used for the project currently but you could also use [AWS CodeCommit](https://aws.amazon.com/codecommit/). 6 | * [Ansible](https://www.ansible.com/) – for configuration management. Chef, Puppet, or Salt using their respective “masterless” or “solo” modes could also be used. 7 | * [Amazon EC2 Run Command](https://aws.amazon.com/ec2/run-command/) – for executing Ansible without requiring SSH access to the instances. 8 | * [AWS Lambda](https://aws.amazon.com/lambda/) – for executing logic and invoking RunCommand. 9 | * [AWS CodePipeline](https://aws.amazon.com/codepipeline/) – for receiving changes from git and triggering Lambda. 10 | 11 | The general idea is that configuration management is now done in the same way we do continuous delivery of applications today. What makes GARLC really exciting though is that there are no central control/orchestration servers to maintain and we no longer need SSH access to the instances to configure them. There are two modes to Project GARLC: continuous and bootstrap. 12 | 13 | ## Continuous Mode 14 | In this mode, configuration management of instances is done automatically, using the above technologies, as configurations are committed to version control. Ansible is an agentless automation and management tool driven by the use of YAML based Playbooks. The idea is that you store idempotent Ansible Playbooks in GitHub and when changes are merged into the master branch CodePipeline picks them up. 15 | 16 | CodePipeline is a continuous delivery service that goes through a series of stages. In the case of GARLC we use two stages: a Source stage that checks GitHub for changes and an Invoke stage that triggers an AWS Lambda function. This AWS Lambda function does several things, including invoking Run Command. 17 | 18 | Run Command is a service that allows you to easily execute scripts or commands on an EC2 Instance. It requires an agent on each instance and can execute commands with administrative privilege on the system. To speed things up, we recommend creating an AMI that already has the Run Command agent installed (the demo uses a public AMI with the agent pre-installed as well as Ansible). For GARLC, we use Run Command to execute Ansible, in local mode, on each instance. Ansible will configure each instance (e.g. install software, modify configuration files, etc) according to a set of roles that are associated with each instance (set via [Tags](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html)). This allows for instances to be automatically updated in a continuous delivery style manner without requiring direct access to each instance. 19 | 20 | ## Bootstrap Mode 21 | Bootstrap mode is done similarly to continuous mode except it includes the addition of [Amazon CloudWatch Events](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/WhatIsCloudWatchEvents.html) and is performed outside the CodePipeline flow. CloudWatch Events lets us define a Rule that takes action when something happens. For GARLC, we create a Rule that invokes an AWS Lambda function any time any instance enters the “Running” state. Like above, the Lambda function will trigger Run Command on the instance with commands to run Ansible locally and configure the host. The latest artifact in the S3 bucket from continuous mode will be used for configuration. The two modes together cover new instances starting up and changes to existing instances. 22 | 23 | # Setup Instructions for a Demo 24 | **WARNING: Using this code may cost you money. Please be sure you understand your current usage and the costs associated with this reference code before launching in your AWS account.** 25 | 26 | 1. Fork this repo. 27 | * Install [Terraform](https://www.terraform.io/downloads.html) to help setup the infrastructure required for GARLC. 28 | * Manually create a [CodePipeline using the AWS Console](http://docs.aws.amazon.com/codepipeline/latest/userguide/how-to-create-pipelines.html) with a single stage for now: 29 | * Source stage should fetch from your fork of this repo on the master branch. 30 | * Output should be "MyApp". 31 | * NOTE: When using the AWS Console to create your Pipeline you will be forced to add a "Beta" stage which you can later delete and replace with the Invoke stage. Just add whatever to get through the wizard. 32 | * From the parent directory of your fork run the below to setup the infrastructure. This will create IAM Roles, the Lambda function, and a couple Auto-Scaling Groups in us-west-2: 33 | 1. `terraform get terraform` 34 | * `terraform plan terraform` 35 | * `terraform apply terraform` 36 | * Go back to CodePipeline and add a second stage for an "Invoke" action on the "garlic" Lambda function created with Terraform in the previous step. 37 | * Input should be "MyApp". 38 | * Update something in the repository (e.g. add something to an Ansible playbook) and then commit to the master branch and watch your changes flow and your instances update automagically :fire:. 39 | 40 | # Additional Information 41 | 42 | ## More Details on AWS Lambda 43 | There are two AWS Lambda functions in use with the continuous mode. The [first Lambda function](https://github.com/awslabs/lambda-runcommand-configuration-management/blob/master/lambda/main.py) is basically a worker that puts all the pieces together so Run Command can do the heavy lifting. The Lambda function does several things: 44 | 45 | 1. The Lambda function will find all EC2 instances with a tag of “has_ssm_agent” and a value of “true” or “True”. This is used to find instances that have the SSM agent (Run Command) installed and are configured with the proper [Instance Profile](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ssm-iam.html) that allows SSM to be run. When you provision instances you should make sure each one gets this tag. 46 | 2. It breaks the list of instances from Step 1 up into smaller chunks that will be processed by a second Lambda function I will talk about later. This is done in order to scale the solution and stay under rate limits for the Run Command service. 47 | 3. It parses the incoming event from CodePipeline which contains metadata about the location of the artifact. The artifact is simply the content of the git repository zipped up and stored in [Amazon S3](https://aws.amazon.com/s3/). The instances will fetch this later to execute the Ansible Playbook(s). 48 | 4. The Lambda function will then build a list of commands that will be sent with Run Command. 49 | * The first command connects to S3 to retrieve the artifact mentioned in step 2 above. 50 | * The next command unpacks the zip in the /tmp directory on the instance. 51 | * The next command runs a shell script to build an Ansible Inventory file locally. More on this in the next section. 52 | * The final command runs ansible-playbook on the instance to configure it. 53 | 5. The last part of the Lambda Function is an API call to invoke a second Lambda function which I will detail next. 54 | 55 | The [second Lambda function](https://github.com/awslabs/lambda-runcommand-configuration-management/blob/master/lambda/runcommand_helper.py) is responsible for invoking Run Command via an API call. This Lambda expects to be passed in a list of Instance ID’s broken down into chunks (a list of lists) and a list of commands to be sent to the instance. The Lambda function will process one chunk, pop that chunk off the list, and then invoke a new instance of the same Lambda function to pick up the work. The reason for doing this is it ensures we always stay under the Run Command API limits and we never have to worry about hitting the max timeout for an AWS Lambda function; we can infinitely scale the solution to however many instances we have. 56 | 57 | In bootstrap mode there is only a [single AWS Lambda function](https://github.com/awslabs/lambda-runcommand-configuration-management/blob/master/lambda/bootstrap.py) that essentially combines the behavior of the two Lambda functions in continuous mode. There are a few differences, though: 58 | * We are provided a single Instance ID per CloudWatch Event trigger, so we do not need to find Instances. 59 | * We have to determine if the instance CloudWatch Event’s sent is actually an instance we should try to perform Run Command on (e.g. does it have the has_ssm_agent tag?). 60 | * We have to find the latest artifact from the CodePipeline bucket. This artifact will be the one retrieved by the instance to configure itself. 61 | 62 | This Lambda function deals with all of these and also includes retry logic in case the Run Command API limits have been exceeded. 63 | 64 | ## Ansible for Configuration Management 65 | If you need a primer on Ansible I highly recommend [this blog post](https://serversforhackers.com/an-ansible-tutorial). Understanding Roles in Ansible is critical in fully recognizing how GARLC coordinates configuration of instances. 66 | 67 | Ansible can be run locally and doesn’t require any sort of centralized master and this was important to get the “serverless” checkbox ticked. To leverage Ansible with GARLC we need a few things in place: an “Ansible_Roles” tag for each instance, playbook roles in Ansible equivalent to the roles defined in the Ansible_Roles tag, and a local Ansible inventory file on each instance. 68 | 69 | When provisioning your instances, each one should get a tag named “Ansible_Roles” with a value that contains a comma separated list of role names. For example, an instance might have an Ansible_Roles tag with the value “webserver, appserver, memcached” which would presumably indicate the instance should get the necessary software to be a web server that hosts an application along with some caching. Using this example, we would then need at least three roles defined in our Ansible playbooks and committed to GitHub to build the instance. These three roles should be named just like above: webserver, appserver, and memcached. 70 | 71 | To tie this together a [script](https://github.com/awslabs/lambda-runcommand-configuration-management/blob/master/generate_inventory_file.sh) is run that builds a local Ansible inventory file describing which roles should be applied to the instance. This inventory file is generated based on the Roles tag, mentioned above, and its values. The script also sets the connection mode to [local](http://docs.ansible.com/ansible/intro_inventory.html) which bypasses an attempted SSH connection. When ansible-playbook is run it will look for this local inventory file (/tmp/inventory) and then execute any playbooks associated with the roles. If the Roles of an instance change, next run a new inventory file be generated and the instance will get updated appropriately. 72 | 73 | ## Performance and Shortcomings 74 | In bootstrap mode a new instance takes about 1.5 seconds for the Lambda function and Run Command then usually configures the instance within a minute or two unless there are many jobs queued up. With the continuous mode, it takes about 15 seconds for the first Lambda to handle 1,000 instances and about 1 second for each Run Command helper Lambda function to process a chunk. A complete run for 1,000 instances takes about 5 minutes to process and then additional time for Run Command to complete (roughly 10 more minutes to get through all 1,000 in testing). 75 | 76 | There is currently no reporting or visualization of Run Command jobs making it difficult to see what failed and when. The Run Command console can be used, but when working in the hundreds or thousands of instance this quickly becomes unwieldy. 77 | 78 | The example provided in this repository currently only supports Linux hosts that are capable of running the [SSM agent](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/install-ssm-agent.html). However, Windows is a supported Operating System for both Ansible and Run Command. 79 | 80 | Instances require Internet access in order for Run Command to work. 81 | 82 | # Theme Song 83 | No ssh here, 84 | sudo git me a beer, 85 | while I Ansible all the things. 86 | 87 | Swaggin with RunCommand, 88 | Lambda don’t move a hand, 89 | flowin through the CodePipeline. 90 | 91 | Smells like GARLC, GARLC, GARLC… 92 | Smells like GARLC, GARLC, GARLC… 93 | Smells like GARLC, GARLC, GARLC… 94 | 95 | # License 96 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 97 | 98 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 99 | 100 | http://aws.amazon.com/apache2.0/ 101 | 102 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 103 | 104 | Note: Other license terms may apply to certain, identified software files contained within or distributed with the accompanying software if such terms are included in the directory containing the accompanying software. Such other license terms will then apply in lieu of the terms of the software license above. 105 | -------------------------------------------------------------------------------- /ansible/index.html: -------------------------------------------------------------------------------- 1 | 2 | GARLC 3 | 4 | Smells like GARLC 5 | 6 | 7 | -------------------------------------------------------------------------------- /ansible/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Main playbook 3 | 4 | - hosts: all 5 | gather_facts: yes 6 | roles: 7 | - common 8 | 9 | - hosts: webserver 10 | roles: 11 | - webserver 12 | 13 | - hosts: appserver 14 | roles: 15 | - appserver 16 | 17 | - hosts: dbserver 18 | roles: 19 | - dbserver 20 | -------------------------------------------------------------------------------- /ansible/roles/appserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure Application Server 3 | 4 | - name: Add index.html 5 | copy: src=index.html dest=/var/www/html mode=0644 6 | -------------------------------------------------------------------------------- /ansible/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Common among all roles 3 | 4 | - name: Install required packages (RedHat) 5 | when: ansible_os_family == "RedHat" 6 | yum: name={{ item }} state=installed 7 | with_items: 8 | - curl 9 | - htop 10 | -------------------------------------------------------------------------------- /ansible/roles/dbserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure Database Server 3 | 4 | - name: Install required packages (RedHat) 5 | when: ansible_os_family == "RedHat" 6 | yum: name={{ item }} state=installed 7 | with_items: 8 | - mysql-server 9 | - name: Start services 10 | when: ansible_os_family == "RedHat" 11 | service: name={{ item }} state=started enabled=yes 12 | with_items: 13 | - mysqld 14 | -------------------------------------------------------------------------------- /ansible/roles/webserver/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Configure Web Server 3 | 4 | - name: Install required packages (RedHat) 5 | when: ansible_os_family == "RedHat" 6 | yum: name={{ item }} state=installed 7 | with_items: 8 | - httpd 9 | - name: Remove packages (RedHat) 10 | when: ansible_os_family == "RedHat" 11 | yum: name={{ item }} state=absent 12 | with_items: 13 | - nginx 14 | - name: Start services 15 | when: ansible_os_family == "RedHat" 16 | service: name={{ item }} state=started enabled=yes 17 | with_items: 18 | - httpd 19 | -------------------------------------------------------------------------------- /ci_ansible_inventory: -------------------------------------------------------------------------------- 1 | [webserver] 2 | localhost ansible_connection=local 3 | [appserver] 4 | localhost ansible_connection=local 5 | [dbserver] 6 | localhost ansible_connection=local 7 | -------------------------------------------------------------------------------- /generate_inventory_file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # joshcb@amazon.com 3 | # Generates an Ansible Inventory file from an EC2 Tag 4 | # v1.1.0 5 | 6 | # Set environment for ec2 tools 7 | source ~/.bash_profile 8 | 9 | # Get Region and Instance ID 10 | region=`curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | awk -F\" '{print $4}'` 11 | instance_id=`/opt/aws/bin/ec2-metadata -i | cut -d ' ' -f2` 12 | 13 | # Query metadata for our instance id and fetch values of the Ansible_Roles tag 14 | tags="$(/opt/aws/bin/ec2-describe-tags --region $region --filter \"resource-type=instance\" \ 15 | --filter \"resource-id=$instance_id\" --filter \"key=Ansible_Roles\" | cut -f5)" 16 | 17 | # Whitespace get outta here we don't need you 18 | tags_no_whitespace="$(echo -e "${tags}" | tr -d '[[:space:]]')" 19 | 20 | # Wipe out existing file :fire: 21 | printf '' > /tmp/inventory 22 | 23 | # http://stackoverflow.com/questions/10586153/split-string-into-an-array-in-bash 24 | IFS=', ' read -r -a array <<< "$tags_no_whitespace" 25 | 26 | # Write out each role into an Ansible host group 27 | for element in "${array[@]}" 28 | do 29 | printf "[$element]\nlocalhost ansible_connection=local\n" >> /tmp/inventory 30 | done 31 | -------------------------------------------------------------------------------- /lambda/bootstrap.py: -------------------------------------------------------------------------------- 1 | """ 2 | This AWS Lambda function is intended to be invoked via a Cloudwatch Event for a 3 | new intance launch. We get the instance ID from the event message, find our pipeline 4 | bucket, the latest artifact in the bucket, tell the new instance to grab the 5 | artifact, and finally execute it locally via runcommand. 6 | chavisb@amazon.com 7 | v1.0.0 8 | """ 9 | import datetime 10 | import logging 11 | import boto3 12 | from botocore.exceptions import ClientError 13 | 14 | LOGGER = logging.getLogger() 15 | LOGGER.setLevel(logging.INFO) 16 | 17 | # assume we're always using a pipeline name GARLC 18 | PIPELINE_NAME = 'GARLC' 19 | 20 | def is_a_garlc_instance(instance_id): 21 | """ 22 | Determine if an instance is GARLC enabled 23 | """ 24 | filters = [ 25 | {'Name': 'tag:has_ssm_agent', 'Values': ['true', 'True']} 26 | ] 27 | try: 28 | ec2 = boto3.client('ec2') 29 | instance = ec2.describe_instances(InstanceIds=[str(instance_id)], Filters=filters) 30 | except ClientError as err: 31 | LOGGER.error(str(err)) 32 | return False 33 | 34 | if instance: 35 | return True 36 | else: 37 | LOGGER.error(str(instance_id) + " is not a GARLC instance!") 38 | return False 39 | 40 | def find_bucket(): 41 | """ 42 | find S3 bucket that codedeploy uses and return bucket name 43 | """ 44 | try: 45 | codepipeline = boto3.client('codepipeline') 46 | pipeline = codepipeline.get_pipeline(name=PIPELINE_NAME) 47 | return str(pipeline['pipeline']['artifactStore']['location']) 48 | except (ClientError, KeyError, TypeError) as err: 49 | LOGGER.error(err) 50 | return False 51 | 52 | def find_newest_artifact(bucket): 53 | """ 54 | find and return the newest artifact in codepipeline bucket 55 | """ 56 | #TODO 57 | #implement boto collections to support more than 1000 artifacts per bucket 58 | try: 59 | aws_s3 = boto3.client('s3') 60 | objects = aws_s3.list_objects(Bucket=bucket) 61 | artifact_list = [artifact for artifact in objects['Contents']] 62 | artifact_list.sort(key=lambda artifact: artifact['LastModified'], reverse=True) 63 | return 's3://' + bucket + '/' + str(artifact_list[0]['Key']) 64 | except (ClientError, KeyError) as err: 65 | LOGGER.error(err) 66 | return False 67 | 68 | def ssm_commands(artifact): 69 | """ 70 | Builds commands to be sent to SSM (Run Command) 71 | """ 72 | utc_datetime = datetime.datetime.utcnow() 73 | timestamp = utc_datetime.strftime("%Y%m%d%H%M%S") 74 | return [ 75 | 'export AWS_DEFAULT_REGION=`curl -s http://169.254.169.254/' \ 76 | "latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{print $4}'`", 77 | 'aws configure set s3.signature_version s3v4', 78 | 'aws s3 cp {0} /tmp/{1}.zip --quiet'.format(artifact, timestamp), 79 | 'unzip -qq /tmp/{0}.zip -d /tmp/{1}'.format(timestamp, timestamp), 80 | 'bash /tmp/{0}/generate_inventory_file.sh'.format(timestamp), 81 | 'ansible-playbook -i "/tmp/inventory" /tmp/{0}/ansible/playbook.yml'.format(timestamp) 82 | ] 83 | 84 | def send_run_command(instance_id, commands): 85 | """ 86 | Sends the Run Command API Call 87 | """ 88 | try: 89 | ssm = boto3.client('ssm') 90 | except ClientError as err: 91 | LOGGER.error("Run Command Failed!\n%s", str(err)) 92 | return False 93 | 94 | try: 95 | ssm.send_command( 96 | InstanceIds=[instance_id], 97 | DocumentName='AWS-RunShellScript', 98 | TimeoutSeconds=900, 99 | Parameters={ 100 | 'commands': commands, 101 | 'executionTimeout': ['600'] # Seconds all commands have to complete in 102 | } 103 | ) 104 | return True 105 | except ClientError as err: 106 | if 'ThrottlingException' in str(err): 107 | LOGGER.info("RunCommand throttled, automatically retrying...") 108 | send_run_command(instance_id, commands) 109 | else: 110 | LOGGER.error("Run Command Failed!\n%s", str(err)) 111 | return False 112 | 113 | def log_event(event): 114 | """Logs event information for debugging""" 115 | LOGGER.info("====================================================") 116 | LOGGER.info(event) 117 | LOGGER.info("====================================================") 118 | 119 | def get_instance_id(event): 120 | """ Grab the instance ID out of the "event" dict sent by cloudwatch events """ 121 | try: 122 | return str(event['detail']['instance-id']) 123 | except (TypeError, KeyError) as err: 124 | LOGGER.error(err) 125 | return False 126 | 127 | def resources_exist(instance_id, bucket): 128 | """ 129 | Validates instance_id and bucket have values 130 | """ 131 | if not instance_id: 132 | LOGGER.error('Unable to retrieve Instance ID!') 133 | return False 134 | elif not bucket: 135 | LOGGER.error('Unable to retrieve Bucket Name!') 136 | return False 137 | else: return True 138 | 139 | 140 | def handle(event, _context): 141 | """ Lambda Handler """ 142 | log_event(event) 143 | instance_id = get_instance_id(event) 144 | bucket = find_bucket() 145 | 146 | if resources_exist(instance_id, bucket) and is_a_garlc_instance(instance_id): 147 | artifact = find_newest_artifact(bucket) 148 | commands = ssm_commands(artifact) 149 | send_run_command(instance_id, commands) 150 | LOGGER.info('===SUCCESS===') 151 | return True 152 | else: 153 | return False 154 | -------------------------------------------------------------------------------- /lambda/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Triggers RunCommand on all instances with tag has_ssm_agent set to true. 3 | Fetches artifact from S3 via CodePipeline, extracts the contents, and finally 4 | runs Ansible locally on the instance to configure itself. Uses 5 | runcommand_helper.py to actually execute RunCommand. 6 | joshcb@amazon.com 7 | v1.0.0 8 | """ 9 | from __future__ import print_function 10 | import json 11 | import datetime 12 | import logging 13 | from botocore.exceptions import ClientError 14 | import boto3 15 | 16 | LOGGER = logging.getLogger() 17 | LOGGER.setLevel(logging.INFO) 18 | 19 | def find_artifact(event): 20 | """ 21 | Returns the S3 Object that holds the artifact 22 | """ 23 | try: 24 | object_key = event['CodePipeline.job']['data']['inputArtifacts'][0] \ 25 | ['location']['s3Location']['objectKey'] 26 | bucket = event['CodePipeline.job']['data']['inputArtifacts'][0] \ 27 | ['location']['s3Location']['bucketName'] 28 | return 's3://{0}/{1}'.format(bucket, object_key) 29 | except KeyError as err: 30 | raise KeyError("Couldn't get S3 object!\n%s", err) 31 | 32 | def ssm_commands(artifact): 33 | """ 34 | Builds commands to be sent to SSM (Run Command) 35 | """ 36 | # TODO 37 | # Error handling in the command generation 38 | utc_datetime = datetime.datetime.utcnow() 39 | timestamp = utc_datetime.strftime("%Y%m%d%H%M%S") 40 | return [ 41 | 'export AWS_DEFAULT_REGION=`curl -s http://169.254.169.254/' \ 42 | "latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{print $4}'`", 43 | 'aws configure set s3.signature_version s3v4', 44 | 'aws s3 cp {0} /tmp/{1}.zip --quiet'.format(artifact, timestamp), 45 | 'unzip -qq /tmp/{0}.zip -d /tmp/{1}'.format(timestamp, timestamp), 46 | 'bash /tmp/{0}/generate_inventory_file.sh'.format(timestamp), 47 | 'ansible-playbook -i "/tmp/inventory" /tmp/{0}/ansible/playbook.yml'.format(timestamp) 48 | ] 49 | 50 | def codepipeline_success(job_id): 51 | """ 52 | Puts CodePipeline Success Result 53 | """ 54 | try: 55 | codepipeline = boto3.client('codepipeline') 56 | codepipeline.put_job_success_result(jobId=job_id) 57 | LOGGER.info('===SUCCESS===') 58 | return True 59 | except ClientError as err: 60 | LOGGER.error("Failed to PutJobSuccessResult for CodePipeline!\n%s", err) 61 | return False 62 | 63 | def codepipeline_failure(job_id, message): 64 | """ 65 | Puts CodePipeline Failure Result 66 | """ 67 | try: 68 | codepipeline = boto3.client('codepipeline') 69 | codepipeline.put_job_failure_result( 70 | jobId=job_id, 71 | failureDetails={'type': 'JobFailed', 'message': message} 72 | ) 73 | LOGGER.info('===FAILURE===') 74 | return True 75 | except ClientError as err: 76 | LOGGER.error("Failed to PutJobFailureResult for CodePipeline!\n%s", err) 77 | return False 78 | 79 | def find_instances(): 80 | """ 81 | Find Instances to invoke Run Command against 82 | """ 83 | instance_ids = [] 84 | filters = [ 85 | {'Name': 'tag:has_ssm_agent', 'Values': ['true', 'True']}, 86 | {'Name': 'instance-state-name', 'Values': ['running']} 87 | ] 88 | try: 89 | instance_ids = find_instance_ids(filters) 90 | print(instance_ids) 91 | except ClientError as err: 92 | LOGGER.error("Failed to DescribeInstances with EC2!\n%s", err) 93 | 94 | return instance_ids 95 | 96 | def find_instance_ids(filters): 97 | """ 98 | EC2 API calls to retrieve instances matched by the filter 99 | """ 100 | ec2 = boto3.resource('ec2') 101 | return [i.id for i in ec2.instances.all().filter(Filters=filters)] 102 | 103 | def break_instance_ids_into_chunks(instance_ids): 104 | """ 105 | Returns successive chunks from instance_ids 106 | """ 107 | size = 3 108 | chunks = [] 109 | for i in range(0, len(instance_ids), size): 110 | chunks.append(instance_ids[i:i + size]) 111 | return chunks 112 | 113 | def execute_runcommand(chunked_instance_ids, commands, job_id): 114 | """ 115 | Handoff RunCommand to the RunCommand Helper AWS Lambda function 116 | """ 117 | try: 118 | client = boto3.client('lambda') 119 | except ClientError as err: 120 | LOGGER.error("Failed to created a Lambda client!\n%s", err) 121 | codepipeline_failure(job_id, err) 122 | return False 123 | 124 | event = { 125 | "ChunkedInstanceIds": chunked_instance_ids, 126 | "Commands": commands 127 | } 128 | response = client.invoke_async( 129 | FunctionName='garlc_runcommand_helper', 130 | InvokeArgs=json.dumps(event) 131 | ) 132 | 133 | if response['Status'] is 202: 134 | codepipeline_success(job_id) 135 | return True 136 | else: 137 | codepipeline_failure(job_id, response) 138 | return False 139 | 140 | def handle(event, _context): 141 | """ 142 | Lambda main handler 143 | """ 144 | LOGGER.info(event) 145 | try: 146 | job_id = event['CodePipeline.job']['id'] 147 | except KeyError as err: 148 | LOGGER.error("Could not retrieve CodePipeline Job ID!\n%s", err) 149 | return False 150 | 151 | instance_ids = find_instances() 152 | commands = ssm_commands(find_artifact(event)) 153 | if len(instance_ids) != 0: 154 | chunked_instance_ids = break_instance_ids_into_chunks(instance_ids) 155 | execute_runcommand(chunked_instance_ids, commands, job_id) 156 | return True 157 | else: 158 | codepipeline_failure(job_id, 'No Instance IDs Provided!') 159 | return False 160 | -------------------------------------------------------------------------------- /lambda/runcommand_helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | Triggers Run Command on all instances specified. This helper function is used 3 | in order to scale out GARLC. The function is recursively called in batches 4 | so AWS Lambda timeout's are avoided. RunCommand throttling is handled as well. 5 | joshcb@amazon.com 6 | v1.0.0 7 | """ 8 | from __future__ import print_function 9 | import json 10 | import logging 11 | from botocore.exceptions import ClientError 12 | import boto3 13 | 14 | LOGGER = logging.getLogger() 15 | LOGGER.setLevel(logging.INFO) 16 | 17 | def send_run_command(instance_ids, commands): 18 | """ 19 | Tries to queue a RunCommand job. If a ThrottlingException is encountered 20 | recursively calls itself until success. 21 | """ 22 | try: 23 | ssm = boto3.client('ssm') 24 | except ClientError as err: 25 | LOGGER.error("Run Command Failed!\n%s", str(err)) 26 | return False 27 | 28 | try: 29 | ssm.send_command( 30 | InstanceIds=instance_ids, 31 | DocumentName='AWS-RunShellScript', 32 | Parameters={ 33 | 'commands': commands, 34 | 'executionTimeout': ['600'] # Seconds all commands have to complete in 35 | } 36 | ) 37 | LOGGER.info('============RunCommand sent successfully') 38 | return True 39 | except ClientError as err: 40 | if 'ThrottlingException' in str(err): 41 | LOGGER.info("RunCommand throttled, automatically retrying...") 42 | send_run_command(instance_ids, commands) 43 | else: 44 | LOGGER.error("Run Command Failed!\n%s", str(err)) 45 | return False 46 | 47 | def invoke_lambda(chunks, commands): 48 | """ 49 | Hands off the remaining work to another Lambda function. This is done 50 | to avoid hitting AWS Lambda timeouts with a single Lambda function. 51 | """ 52 | if len(chunks) == 0: 53 | LOGGER.info('No more chunks of instances to process') 54 | return True 55 | else: 56 | try: 57 | client = boto3.client('lambda') 58 | except ClientError as err: 59 | # Log the error and keep trying until we timeout 60 | LOGGER.error("Failed to create a Lambda client!\n%s", err) 61 | invoke_lambda(chunks, commands) 62 | return False 63 | 64 | event = { 65 | "ChunkedInstanceIds": chunks, 66 | "Commands": commands 67 | } 68 | response = client.invoke_async( 69 | FunctionName='garlc_runcommand_helper', 70 | InvokeArgs=json.dumps(event) 71 | ) 72 | 73 | if response['Status'] is 202: 74 | LOGGER.info('Invoked the next Lambda function to continue...') 75 | return True 76 | else: 77 | LOGGER.error(response) 78 | return False 79 | 80 | def handle(event, _context): 81 | """ 82 | Lambda main handler 83 | """ 84 | LOGGER.info(event) 85 | try: 86 | chunked_instance_ids = event['ChunkedInstanceIds'] 87 | LOGGER.debug('==========Chunks remaining:') 88 | LOGGER.debug(len(chunked_instance_ids)) 89 | commands = event['Commands'] 90 | except (TypeError, KeyError) as err: 91 | LOGGER.error("Could not parse event!\n%s", err) 92 | return False 93 | 94 | # We work on one chunk at a time until there are no more chunks left. 95 | # Each chunk is handled by a new AWS Lambda function. 96 | instance_ids = chunked_instance_ids.pop(0) 97 | LOGGER.debug('==========Instances in this chunk:') 98 | LOGGER.debug(instance_ids) 99 | send_run_command(instance_ids, commands) 100 | invoke_lambda(chunked_instance_ids, commands) 101 | return True 102 | -------------------------------------------------------------------------------- /lambda/test_bootstrap.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit Tests for trigger_run_command Lambda function 3 | """ 4 | from mock import patch, MagicMock 5 | from aws_lambda_sample_events import SampleEvent 6 | from botocore.exceptions import ClientError 7 | from bootstrap import find_bucket 8 | from bootstrap import is_a_garlc_instance 9 | from bootstrap import find_newest_artifact 10 | from bootstrap import get_instance_id 11 | from bootstrap import resources_exist 12 | from bootstrap import send_run_command 13 | from bootstrap import handle 14 | 15 | @patch('boto3.client') 16 | def test_find_bucket(mock_client): 17 | """ 18 | Test the find_bucket function with valid input 19 | """ 20 | pipeline_object = { 21 | "pipeline": { 22 | "artifactStore": { 23 | "location": "blah" 24 | } 25 | } 26 | } 27 | codepipeline = MagicMock() 28 | mock_client.return_value = codepipeline 29 | codepipeline.get_pipeline.return_value = pipeline_object 30 | assert find_bucket() == 'blah' 31 | 32 | @patch('boto3.client') 33 | def test_find_bucket_with_typeerror(mock_client): 34 | """ 35 | Test the find_bucket functin with a TypeError 36 | """ 37 | codepipeline = MagicMock() 38 | mock_client.return_value = codepipeline 39 | codepipeline.get_pipeline.return_value = {'blah'} 40 | assert find_bucket() is False 41 | 42 | @patch('boto3.client') 43 | def test_find_bucket_with_keyerror(mock_client): 44 | """ 45 | Test the find_bucket functin with a KeyError 46 | """ 47 | codepipeline = MagicMock() 48 | mock_client.return_value = codepipeline 49 | codepipeline.get_pipeline.return_value = {"blah": "blah"} 50 | assert find_bucket() is False 51 | 52 | @patch('boto3.client') 53 | def test_find_bucket_with_clienterror(mock_client): 54 | """ 55 | Test the find_bucket function with a KeyError 56 | """ 57 | err_msg = { 58 | 'Error': { 59 | 'Code': 400, 60 | 'Message': 'Boom!' 61 | } 62 | } 63 | mock_client.side_effect = ClientError(err_msg, 'blah') 64 | assert find_bucket() is False 65 | 66 | @patch('boto3.client') 67 | def test_is_a_garlc_instance(mock_client): 68 | """ 69 | Test is_a_garlc_instance returns true when EC2 API Call 70 | returns a string 71 | """ 72 | ec2 = MagicMock() 73 | mock_client.return_value = ec2 74 | ec2.describe_instances.return_value = "instance" 75 | assert is_a_garlc_instance("instance") is True 76 | 77 | @patch('boto3.client') 78 | def test_is_a_garlc_instance_except_when_its_not(mock_client): 79 | """ 80 | Test is_a_garlc_instance returns false when it's missing the tag 81 | """ 82 | ec2 = MagicMock() 83 | mock_client.return_value = ec2 84 | ec2.describe_instances.return_value = "" 85 | assert is_a_garlc_instance("instance") is False 86 | 87 | @patch('boto3.client') 88 | def test_is_a_garlc_instance_with_clienterror(mock_client): 89 | """ 90 | Test is_a_garlc_instance with ClientError 91 | """ 92 | err_msg = { 93 | 'Error': { 94 | 'Code': 400, 95 | 'Message': 'Sad Code' 96 | } 97 | } 98 | mock_client.side_effect = ClientError(err_msg, 'sad response') 99 | assert is_a_garlc_instance('foo') is False 100 | 101 | @patch('boto3.client') 102 | def test_find_newest_artifact(mock_client): 103 | """ 104 | test find_newest_artifact returns a properly formatted string 105 | """ 106 | bucket_objects = { 107 | "Contents": [ 108 | { 109 | "Key": "blah", 110 | "LastModified": "datetime.datetime(2016, 3, 18, 19, 20, 29, tzinfo=tzutc())" 111 | } 112 | ] 113 | } 114 | aws_s3 = MagicMock() 115 | mock_client.return_value = aws_s3 116 | aws_s3.list_objects.return_value = bucket_objects 117 | assert find_newest_artifact('blah') == 's3://blah/blah' 118 | 119 | @patch('boto3.client') 120 | def test_find_newest_artifact_with_keyerror(mock_client): 121 | """ 122 | test find_newest_artifact returns false with a KeyError 123 | """ 124 | bucket_objects = { 125 | "Contents": [ 126 | { 127 | "LastModified": "datetime.datetime(2016, 3, 18, 19, 20, 29, tzinfo=tzutc())" 128 | } 129 | ] 130 | } 131 | aws_s3 = MagicMock() 132 | mock_client.return_value = aws_s3 133 | aws_s3.list_objects.return_value = bucket_objects 134 | assert find_newest_artifact('blah') is False 135 | 136 | @patch('boto3.client') 137 | def test_find_newest_artifact_with_clienterror(mock_client): 138 | """ 139 | test find_newest_artifact returns false with a ClientError 140 | """ 141 | err_msg = { 142 | 'Error': { 143 | 'Code': 400, 144 | 'Message': 'Boom!' 145 | } 146 | } 147 | mock_client.side_effect = ClientError(err_msg, 'blah') 148 | assert find_newest_artifact('blah') is False 149 | 150 | @patch('boto3.client') 151 | def test_get_instance_id(mock_event): 152 | """ 153 | test get instance id returns instance id string 154 | """ 155 | mock_event = { 156 | 'detail': { 157 | 'instance-id': 'i-12345678' 158 | } 159 | } 160 | assert get_instance_id(mock_event) == mock_event['detail']['instance-id'] 161 | 162 | @patch('boto3.client') 163 | def test_resources_exist(mock_event): 164 | """ 165 | tests resources_exist function 166 | """ 167 | instance_id = "i-12345678" 168 | bucket = "buckette" 169 | assert resources_exist(instance_id, bucket) is True 170 | 171 | @patch('boto3.client') 172 | def test_resources_exist_when_missing_instance_id(mock_event): 173 | """ 174 | tests resources_exist function returns false when instance_id is blank 175 | """ 176 | instance_id = '' 177 | bucket = 'buckette' 178 | assert resources_exist(instance_id, bucket) is False 179 | 180 | @patch('boto3.client') 181 | def test_resources_exist_when_missing_bucket(mock_event): 182 | """ 183 | tests resources_exist function returns false when bucket is None 184 | """ 185 | instance_id = "i-12345678" 186 | bucket = None 187 | assert resources_exist(instance_id, bucket) is False 188 | 189 | @patch('boto3.client') 190 | def test_send_run_command(mock_client): 191 | """ 192 | Test the send_run_command function without errors 193 | """ 194 | ssm = MagicMock() 195 | mock_client.return_value = ssm 196 | ssm.send_command.return_value = True 197 | assert send_run_command(['i-12345678'], ['blah']) 198 | 199 | @patch('boto3.client') 200 | def test_send_run_command_with_clienterror(mock_client): 201 | """ 202 | Test the send_run_command function with ClientError 203 | """ 204 | err_msg = { 205 | 'Error': { 206 | 'Code': 400, 207 | 'Message': 'Boom!' 208 | } 209 | } 210 | mock_client.side_effect = ClientError(err_msg, 'blah') 211 | assert send_run_command('blah', 'blah') is False 212 | 213 | @patch('boto3.client') 214 | def test_send_run_command_with_clienterror_during_send_command(mock_client): 215 | """ 216 | Test the send_run_command function with a ClientError from send_command 217 | """ 218 | err_msg = { 219 | 'Error': { 220 | 'Code': 400, 221 | 'Message': 'Boom!' 222 | } 223 | } 224 | ssm = MagicMock() 225 | mock_client.return_value = ssm 226 | ssm.send_command.side_effect = ClientError(err_msg, 'blah') 227 | assert send_run_command('blah', 'blah') is False 228 | 229 | @patch('bootstrap.send_run_command') 230 | @patch('boto3.client') 231 | def test_send_run_command_with_throttlingexception(mock_client, mock_run_command): 232 | """ 233 | Test the send_run_command function with a ThrottlingException 234 | """ 235 | err_msg = { 236 | 'Error': { 237 | 'Code': 400, 238 | 'Message': 'ThrottlingException' 239 | } 240 | } 241 | ssm = MagicMock() 242 | mock_client.return_value = ssm 243 | ssm.send_command.side_effect = ClientError(err_msg, 'blah') 244 | assert send_run_command('blah', 'blah') is not False 245 | 246 | @patch('bootstrap.send_run_command') 247 | @patch('bootstrap.find_newest_artifact') 248 | @patch('bootstrap.is_a_garlc_instance') 249 | @patch('bootstrap.find_bucket') 250 | def test_handle(mock_find_bucket, mock_is_instance, mock_artifact, mock_ssm): 251 | """ 252 | Test the handle function with valid input 253 | """ 254 | event = SampleEvent('cloudwatch_events') 255 | mock_find_bucket.return_value = 'buckette' 256 | mock_is_instance.return_value = True 257 | mock_artifact.return_value = 's3://blah/blah.zip' 258 | mock_ssm.return_value = True 259 | assert handle(event.event, 'blah') is True 260 | 261 | @patch('bootstrap.find_bucket') 262 | def test_handle_with_invalid_bucket(mock_find_bucket): 263 | """ 264 | Test the handle function with invalid bucket 265 | """ 266 | event = SampleEvent('cloudwatch_events') 267 | mock_find_bucket.return_value = '' 268 | assert handle(event.event, 'blah') is False 269 | 270 | @patch('bootstrap.find_bucket') 271 | def test_handle_with_invalid_event(mock_find_bucket): 272 | """ 273 | Test the handle function with invalid event 274 | """ 275 | event = 'blah' 276 | mock_find_bucket.return_value = 'buckette' 277 | assert handle(event, 'blah') is False 278 | -------------------------------------------------------------------------------- /lambda/test_main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit Tests for trigger_run_command Lambda function 3 | """ 4 | import pytest 5 | import boto3 6 | from botocore.exceptions import ClientError 7 | from mock import MagicMock, patch 8 | from main import find_artifact 9 | from main import ssm_commands 10 | from main import codepipeline_success 11 | from main import codepipeline_failure 12 | from main import find_instances 13 | from main import handle 14 | from main import execute_runcommand 15 | from main import find_instance_ids 16 | from freezegun import freeze_time 17 | from aws_lambda_sample_events import SampleEvent 18 | 19 | def test_find_artifact(): 20 | """ 21 | Test the find_artifact function with valid event 22 | """ 23 | codepipeline = SampleEvent('codepipeline') 24 | assert find_artifact(codepipeline.event) == \ 25 | 's3://codepipeline-us-east-1-123456789000/pipeline/MyApp/random.zip' 26 | 27 | def test_find_artifact_invalid(): 28 | """ 29 | Test the find_artifact function with invalid event 30 | """ 31 | event = {} 32 | with pytest.raises(KeyError): 33 | assert find_artifact(event) == 'blah' 34 | 35 | @freeze_time('2016-01-01') 36 | def test_ssm_commands(): 37 | """ 38 | Test the ssm_commands function 39 | """ 40 | artifact = 'bucket/test/key' 41 | commands = [ 42 | 'export AWS_DEFAULT_REGION=`curl -s http://169.254.169.254/' \ 43 | "latest/dynamic/instance-identity/document | grep region | awk -F\\\" '{print $4}'`", 44 | 'aws configure set s3.signature_version s3v4', 45 | 'aws s3 cp bucket/test/key /tmp/20160101000000.zip --quiet', 46 | 'unzip -qq /tmp/20160101000000.zip -d /tmp/20160101000000', 47 | 'bash /tmp/20160101000000/generate_inventory_file.sh', 48 | 'ansible-playbook -i "/tmp/inventory" /tmp/20160101000000/ansible/playbook.yml' 49 | ] 50 | assert ssm_commands(artifact) == commands 51 | 52 | @patch('boto3.client') 53 | def test_codepipeline_success(mock_client): 54 | """ 55 | Test the codepipeline_success function with valid data 56 | """ 57 | codepipeline = MagicMock() 58 | mock_client.return_value = codepipeline 59 | codepipeline.put_job_success_result.return_value = True 60 | assert codepipeline_success(1) 61 | 62 | @patch('boto3.client') 63 | def test_codepipeline_success_with_exception(mock_client): 64 | """ 65 | Test the codepipeline_success function when a boto exception occurs 66 | """ 67 | codepipeline = MagicMock() 68 | mock_client.return_value = codepipeline 69 | err_msg = { 70 | 'Error': { 71 | 'Code': 400, 72 | 'Message': 'Boom!' 73 | } 74 | } 75 | codepipeline.put_job_success_result.side_effect = ClientError(err_msg, 'Test') 76 | assert codepipeline_success(1) is False 77 | 78 | @patch('boto3.client') 79 | def test_codepipeline_failure(mock_client): 80 | """ 81 | Test the codepipeline_failure function with valid data 82 | """ 83 | codepipeline = MagicMock() 84 | mock_client.return_value = codepipeline 85 | codepipeline.put_job_failure_result.return_value = True 86 | assert codepipeline_failure(1, 'blah') 87 | 88 | @patch('boto3.client') 89 | def test_codepipeline_failure_with_exception(mock_client): 90 | """ 91 | Test the codepipeline_failure function when a boto exception occurs 92 | """ 93 | codepipeline = MagicMock() 94 | mock_client.return_value = codepipeline 95 | err_msg = { 96 | 'Error': { 97 | 'Code': 400, 98 | 'Message': 'Boom!' 99 | } 100 | } 101 | codepipeline.put_job_failure_result.side_effect = ClientError(err_msg, 'Test') 102 | assert codepipeline_failure(1, 'blah') is False 103 | 104 | @patch('main.find_instance_ids') 105 | def test_find_instances(mock_instances): 106 | """ 107 | Test the find_instances function without errors 108 | """ 109 | instances = ['abcdef-12345'] 110 | mock_instances.return_value = instances 111 | assert find_instances() == instances 112 | 113 | @patch('boto3.resource') 114 | def test_find_instances_boto_error(mock_client): 115 | """ 116 | Test the find_instances function when a boto exception occurs 117 | """ 118 | err_msg = { 119 | 'Error': { 120 | 'Code': 400, 121 | 'Message': 'Boom!' 122 | } 123 | } 124 | mock_client.side_effect = ClientError(err_msg, 'Test') 125 | assert find_instances() == [] 126 | 127 | @patch('boto3.resources.collection.ResourceCollection.filter') 128 | def test_find_instance_ids(mock_resource): 129 | """ 130 | Test the find_instance_ids function 131 | """ 132 | instance_id = 'abcdef-12345' 133 | instance = [MagicMock(id=instance_id)] 134 | boto3.setup_default_session(region_name='us-east-1') 135 | mock_resource.return_value = instance 136 | assert find_instance_ids('blah') == [instance_id] 137 | 138 | @patch('main.codepipeline_success') 139 | @patch('main.execute_runcommand') 140 | @patch('main.find_artifact') 141 | @patch('main.ssm_commands') 142 | @patch('main.find_instances') 143 | def test_handle(mock_instances, mock_commands, mock_artifact, mock_run_command, 144 | mock_success): 145 | """ 146 | Test the handle function with valid input and instances 147 | """ 148 | mock_instances.return_value = ['abcdef-12345'] 149 | mock_commands.return_value = True 150 | mock_artifact.return_value = True 151 | mock_run_command.return_value = True 152 | mock_success.return_value = True 153 | codepipeline = SampleEvent('codepipeline') 154 | assert handle(codepipeline.event, 'Test') 155 | 156 | @patch('main.codepipeline_failure') 157 | @patch('main.find_artifact') 158 | @patch('main.ssm_commands') 159 | @patch('main.find_instances') 160 | def test_handle_no_instances(mock_instances, mock_commands, mock_artifact, 161 | mock_failure): 162 | """ 163 | Test the handle function with valid input and no instances 164 | """ 165 | mock_instances.return_value = [] 166 | mock_commands.return_value = True 167 | mock_artifact.return_value = True 168 | mock_failure.return_value = True 169 | codepipeline = SampleEvent('codepipeline') 170 | assert handle(codepipeline.event, 'Test') is False 171 | 172 | def test_handle_invalid_event(): 173 | """ 174 | Test the handle function with an invalid event 175 | """ 176 | event = {} 177 | assert handle(event, 'Test') is False 178 | 179 | @patch('main.codepipeline_success') 180 | @patch('boto3.client') 181 | def test_execute_runcommand(mock_client, mock_success): 182 | """ 183 | Test the execute_runcommand function with valid input 184 | """ 185 | client = MagicMock() 186 | mock_client.return_value = client 187 | client.invoke_async.return_value = {"Status": 202} 188 | mock_success.return_value = True 189 | chunked_instance_ids = ['abcdef-12345'] 190 | commands = ['blah'] 191 | job_id = 1 192 | assert execute_runcommand(chunked_instance_ids, commands, job_id) is True 193 | 194 | @patch('main.codepipeline_failure') 195 | @patch('boto3.client') 196 | def test_execute_runcommand_with_failed_status(mock_client, mock_failure): 197 | """ 198 | Test the execute_runcommand function with a failed status code 199 | """ 200 | client = MagicMock() 201 | mock_client.return_value = client 202 | client.invoke_async.return_value = {"Status": 400} 203 | mock_failure.return_value = True 204 | chunked_instance_ids = ['abcdef-12345'] 205 | commands = ['blah'] 206 | job_id = 1 207 | assert execute_runcommand(chunked_instance_ids, commands, job_id) is False 208 | 209 | @patch('boto3.client') 210 | def test_execute_runcommand_with_clienterror(mock_client): 211 | """ 212 | Test the execute_runcommand function with a ClientError 213 | """ 214 | err_msg = { 215 | 'Error': { 216 | 'Code': 400, 217 | 'Message': 'Boom!' 218 | } 219 | } 220 | mock_client.side_effect = ClientError(err_msg, 'Test') 221 | chunked_instance_ids = ['abcdef-12345'] 222 | commands = ['blah'] 223 | job_id = 1 224 | assert execute_runcommand(chunked_instance_ids, commands, job_id) is False 225 | -------------------------------------------------------------------------------- /lambda/test_runcommand_helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit Tests for trigger_run_command Lambda function 3 | """ 4 | from mock import patch, MagicMock 5 | from botocore.exceptions import ClientError 6 | from runcommand_helper import send_run_command 7 | from runcommand_helper import invoke_lambda 8 | from runcommand_helper import handle 9 | 10 | @patch('boto3.client') 11 | def test_send_run_command(mock_client): 12 | """ 13 | Test the send_run_command function without errors 14 | """ 15 | ssm = MagicMock() 16 | mock_client.return_value = ssm 17 | ssm.send_command.return_value = True 18 | assert send_run_command(['i-12345678'], ['blah']) is True 19 | 20 | @patch('boto3.client') 21 | def test_send_run_command_with_clienterror(mock_client): 22 | """ 23 | Test the send_run_command function with ClientError 24 | """ 25 | err_msg = { 26 | 'Error': { 27 | 'Code': 400, 28 | 'Message': 'Boom!' 29 | } 30 | } 31 | mock_client.side_effect = ClientError(err_msg, 'blah') 32 | assert send_run_command('blah', 'blah') is False 33 | 34 | @patch('runcommand_helper.send_run_command') 35 | @patch('boto3.client') 36 | def test_send_run_command_with_throttlingexception(mock_client, mock_run_command): 37 | """ 38 | Test the send_run_command function with a ThrottlingException 39 | """ 40 | err_msg = { 41 | 'Error': { 42 | 'Code': 400, 43 | 'Message': 'ThrottlingException' 44 | } 45 | } 46 | ssm = MagicMock() 47 | mock_client.return_value = ssm 48 | ssm.send_command.side_effect = ClientError(err_msg, 'blah') 49 | assert send_run_command('blah', 'blah') is not False 50 | 51 | @patch('boto3.client') 52 | def test_send_run_command_with_clienterror_during_send_command(mock_client): 53 | """ 54 | Test the send_run_command function with a ClientError from send_command 55 | """ 56 | err_msg = { 57 | 'Error': { 58 | 'Code': 400, 59 | 'Message': 'Boom!' 60 | } 61 | } 62 | ssm = MagicMock() 63 | mock_client.return_value = ssm 64 | ssm.send_command.side_effect = ClientError(err_msg, 'blah') 65 | assert send_run_command('blah', 'blah') is False 66 | 67 | def test_invoke_lambda_with_no_chunks_remaining(): 68 | """ 69 | Test invoke_lambda when there are no chunks remaining to process 70 | """ 71 | assert invoke_lambda([], []) is True 72 | 73 | @patch('boto3.client') 74 | def test_invoke_lambda(mock_client): 75 | """ 76 | Test invoke_lambda with valid input and no errors 77 | """ 78 | client = MagicMock() 79 | mock_client.return_value = client 80 | client.invoke_async.return_value = {"Status": 202} 81 | assert invoke_lambda([["blah"]], ["blah"]) is True 82 | 83 | @patch('boto3.client') 84 | def test_invoke_lambda_with_bad_status_code(mock_client): 85 | """ 86 | Test invoke_lambda when a bad status code is returned 87 | """ 88 | client = MagicMock() 89 | mock_client.return_value = client 90 | client.invoke_async.return_value = {"Status": 400} 91 | assert invoke_lambda([["blah"]], ["blah"]) is False 92 | 93 | @patch('runcommand_helper.invoke_lambda') 94 | @patch('boto3.client') 95 | def test_invoke_lambda_with_clienterror(mock_client, mock_invoke): 96 | """ 97 | Test invoke_lambda with valid input and a ClientError 98 | """ 99 | err_msg = { 100 | 'Error': { 101 | 'Code': 400, 102 | 'Message': 'Boom!' 103 | } 104 | } 105 | mock_client.side_effect = ClientError(err_msg, 'blah') 106 | assert invoke_lambda([["blah"]], ["blah"]) is False 107 | 108 | @patch('runcommand_helper.invoke_lambda') 109 | @patch('runcommand_helper.send_run_command') 110 | def test_handle(mock_ssm, mock_invoke): 111 | """ 112 | Test the handle function with valid input and no errors 113 | """ 114 | mock_ssm.return_value = True 115 | mock_invoke.return_value = True 116 | event = { 117 | "ChunkedInstanceIds": [[1,2], [3,4]], 118 | "Commands": ["blah"] 119 | } 120 | assert handle(event, 'blah') is True 121 | 122 | def test_handle_with_typeerror(): 123 | """ 124 | Test the handle function with invalid event and TypeError 125 | """ 126 | event = "" 127 | assert handle(event, 'blah') is False 128 | 129 | def test_handle_with_keyerror(): 130 | """ 131 | Test the handle function with invalid event and KeyError 132 | """ 133 | event = { 134 | "ChunkedInstanceIds": [[1,2], [3,4]], 135 | "Blah": ["blah"] 136 | } 137 | assert handle(event, 'blah') is False 138 | -------------------------------------------------------------------------------- /lambda_bootstrap_payload.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/lambda-runcommand-configuration-management/381ab71b88bcd7ac647b727c06a852026be52c0b/lambda_bootstrap_payload.zip -------------------------------------------------------------------------------- /lambda_main_payload.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/lambda-runcommand-configuration-management/381ab71b88bcd7ac647b727c06a852026be52c0b/lambda_main_payload.zip -------------------------------------------------------------------------------- /lambda_runcommand_helper_payload.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-archives/lambda-runcommand-configuration-management/381ab71b88bcd7ac647b727c06a852026be52c0b/lambda_runcommand_helper_payload.zip -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==2.8.7 2 | pytest-cov==2.2.1 3 | freezegun==0.3.6 4 | boto3==1.2.6 5 | botocore==1.3.22 6 | mock==1.3.0 7 | aws_lambda_sample_events==1.0.1 8 | ansible==2.0.0.2 9 | -------------------------------------------------------------------------------- /solano.yml: -------------------------------------------------------------------------------- 1 | python: 2 | python_version: 2.7 3 | pip_requirements_file: requirements.txt 4 | tests: 5 | - ansible-playbook -i ci_ansible_inventory ansible/playbook.yml --check 6 | - py.test lambda/ --cov-report term-missing --cov lambda/ 7 | coverage: 8 | version: 2 9 | enabled: true 10 | ratchet: 90 11 | -------------------------------------------------------------------------------- /terraform/asg/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_autoscaling_group" "asg" { 2 | lifecycle { create_before_destroy = true } 3 | name = "${var.asg_name}" 4 | launch_configuration = "${var.lcid}" 5 | max_size = "${var.max_size}" 6 | min_size = "${var.min_size}" 7 | desired_capacity = "${var.desired_capacity}" 8 | vpc_zone_identifier = ["${var.subnet1}", "${var.subnet2}"] 9 | tag { 10 | key = "Ansible_Roles" 11 | value = "${var.ansible_roles}" 12 | propagate_at_launch = true 13 | } 14 | tag { 15 | key = "has_ssm_agent" 16 | value = "true" 17 | propagate_at_launch = true 18 | } 19 | } 20 | 21 | output "asgid" { 22 | value = "${aws_autoscaling_group.asg.name}" 23 | } 24 | -------------------------------------------------------------------------------- /terraform/asg/variables.tf: -------------------------------------------------------------------------------- 1 | variable "max_size" { 2 | default = "1" 3 | } 4 | variable "min_size" { 5 | default = "1" 6 | } 7 | variable "desired_capacity" { 8 | default = "1" 9 | } 10 | variable "lcid" {} 11 | variable "asg_name" {} 12 | variable "ansible_roles" {} 13 | variable "subnet1" {} 14 | variable "subnet2" {} 15 | -------------------------------------------------------------------------------- /terraform/iam/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_instance_profile" "profile" { 2 | name = "garlc_instance_profile" 3 | roles = ["${aws_iam_role.role.name}"] 4 | # Hack to fix an issue where the profile isn't 5 | # propagted before the LC tries to launch instances. 6 | provisioner "local-exec" { 7 | command = "sleep 10" 8 | } 9 | } 10 | 11 | # Add an EC2 Read Only Policy 12 | resource "aws_iam_role_policy" "ec2_readonly_policy" { 13 | name = "ec2_readonly_policy" 14 | role = "${aws_iam_role.role.id}" 15 | policy = <