├── .gitignore ├── LICENSE ├── README.md ├── ansible.cfg ├── inventory ├── ec2.ini ├── ec2.py └── localhost ├── provision_instances.yaml ├── requirements.txt ├── roles ├── hostname-configured │ └── tasks │ │ └── main.yaml ├── ssh-reachable │ └── tasks │ │ └── main.yaml └── ubuntu-updated │ └── tasks │ └── main.yaml ├── setup_instances.yaml ├── tasks ├── instance.yaml ├── keypair.yaml └── securitygroup.yaml └── terminate_instances.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | keys 2 | galaxy 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Brian Schott 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ansible-dc-ec2-tutorial 2 | ======================= 3 | 4 | This is the source code for a tutorial on using the ec2.py dynamic inventory feature of Ansible. This requires the python boto library for Amazon EC2. 5 | 6 | WARNING: terminate_instances.yaml terminates all instances in your cloud account! You have been warned. 7 | 8 | 9 | Installing: 10 | ----------- 11 | 12 | Install ansible and boto:: 13 | 14 | pip install -r requirements.txt 15 | 16 | Make sure that ec2.py is executable:: 17 | 18 | chmod a+x inventory/ec2.py 19 | 20 | Set AWS_ACCESS_KEY and AWS_SECRET_KEY for Ansible and AWS_SECRET_KEY and AWS_SECRET_ACCESS_KEY for Boto:: 21 | 22 | export AWS_ACCESS_KEY="AK******************" 23 | export AWS_ACCESS_KEY_ID="AK********************" 24 | export AWS_SECRET_KEY="***************************************" 25 | export AWS_SECRET_ACCESS_KEY="*************************************" 26 | 27 | Set the AWS default region:: 28 | 29 | export AWS_DEFAULT_REGION='us-east-1' 30 | 31 | Disable host key checking:: 32 | 33 | export ANSIBLE_HOST_KEY_CHECKING=False 34 | 35 | Sanity test the ec2.py dynamic environment script:: 36 | 37 | $ cd inventory 38 | $ ./ec2.py --list 39 | { 40 | "_meta": { 41 | "hostvars": {} 42 | } 43 | } 44 | $ cd .. 45 | 46 | Note: The ec2.py inventory script fails silently if AWS_ACCESS_KEY_ID is not set. 47 | 48 | Galaxy Components: 49 | ------------------ 50 | 51 | Grab some external modules from Ansible Galaxy:: 52 | 53 | mkdir galaxy 54 | $ ansible-galaxy -p galaxy install jdauphant.nginx 55 | - downloading role 'nginx', owned by jdauphant 56 | - downloading role from https://github.com/jdauphant/ansible-role-nginx/archive/v1.1.1.tar.gz 57 | - extracting jdauphant.nginx to galaxy/jdauphant.nginx 58 | - jdauphant.nginx was installed successfully 59 | 60 | [ansible-dc-ec2-tutorial] bschott@ironman-2 ~/Source/ansible-dc-ec2-tutorial (master) 61 | $ ansible-galaxy -p galaxy install flmmartins.postgres 62 | - downloading role 'postgres', owned by flmmartins 63 | - downloading role from https://github.com/flmmartins/ansible-postgres/archive/master.tar.gz 64 | - extracting flmmartins.postgres to galaxy/flmmartins.postgres 65 | - flmmartins.postgres was installed successfully 66 | 67 | Provisioning Example: 68 | --------------------- 69 | 70 | Start the provision_instances playbook:: 71 | 72 | $ ansible-playbook -i ./inventory provision_instances.yaml 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | hostfile = inventory 3 | host_key_checking = false 4 | roles_path = galaxy 5 | private_key_file = keys/sitekey 6 | remote_user = ubuntu 7 | -------------------------------------------------------------------------------- /inventory/ec2.ini: -------------------------------------------------------------------------------- 1 | # Ansible EC2 external inventory script settings 2 | # 3 | 4 | [ec2] 5 | 6 | # to talk to a private eucalyptus instance uncomment these lines 7 | # and edit edit eucalyptus_host to be the host name of your cloud controller 8 | #eucalyptus = True 9 | #eucalyptus_host = clc.cloud.domain.org 10 | 11 | # AWS regions to make calls to. Set this to 'all' to make request to all regions 12 | # in AWS and merge the results together. Alternatively, set this to a comma 13 | # separated list of regions. E.g. 'us-east-1,us-west-1,us-west-2' 14 | regions = all 15 | regions_exclude = us-gov-west-1,cn-north-1 16 | 17 | # When generating inventory, Ansible needs to know how to address a server. 18 | # Each EC2 instance has a lot of variables associated with it. Here is the list: 19 | # http://docs.pythonboto.org/en/latest/ref/ec2.html#module-boto.ec2.instance 20 | # Below are 2 variables that are used as the address of a server: 21 | # - destination_variable 22 | # - vpc_destination_variable 23 | 24 | # This is the normal destination variable to use. If you are running Ansible 25 | # from outside EC2, then 'public_dns_name' makes the most sense. If you are 26 | # running Ansible from within EC2, then perhaps you want to use the internal 27 | # address, and should set this to 'private_dns_name'. 28 | destination_variable = public_dns_name 29 | 30 | # For server inside a VPC, using DNS names may not make sense. When an instance 31 | # has 'subnet_id' set, this variable is used. If the subnet is public, setting 32 | # this to 'ip_address' will return the public IP address. For instances in a 33 | # private subnet, this should be set to 'private_ip_address', and Ansible must 34 | # be run from with EC2. 35 | vpc_destination_variable = ip_address 36 | 37 | # To tag instances on EC2 with the resource records that point to them from 38 | # Route53, uncomment and set 'route53' to True. 39 | route53 = False 40 | 41 | # To exclude RDS instances from the inventory, uncomment and set to False. 42 | #rds = False 43 | 44 | # Additionally, you can specify the list of zones to exclude looking up in 45 | # 'route53_excluded_zones' as a comma-separated list. 46 | # route53_excluded_zones = samplezone1.com, samplezone2.com 47 | 48 | # By default, only EC2 instances in the 'running' state are returned. Set 49 | # 'all_instances' to True to return all instances regardless of state. 50 | all_instances = False 51 | 52 | # By default, only RDS instances in the 'available' state are returned. Set 53 | # 'all_rds_instances' to True return all RDS instances regardless of state. 54 | all_rds_instances = False 55 | 56 | # API calls to EC2 are slow. For this reason, we cache the results of an API 57 | # call. Set this to the path you want cache files to be written to. Two files 58 | # will be written to this directory: 59 | # - ansible-ec2.cache 60 | # - ansible-ec2.index 61 | cache_path = ~/.ansible/tmp 62 | 63 | # The number of seconds a cache file is considered valid. After this many 64 | # seconds, a new API call will be made, and the cache file will be updated. 65 | # To disable the cache, set this value to 0 66 | cache_max_age = 120 67 | 68 | # Organize groups into a nested/hierarchy instead of a flat namespace. 69 | nested_groups = False 70 | 71 | # If you only want to include hosts that match a certain regular expression 72 | # pattern_include = stage-* 73 | 74 | # If you want to exclude any hosts that match a certain regular expression 75 | # pattern_exclude = stage-* 76 | 77 | # Instance filters can be used to control which instances are retrieved for 78 | # inventory. For the full list of possible filters, please read the EC2 API 79 | # docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters 80 | # Filters are key/value pairs separated by '=', to list multiple filters use 81 | # a list separated by commas. See examples below. 82 | 83 | # Retrieve only instances with (key=value) env=stage tag 84 | # instance_filters = tag:env=stage 85 | 86 | # Retrieve only instances with role=webservers OR role=dbservers tag 87 | # instance_filters = tag:role=webservers,tag:role=dbservers 88 | 89 | # Retrieve only t1.micro instances OR instances with tag env=stage 90 | # instance_filters = instance-type=t1.micro,tag:env=stage 91 | 92 | # You can use wildcards in filter values also. Below will list instances which 93 | # tag Name value matches webservers1* 94 | # (ex. webservers15, webservers1a, webservers123 etc) 95 | # instance_filters = tag:Name=webservers1* 96 | -------------------------------------------------------------------------------- /inventory/ec2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | EC2 external inventory script 5 | ================================= 6 | 7 | Generates inventory that Ansible can understand by making API request to 8 | AWS EC2 using the Boto library. 9 | 10 | NOTE: This script assumes Ansible is being executed where the environment 11 | variables needed for Boto have already been set: 12 | export AWS_ACCESS_KEY_ID='AK123' 13 | export AWS_SECRET_ACCESS_KEY='abc123' 14 | 15 | This script also assumes there is an ec2.ini file alongside it. To specify a 16 | different path to ec2.ini, define the EC2_INI_PATH environment variable: 17 | 18 | export EC2_INI_PATH=/path/to/my_ec2.ini 19 | 20 | If you're using eucalyptus you need to set the above variables and 21 | you need to define: 22 | 23 | export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus 24 | 25 | For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html 26 | 27 | When run against a specific host, this script returns the following variables: 28 | - ec2_ami_launch_index 29 | - ec2_architecture 30 | - ec2_association 31 | - ec2_attachTime 32 | - ec2_attachment 33 | - ec2_attachmentId 34 | - ec2_client_token 35 | - ec2_deleteOnTermination 36 | - ec2_description 37 | - ec2_deviceIndex 38 | - ec2_dns_name 39 | - ec2_eventsSet 40 | - ec2_group_name 41 | - ec2_hypervisor 42 | - ec2_id 43 | - ec2_image_id 44 | - ec2_instanceState 45 | - ec2_instance_type 46 | - ec2_ipOwnerId 47 | - ec2_ip_address 48 | - ec2_item 49 | - ec2_kernel 50 | - ec2_key_name 51 | - ec2_launch_time 52 | - ec2_monitored 53 | - ec2_monitoring 54 | - ec2_networkInterfaceId 55 | - ec2_ownerId 56 | - ec2_persistent 57 | - ec2_placement 58 | - ec2_platform 59 | - ec2_previous_state 60 | - ec2_private_dns_name 61 | - ec2_private_ip_address 62 | - ec2_publicIp 63 | - ec2_public_dns_name 64 | - ec2_ramdisk 65 | - ec2_reason 66 | - ec2_region 67 | - ec2_requester_id 68 | - ec2_root_device_name 69 | - ec2_root_device_type 70 | - ec2_security_group_ids 71 | - ec2_security_group_names 72 | - ec2_shutdown_state 73 | - ec2_sourceDestCheck 74 | - ec2_spot_instance_request_id 75 | - ec2_state 76 | - ec2_state_code 77 | - ec2_state_reason 78 | - ec2_status 79 | - ec2_subnet_id 80 | - ec2_tenancy 81 | - ec2_virtualization_type 82 | - ec2_vpc_id 83 | 84 | These variables are pulled out of a boto.ec2.instance object. There is a lack of 85 | consistency with variable spellings (camelCase and underscores) since this 86 | just loops through all variables the object exposes. It is preferred to use the 87 | ones with underscores when multiple exist. 88 | 89 | In addition, if an instance has AWS Tags associated with it, each tag is a new 90 | variable named: 91 | - ec2_tag_[Key] = [Value] 92 | 93 | Security groups are comma-separated in 'ec2_security_group_ids' and 94 | 'ec2_security_group_names'. 95 | ''' 96 | 97 | # (c) 2012, Peter Sankauskas 98 | # 99 | # This file is part of Ansible, 100 | # 101 | # Ansible is free software: you can redistribute it and/or modify 102 | # it under the terms of the GNU General Public License as published by 103 | # the Free Software Foundation, either version 3 of the License, or 104 | # (at your option) any later version. 105 | # 106 | # Ansible is distributed in the hope that it will be useful, 107 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 108 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 109 | # GNU General Public License for more details. 110 | # 111 | # You should have received a copy of the GNU General Public License 112 | # along with Ansible. If not, see . 113 | 114 | ###################################################################### 115 | 116 | import sys 117 | import os 118 | import argparse 119 | import re 120 | from time import time 121 | import boto 122 | from boto import ec2 123 | from boto import rds 124 | from boto import route53 125 | import ConfigParser 126 | from collections import defaultdict 127 | 128 | try: 129 | import json 130 | except ImportError: 131 | import simplejson as json 132 | 133 | 134 | class Ec2Inventory(object): 135 | def _empty_inventory(self): 136 | return {"_meta" : {"hostvars" : {}}} 137 | 138 | def __init__(self): 139 | ''' Main execution path ''' 140 | 141 | # Inventory grouped by instance IDs, tags, security groups, regions, 142 | # and availability zones 143 | self.inventory = self._empty_inventory() 144 | 145 | # Index of hostname (address) to instance ID 146 | self.index = {} 147 | 148 | # Read settings and parse CLI arguments 149 | self.read_settings() 150 | self.parse_cli_args() 151 | 152 | # Cache 153 | if self.args.refresh_cache: 154 | self.do_api_calls_update_cache() 155 | elif not self.is_cache_valid(): 156 | self.do_api_calls_update_cache() 157 | 158 | # Data to print 159 | if self.args.host: 160 | data_to_print = self.get_host_info() 161 | 162 | elif self.args.list: 163 | # Display list of instances for inventory 164 | if self.inventory == self._empty_inventory(): 165 | data_to_print = self.get_inventory_from_cache() 166 | else: 167 | data_to_print = self.json_format_dict(self.inventory, True) 168 | 169 | print data_to_print 170 | 171 | 172 | def is_cache_valid(self): 173 | ''' Determines if the cache files have expired, or if it is still valid ''' 174 | 175 | if os.path.isfile(self.cache_path_cache): 176 | mod_time = os.path.getmtime(self.cache_path_cache) 177 | current_time = time() 178 | if (mod_time + self.cache_max_age) > current_time: 179 | if os.path.isfile(self.cache_path_index): 180 | return True 181 | 182 | return False 183 | 184 | 185 | def read_settings(self): 186 | ''' Reads the settings from the ec2.ini file ''' 187 | 188 | config = ConfigParser.SafeConfigParser() 189 | ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini') 190 | ec2_ini_path = os.environ.get('EC2_INI_PATH', ec2_default_ini_path) 191 | config.read(ec2_ini_path) 192 | 193 | # is eucalyptus? 194 | self.eucalyptus_host = None 195 | self.eucalyptus = False 196 | if config.has_option('ec2', 'eucalyptus'): 197 | self.eucalyptus = config.getboolean('ec2', 'eucalyptus') 198 | if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): 199 | self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') 200 | 201 | # Regions 202 | self.regions = [] 203 | configRegions = config.get('ec2', 'regions') 204 | configRegions_exclude = config.get('ec2', 'regions_exclude') 205 | if (configRegions == 'all'): 206 | if self.eucalyptus_host: 207 | self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) 208 | else: 209 | for regionInfo in ec2.regions(): 210 | if regionInfo.name not in configRegions_exclude: 211 | self.regions.append(regionInfo.name) 212 | else: 213 | self.regions = configRegions.split(",") 214 | 215 | # Destination addresses 216 | self.destination_variable = config.get('ec2', 'destination_variable') 217 | self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') 218 | 219 | # Route53 220 | self.route53_enabled = config.getboolean('ec2', 'route53') 221 | self.route53_excluded_zones = [] 222 | if config.has_option('ec2', 'route53_excluded_zones'): 223 | self.route53_excluded_zones.extend( 224 | config.get('ec2', 'route53_excluded_zones', '').split(',')) 225 | 226 | # Include RDS instances? 227 | self.rds_enabled = True 228 | if config.has_option('ec2', 'rds'): 229 | self.rds_enabled = config.getboolean('ec2', 'rds') 230 | 231 | # Return all EC2 and RDS instances (if RDS is enabled) 232 | if config.has_option('ec2', 'all_instances'): 233 | self.all_instances = config.getboolean('ec2', 'all_instances') 234 | else: 235 | self.all_instances = False 236 | if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled: 237 | self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances') 238 | else: 239 | self.all_rds_instances = False 240 | 241 | # Cache related 242 | cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) 243 | if not os.path.exists(cache_dir): 244 | os.makedirs(cache_dir) 245 | 246 | self.cache_path_cache = cache_dir + "/ansible-ec2.cache" 247 | self.cache_path_index = cache_dir + "/ansible-ec2.index" 248 | self.cache_max_age = config.getint('ec2', 'cache_max_age') 249 | 250 | # Configure nested groups instead of flat namespace. 251 | if config.has_option('ec2', 'nested_groups'): 252 | self.nested_groups = config.getboolean('ec2', 'nested_groups') 253 | else: 254 | self.nested_groups = False 255 | 256 | # Do we need to just include hosts that match a pattern? 257 | try: 258 | pattern_include = config.get('ec2', 'pattern_include') 259 | if pattern_include and len(pattern_include) > 0: 260 | self.pattern_include = re.compile(pattern_include) 261 | else: 262 | self.pattern_include = None 263 | except ConfigParser.NoOptionError, e: 264 | self.pattern_include = None 265 | 266 | # Do we need to exclude hosts that match a pattern? 267 | try: 268 | pattern_exclude = config.get('ec2', 'pattern_exclude'); 269 | if pattern_exclude and len(pattern_exclude) > 0: 270 | self.pattern_exclude = re.compile(pattern_exclude) 271 | else: 272 | self.pattern_exclude = None 273 | except ConfigParser.NoOptionError, e: 274 | self.pattern_exclude = None 275 | 276 | # Instance filters (see boto and EC2 API docs) 277 | self.ec2_instance_filters = defaultdict(list) 278 | if config.has_option('ec2', 'instance_filters'): 279 | for x in config.get('ec2', 'instance_filters', '').split(','): 280 | filter_key, filter_value = x.split('=') 281 | self.ec2_instance_filters[filter_key].append(filter_value) 282 | 283 | def parse_cli_args(self): 284 | ''' Command line argument processing ''' 285 | 286 | parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2') 287 | parser.add_argument('--list', action='store_true', default=True, 288 | help='List instances (default: True)') 289 | parser.add_argument('--host', action='store', 290 | help='Get all the variables about a specific instance') 291 | parser.add_argument('--refresh-cache', action='store_true', default=False, 292 | help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') 293 | self.args = parser.parse_args() 294 | 295 | 296 | def do_api_calls_update_cache(self): 297 | ''' Do API calls to each region, and save data in cache files ''' 298 | 299 | if self.route53_enabled: 300 | self.get_route53_records() 301 | 302 | for region in self.regions: 303 | self.get_instances_by_region(region) 304 | if self.rds_enabled: 305 | self.get_rds_instances_by_region(region) 306 | 307 | self.write_to_cache(self.inventory, self.cache_path_cache) 308 | self.write_to_cache(self.index, self.cache_path_index) 309 | 310 | 311 | def get_instances_by_region(self, region): 312 | ''' Makes an AWS EC2 API call to the list of instances in a particular 313 | region ''' 314 | 315 | try: 316 | if self.eucalyptus: 317 | conn = boto.connect_euca(host=self.eucalyptus_host) 318 | conn.APIVersion = '2010-08-31' 319 | else: 320 | conn = ec2.connect_to_region(region) 321 | 322 | # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported 323 | if conn is None: 324 | print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) 325 | sys.exit(1) 326 | 327 | reservations = [] 328 | if self.ec2_instance_filters: 329 | for filter_key, filter_values in self.ec2_instance_filters.iteritems(): 330 | reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values })) 331 | else: 332 | reservations = conn.get_all_instances() 333 | 334 | for reservation in reservations: 335 | for instance in reservation.instances: 336 | self.add_instance(instance, region) 337 | 338 | except boto.exception.BotoServerError, e: 339 | if not self.eucalyptus: 340 | print "Looks like AWS is down again:" 341 | print e 342 | sys.exit(1) 343 | 344 | def get_rds_instances_by_region(self, region): 345 | ''' Makes an AWS API call to the list of RDS instances in a particular 346 | region ''' 347 | 348 | try: 349 | conn = rds.connect_to_region(region) 350 | if conn: 351 | instances = conn.get_all_dbinstances() 352 | for instance in instances: 353 | self.add_rds_instance(instance, region) 354 | except boto.exception.BotoServerError, e: 355 | if not e.reason == "Forbidden": 356 | print "Looks like AWS RDS is down: " 357 | print e 358 | sys.exit(1) 359 | 360 | def get_instance(self, region, instance_id): 361 | ''' Gets details about a specific instance ''' 362 | if self.eucalyptus: 363 | conn = boto.connect_euca(self.eucalyptus_host) 364 | conn.APIVersion = '2010-08-31' 365 | else: 366 | conn = ec2.connect_to_region(region) 367 | 368 | # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported 369 | if conn is None: 370 | print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) 371 | sys.exit(1) 372 | 373 | reservations = conn.get_all_instances([instance_id]) 374 | for reservation in reservations: 375 | for instance in reservation.instances: 376 | return instance 377 | 378 | def add_instance(self, instance, region): 379 | ''' Adds an instance to the inventory and index, as long as it is 380 | addressable ''' 381 | 382 | # Only want running instances unless all_instances is True 383 | if not self.all_instances and instance.state != 'running': 384 | return 385 | 386 | # Select the best destination address 387 | if instance.subnet_id: 388 | dest = getattr(instance, self.vpc_destination_variable) 389 | else: 390 | dest = getattr(instance, self.destination_variable) 391 | 392 | if not dest: 393 | # Skip instances we cannot address (e.g. private VPC subnet) 394 | return 395 | 396 | # if we only want to include hosts that match a pattern, skip those that don't 397 | if self.pattern_include and not self.pattern_include.match(dest): 398 | return 399 | 400 | # if we need to exclude hosts that match a pattern, skip those 401 | if self.pattern_exclude and self.pattern_exclude.match(dest): 402 | return 403 | 404 | # Add to index 405 | self.index[dest] = [region, instance.id] 406 | 407 | # Inventory: Group by instance ID (always a group of 1) 408 | self.inventory[instance.id] = [dest] 409 | if self.nested_groups: 410 | self.push_group(self.inventory, 'instances', instance.id) 411 | 412 | # Inventory: Group by region 413 | if self.nested_groups: 414 | self.push_group(self.inventory, 'regions', region) 415 | else: 416 | self.push(self.inventory, region, dest) 417 | 418 | # Inventory: Group by availability zone 419 | self.push(self.inventory, instance.placement, dest) 420 | if self.nested_groups: 421 | self.push_group(self.inventory, region, instance.placement) 422 | 423 | # Inventory: Group by instance type 424 | type_name = self.to_safe('type_' + instance.instance_type) 425 | self.push(self.inventory, type_name, dest) 426 | if self.nested_groups: 427 | self.push_group(self.inventory, 'types', type_name) 428 | 429 | # Inventory: Group by key pair 430 | if instance.key_name: 431 | key_name = self.to_safe('key_' + instance.key_name) 432 | self.push(self.inventory, key_name, dest) 433 | if self.nested_groups: 434 | self.push_group(self.inventory, 'keys', key_name) 435 | 436 | # Inventory: Group by security group 437 | try: 438 | for group in instance.groups: 439 | key = self.to_safe("security_group_" + group.name) 440 | self.push(self.inventory, key, dest) 441 | if self.nested_groups: 442 | self.push_group(self.inventory, 'security_groups', key) 443 | except AttributeError: 444 | print 'Package boto seems a bit older.' 445 | print 'Please upgrade boto >= 2.3.0.' 446 | sys.exit(1) 447 | 448 | # Inventory: Group by tag keys 449 | for k, v in instance.tags.iteritems(): 450 | key = self.to_safe("tag_" + k + "=" + v) 451 | self.push(self.inventory, key, dest) 452 | if self.nested_groups: 453 | self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) 454 | self.push_group(self.inventory, self.to_safe("tag_" + k), key) 455 | 456 | # Inventory: Group by Route53 domain names if enabled 457 | if self.route53_enabled: 458 | route53_names = self.get_instance_route53_names(instance) 459 | for name in route53_names: 460 | self.push(self.inventory, name, dest) 461 | if self.nested_groups: 462 | self.push_group(self.inventory, 'route53', name) 463 | 464 | # Global Tag: tag all EC2 instances 465 | self.push(self.inventory, 'ec2', dest) 466 | 467 | self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) 468 | 469 | 470 | def add_rds_instance(self, instance, region): 471 | ''' Adds an RDS instance to the inventory and index, as long as it is 472 | addressable ''' 473 | 474 | # Only want available instances unless all_rds_instances is True 475 | if not self.all_rds_instances and instance.status != 'available': 476 | return 477 | 478 | # Select the best destination address 479 | #if instance.subnet_id: 480 | #dest = getattr(instance, self.vpc_destination_variable) 481 | #else: 482 | #dest = getattr(instance, self.destination_variable) 483 | dest = instance.endpoint[0] 484 | 485 | if not dest: 486 | # Skip instances we cannot address (e.g. private VPC subnet) 487 | return 488 | 489 | # Add to index 490 | self.index[dest] = [region, instance.id] 491 | 492 | # Inventory: Group by instance ID (always a group of 1) 493 | self.inventory[instance.id] = [dest] 494 | if self.nested_groups: 495 | self.push_group(self.inventory, 'instances', instance.id) 496 | 497 | # Inventory: Group by region 498 | if self.nested_groups: 499 | self.push_group(self.inventory, 'regions', region) 500 | else: 501 | self.push(self.inventory, region, dest) 502 | 503 | # Inventory: Group by availability zone 504 | self.push(self.inventory, instance.availability_zone, dest) 505 | if self.nested_groups: 506 | self.push_group(self.inventory, region, instance.availability_zone) 507 | 508 | # Inventory: Group by instance type 509 | type_name = self.to_safe('type_' + instance.instance_class) 510 | self.push(self.inventory, type_name, dest) 511 | if self.nested_groups: 512 | self.push_group(self.inventory, 'types', type_name) 513 | 514 | # Inventory: Group by security group 515 | try: 516 | if instance.security_group: 517 | key = self.to_safe("security_group_" + instance.security_group.name) 518 | self.push(self.inventory, key, dest) 519 | if self.nested_groups: 520 | self.push_group(self.inventory, 'security_groups', key) 521 | 522 | except AttributeError: 523 | print 'Package boto seems a bit older.' 524 | print 'Please upgrade boto >= 2.3.0.' 525 | sys.exit(1) 526 | 527 | # Inventory: Group by engine 528 | self.push(self.inventory, self.to_safe("rds_" + instance.engine), dest) 529 | if self.nested_groups: 530 | self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine)) 531 | 532 | # Inventory: Group by parameter group 533 | self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest) 534 | if self.nested_groups: 535 | self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name)) 536 | 537 | # Global Tag: all RDS instances 538 | self.push(self.inventory, 'rds', dest) 539 | 540 | self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) 541 | 542 | 543 | def get_route53_records(self): 544 | ''' Get and store the map of resource records to domain names that 545 | point to them. ''' 546 | 547 | r53_conn = route53.Route53Connection() 548 | all_zones = r53_conn.get_zones() 549 | 550 | route53_zones = [ zone for zone in all_zones if zone.name[:-1] 551 | not in self.route53_excluded_zones ] 552 | 553 | self.route53_records = {} 554 | 555 | for zone in route53_zones: 556 | rrsets = r53_conn.get_all_rrsets(zone.id) 557 | 558 | for record_set in rrsets: 559 | record_name = record_set.name 560 | 561 | if record_name.endswith('.'): 562 | record_name = record_name[:-1] 563 | 564 | for resource in record_set.resource_records: 565 | self.route53_records.setdefault(resource, set()) 566 | self.route53_records[resource].add(record_name) 567 | 568 | 569 | def get_instance_route53_names(self, instance): 570 | ''' Check if an instance is referenced in the records we have from 571 | Route53. If it is, return the list of domain names pointing to said 572 | instance. If nothing points to it, return an empty list. ''' 573 | 574 | instance_attributes = [ 'public_dns_name', 'private_dns_name', 575 | 'ip_address', 'private_ip_address' ] 576 | 577 | name_list = set() 578 | 579 | for attrib in instance_attributes: 580 | try: 581 | value = getattr(instance, attrib) 582 | except AttributeError: 583 | continue 584 | 585 | if value in self.route53_records: 586 | name_list.update(self.route53_records[value]) 587 | 588 | return list(name_list) 589 | 590 | 591 | def get_host_info_dict_from_instance(self, instance): 592 | instance_vars = {} 593 | for key in vars(instance): 594 | value = getattr(instance, key) 595 | key = self.to_safe('ec2_' + key) 596 | 597 | # Handle complex types 598 | # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518 599 | if key == 'ec2__state': 600 | instance_vars['ec2_state'] = instance.state or '' 601 | instance_vars['ec2_state_code'] = instance.state_code 602 | elif key == 'ec2__previous_state': 603 | instance_vars['ec2_previous_state'] = instance.previous_state or '' 604 | instance_vars['ec2_previous_state_code'] = instance.previous_state_code 605 | elif type(value) in [int, bool]: 606 | instance_vars[key] = value 607 | elif type(value) in [str, unicode]: 608 | instance_vars[key] = value.strip() 609 | elif type(value) == type(None): 610 | instance_vars[key] = '' 611 | elif key == 'ec2_region': 612 | instance_vars[key] = value.name 613 | elif key == 'ec2__placement': 614 | instance_vars['ec2_placement'] = value.zone 615 | elif key == 'ec2_tags': 616 | for k, v in value.iteritems(): 617 | key = self.to_safe('ec2_tag_' + k) 618 | instance_vars[key] = v 619 | elif key == 'ec2_groups': 620 | group_ids = [] 621 | group_names = [] 622 | for group in value: 623 | group_ids.append(group.id) 624 | group_names.append(group.name) 625 | instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids]) 626 | instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names]) 627 | else: 628 | pass 629 | # TODO Product codes if someone finds them useful 630 | #print key 631 | #print type(value) 632 | #print value 633 | 634 | return instance_vars 635 | 636 | def get_host_info(self): 637 | ''' Get variables about a specific host ''' 638 | 639 | if len(self.index) == 0: 640 | # Need to load index from cache 641 | self.load_index_from_cache() 642 | 643 | if not self.args.host in self.index: 644 | # try updating the cache 645 | self.do_api_calls_update_cache() 646 | if not self.args.host in self.index: 647 | # host migh not exist anymore 648 | return self.json_format_dict({}, True) 649 | 650 | (region, instance_id) = self.index[self.args.host] 651 | 652 | instance = self.get_instance(region, instance_id) 653 | return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 654 | 655 | def push(self, my_dict, key, element): 656 | ''' Push an element onto an array that may not have been defined in 657 | the dict ''' 658 | group_info = my_dict.setdefault(key, []) 659 | if isinstance(group_info, dict): 660 | host_list = group_info.setdefault('hosts', []) 661 | host_list.append(element) 662 | else: 663 | group_info.append(element) 664 | 665 | def push_group(self, my_dict, key, element): 666 | ''' Push a group as a child of another group. ''' 667 | parent_group = my_dict.setdefault(key, {}) 668 | if not isinstance(parent_group, dict): 669 | parent_group = my_dict[key] = {'hosts': parent_group} 670 | child_groups = parent_group.setdefault('children', []) 671 | if element not in child_groups: 672 | child_groups.append(element) 673 | 674 | def get_inventory_from_cache(self): 675 | ''' Reads the inventory from the cache file and returns it as a JSON 676 | object ''' 677 | 678 | cache = open(self.cache_path_cache, 'r') 679 | json_inventory = cache.read() 680 | return json_inventory 681 | 682 | 683 | def load_index_from_cache(self): 684 | ''' Reads the index from the cache file sets self.index ''' 685 | 686 | cache = open(self.cache_path_index, 'r') 687 | json_index = cache.read() 688 | self.index = json.loads(json_index) 689 | 690 | 691 | def write_to_cache(self, data, filename): 692 | ''' Writes data in JSON format to a file ''' 693 | 694 | json_data = self.json_format_dict(data, True) 695 | cache = open(filename, 'w') 696 | cache.write(json_data) 697 | cache.close() 698 | 699 | 700 | def to_safe(self, word): 701 | ''' Converts 'bad' characters in a string to underscores so they can be 702 | used as Ansible groups ''' 703 | 704 | return re.sub("[^A-Za-z0-9\-]", "_", word) 705 | 706 | 707 | def json_format_dict(self, data, pretty=False): 708 | ''' Converts a dict to a JSON object and dumps it as a formatted 709 | string ''' 710 | 711 | if pretty: 712 | return json.dumps(data, sort_keys=True, indent=2) 713 | else: 714 | return json.dumps(data) 715 | 716 | 717 | # Run the script 718 | Ec2Inventory() 719 | 720 | -------------------------------------------------------------------------------- /inventory/localhost: -------------------------------------------------------------------------------- 1 | localhost ansible_python_interpreter=~/.virtualenvs/ansible-dc-ec2-tutorial/bin/python 2 | -------------------------------------------------------------------------------- /provision_instances.yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook -i inventory 2 | --- 3 | 4 | - name: Provision the instances 5 | hosts: localhost 6 | gather_facts: false 7 | connection: local 8 | tasks: 9 | - include: tasks/keypair.yaml name="sitekey" 10 | - include: tasks/securitygroup.yaml name="webservers" 11 | - include: tasks/securitygroup.yaml name="dbservers" 12 | - include: tasks/instance.yaml name="web1" securitygroup="webservers" keypair="sitekey" 13 | - include: tasks/instance.yaml name="web2" securitygroup="webservers" keypair="sitekey" 14 | - include: tasks/instance.yaml name="db1" securitygroup="dbservers" keypair="sitekey" 15 | - include: tasks/instance.yaml name="db2" securitygroup="dbservers" keypair="sitekey" 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto 2 | ansible 3 | awscli 4 | -------------------------------------------------------------------------------- /roles/hostname-configured/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: set hostname to tag name 4 | hostname: name={{ ec2_tag_Name }} 5 | when: ec2_tag_Name is defined 6 | -------------------------------------------------------------------------------- /roles/ssh-reachable/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: wait for ssh to be available 4 | local_action: wait_for host="{{ host|default(ec2_public_dns_name) }}" 5 | port="{{ port|default(22) }}" state="started" 6 | sudo: False 7 | -------------------------------------------------------------------------------- /roles/ubuntu-updated/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # update ubuntu to the latest packages 3 | 4 | - name: upgrade packages 5 | apt: > 6 | install_recommends=no 7 | cache_valid_time=3600 8 | update_cache=yes 9 | upgrade=yes 10 | 11 | -------------------------------------------------------------------------------- /setup_instances.yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook 2 | --- 3 | 4 | - name: Configure common packages 5 | hosts: 6 | - security_group_webservers 7 | - security_group_dbservers 8 | sudo: true 9 | roles: 10 | - ssh-reachable 11 | - ubuntu-updated 12 | - hostname-configured 13 | 14 | - name: Configure webserver packages 15 | hosts: security_group_webservers 16 | sudo: true 17 | roles: 18 | - role: jdauphant.nginx 19 | nginx_http_params: 20 | - sendfile "on" 21 | - access_log "/var/log/nginx/access.log" 22 | nginx_sites: 23 | bar: 24 | - listen 8080 25 | - location / { try_files $uri $uri/ /index.html; } 26 | - location /images/ { try_files $uri $uri/ /index.html; } 27 | nginx_configs: 28 | proxy: 29 | - proxy_set_header X-Real-IP $remote_addr 30 | - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for 31 | 32 | - name: Configure dbserver packages 33 | hosts: security_group_dbservers 34 | sudo: true 35 | roles: 36 | - { role: ANXS.postgresql } 37 | -------------------------------------------------------------------------------- /tasks/instance.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: search for an ami image 3 | ec2_ami_search: > 4 | region={{ region|default('us-east-1') }} 5 | store={{ store|default('ebs') }} 6 | virt={{ virt|default('hvm') }} 7 | distro={{ distro|default('ubuntu') }} 8 | release={{ release|default('precise') }} 9 | register: ami_data 10 | 11 | - name: debug ami image 12 | debug: 'msg="{{ ami_data }}"' 13 | 14 | - name: start the ec2 instance 15 | local_action: 16 | module: ec2 17 | key_name: "{{ keypair|default('mykey') }}" 18 | group: "{{ securitygroup|default('mygroup') }}" 19 | instance_type: "{{ instancetype|default('t2.micro')}}" 20 | image: "{{ ami_data.ami }}" 21 | region: "{{ region|default('us-east-1') }}" 22 | wait: no 23 | instance_tags: 24 | Name: "{{ name|default('myinstance')}}" 25 | foo: bar 26 | register: instance_data 27 | 28 | - name: debug instance start 29 | debug: 'msg="{{instance_data }}"' 30 | -------------------------------------------------------------------------------- /tasks/keypair.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: make sure local keys directory exists 4 | local_action: 5 | module: file 6 | state: directory 7 | path: "{{ playbook_dir }}/keys" 8 | 9 | - name: remove existing keypair when force_keypair=True 10 | local_action: 11 | module: ec2_key 12 | name: "{{ name|default('tutorial') }}" 13 | region: "{{ region|default('us-east-1') }}" 14 | state: absent 15 | when: force_keypair|default(False) 16 | 17 | - name: make a keypair with the specified name 18 | local_action: 19 | module: ec2_key 20 | name: "{{ name|default('tutorial') }}" 21 | region: "{{ region|default('us-east-1') }}" 22 | wait: yes 23 | register: key_data 24 | 25 | - name: save the key file to local host 26 | local_action: 27 | module: copy 28 | content: "{{ key_data.key.private_key }}" 29 | dest: "keys/{{ name|default('tutorial') }}" 30 | mode: 0600 31 | when: key_data.key.private_key|default(0) 32 | 33 | - name: debug keypair 34 | debug: msg="{{ key_data }}" 35 | -------------------------------------------------------------------------------- /tasks/securitygroup.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: create the security group 4 | local_action: 5 | module: ec2_group 6 | name: "{{ name|default('mygroup') }}" 7 | description: "ansible generated group" 8 | region: "{{ region|default('us-east-1') }}" 9 | rules: 10 | - proto: tcp 11 | from_port: 80 12 | to_port: 80 13 | cidr_ip: 0.0.0.0/0 14 | - proto: tcp 15 | from_port: 443 16 | to_port: 443 17 | cidr_ip: 0.0.0.0/0 18 | - proto: tcp 19 | from_port: 22 20 | to_port: 22 21 | cidr_ip: 0.0.0.0/0 22 | rules_egress: 23 | - proto: all 24 | cidr_ip: "0.0.0.0/0" 25 | register: securitygroup_data 26 | 27 | - name: debug security group 28 | debug: "msg='{{ securitygroup_data }}'" 29 | -------------------------------------------------------------------------------- /terminate_instances.yaml: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ansible-playbook 2 | --- 3 | 4 | - name: Terminate instances 5 | hosts: 6 | - security_group_webservers 7 | - security_group_dbservers 8 | connection: local 9 | gather_facts: false 10 | tasks: 11 | - name: Terminate instances that were previously launched 12 | local_action: 13 | module: ec2 14 | region: '{{ ec2_region }}' 15 | instance_ids: '{{ ec2_id }}' 16 | state: 'absent' 17 | 18 | --------------------------------------------------------------------------------