. }}" 44 | [section] 45 | variable=value 46 | - example with adhoc command: 47 | ansible -m setup -a 'filter=ansible_local' 48 | - automatic "magic" variables that all hosts have: 49 | {{ hostvars }} <-- access variables for another host 50 | {{ group_names }} <-- list (array) of all the groups the current host is in 51 | {{ groups }} <-- list of all the groups (and hosts) in the inventory 52 | {{ inventory_hostname }} <-- hostname as configured in Ansible’s inventory host file 53 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 54 | 55 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 56 | Inclusion 57 | - include: tasks/db.yml 58 | - vars_files <-- added immediately when play starts 59 | - include_vars: vars/variables_db.yml <-- added when play reaches this point 60 | - host_vars <-- folder contain files named afer hosts 61 | - group_vars <-- folder contain files named after groups, can be 'all' 62 | - include_role <-- helps with sequentially running tasks (instead of pre_tasks/roles/post_tasks) 63 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 64 | 65 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 66 | Loops 67 | "{{ item.name }}" 68 | "{{ item.sex }}" 69 | with_items: 70 | - { name:'a', sex:'m' } 71 | - { name:'b', sex:'f' } 72 | 73 | Loops - nested 74 | "{{ item[0] }}" 75 | "{{ item[1] }}" 76 | with_nested: 77 | - ['a', 'b'] 78 | - ['id', 'sex'] 79 | 80 | Loops - other 81 | with_file <-- iterates over the content of a list of files, item will be set to the content of each file in sequence 82 | with_fileglob <-- matches all files in a single directory, non-recursively, that match a pattern 83 | with_sequence <-- generates a sequence of items 84 | with_random_choice 85 | with_dict 86 | with_subelements 87 | 88 | Loop over vars file containing linux groups and their corresponding users: 89 | wheel: 90 | - john 91 | - paul 92 | - margaret 93 | docker: 94 | - mark 95 | Solution: 96 | - debug: 97 | msg: "I am creating user {{ item.name }} in a group {{ item.group }}" 98 | with_items: 99 | - { group: 'wheel', name: "{{ wheel }}" } 100 | - { group: 'docker', name: "{{ docker }}" } 101 | 102 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 103 | 104 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 105 | Conditionals 106 | when: 107 | == 108 | != 109 | < 110 | <= 111 | > 112 | >= 113 | is defined 114 | is not defined 115 | <-- is true 116 | not <-- is false 117 | in <-- variable is present in other variable's list 118 | 119 | Multiple conditions: and, or 120 | result.rc == 0 <-- test exit codes of the previous command output 121 | e.g. #1 122 | when: inventory_hostname in groups['webservers'] and "(ansible_memory_mb.real.total) > (memory)" 123 | e.g. #2 124 | - stat: 125 | path: /.../... 126 | register: OUTPUT 127 | - shell: mv ... ... 128 | when: OUTPUT.stat.exists 129 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 130 | 131 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 132 | Handlers 133 | TRIGGERED AT THE END OF A BLOCK OF TASKS IN PLAYBOOK AND ONLY IF TASK=CHANGED 134 | notify: restart_apache 135 | handlers: 136 | - name: restart_apache 137 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 138 | 139 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 140 | Tags 141 | tags: 142 | - packages 143 | - always <-- special one! 144 | ansible-playbook <>.yml --tags 'packages' 145 | ansible-playbook <>.yml --skip-tags 'packages' 146 | Special flags for the above command: 'tagged', 'untagged', 'all' 147 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 148 | 149 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 150 | Errors 151 | ignore_errors: yes 152 | force_handlers: yes <-- even if status:failed 153 | failed_when: <-- it runs the task first! 154 | e.g. #1 155 | register: cmd_result 156 | failed_when: "'Password' in cmd_result.stdout" 157 | changed_when: <-- if you know the task itself will never change the host, use 'false' here 158 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 159 | 160 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 161 | Blocks 162 | - block: 163 | rescue: <-- run if the 'block:' fails 164 | always: 165 | BLOCKS IS A NICE WAY OF SOLVING HANDLERS - YOU CAN CALL HANDLER IN BETWEEN THE BLOCKS. 166 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 167 | 168 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 169 | Jinja2 170 | {% EXPRESSION %} <-- for logic 171 | {# COMMENT #} 172 | {{ ANSIBLE EXPRESSION }} <-- for result output 173 | e.g. 174 | {% for... %} 175 | {{...}} 176 | {% endfor %} 177 | {% if... %} 178 | {{...}} 179 | {% endif %} 180 | {{ output | to_json }} 181 | | to_yaml 182 | | to_nice_json 183 | | to_nice_yaml 184 | | from_json 185 | | from_yaml 186 | Special variable: {{ ansible_managed }} 187 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 188 | 189 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 190 | Roles 191 | ANSIBLE LOOKS FOR SUBDIRECTORY "ROLES" INSIDE THE PROJECT, OR REFERENCE BY "roles_path" IN 'ansible.cfg' 192 | NORMALLY, THE TASKS OF ROLES EXECUTE BEFORE THE TASKS OF THE PLAYBOOKS THAT USE THEM. USE PRE_/POST_ FOR IT: 193 | - hosts: all 194 | pre_tasks: 195 | roles: 196 | - role1 197 | - role2 198 | tasks: 199 | post_tasks: 200 | 201 | Roles with overwriting variables 202 | - hosts: all 203 | roles: 204 | - role: role1 205 | var: value1 206 | - role: role2 207 | var: value2 208 | 209 | Role dependencies 210 | ./meta/main.yml 211 | e.g. 212 | --- 213 | dependencies: 214 | - { role: myfw, myvar: httpd } 215 | 216 | Selectively run tasks in play: 217 | --- 218 | - hosts: node1, node2 219 | tasks: 220 | - name: ... 221 | - hosts: node3, node4 222 | tasks: 223 | - name: ... 224 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 225 | 226 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 227 | Galaxy 228 | ansible-galaxy search 'text' --author --platforms --galaxy-tags 229 | ansible-galaxy info 230 | ansible-galaxy install -p <-- without '-p' it will use the 'roles_path' 231 | ansible-galaxy installs -r 232 | ansible-galaxy init --offline 233 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 234 | 235 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 236 | Optimizations 237 | Inventory special group: 'ungrouped' 238 | Wildcard host pattern: 239 | ansible '*.example.com' -i myinventory --list-hosts 240 | ansible '192.168.2.*' -i myinventory --list-hosts 241 | ansible 'datacenter*' -i myinventory --list-hosts 242 | 243 | IF YOU PROVIDE COMMA-SEPARATED LIST OF MANAGED HOSTS OR GROUPS, THEY WILL BE TARGETED 244 | IF THE ITEM STARTS WITH '&' THEN HOSTS MUST MATCH THAT ITEM IN ORDER TO MATCH HOST PATTERN (LOGICAL 'AND'): 245 | ansible 'lab,&datacenter' ... <-- 'lab' group only if it is also in 'datacenter' group 246 | ansible -i ec2.py tag_prod&tag_webserver -m ping <-- only webservers in prod 247 | EXCLUDING HOSTS USING '!' (LOGICAL 'NOT'): 248 | ansible 'datacenter,!test.lab.example.com' <-- 'datacenter' group with exception of 'test.lab.example.com' 249 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 250 | 251 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 252 | Delegation 253 | e.g. #1 254 | delegate_to: localhost <-- performed on host running Ansible 255 | e.g. #2 256 | delegate_to: loadbalancer-host 257 | e.g. #3 258 | delegate_to: "{{ item }}" 259 | with_items: "{{ groups['proxyservers'] }} <-- if it exists in the inventory 260 | - add_host: <-- if it doesn't exist in the invetory, you need to add it 261 | name: DEMO 262 | ansible_host: 172.25.250.10 263 | ansible_user: devops 264 | name: ... 265 | delegate_to: DEMO 266 | delegate_facts: true <-- assign gathered facts to delegated hosts 267 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 268 | 269 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 270 | Parallelism 271 | BY DEFAULT ANSIBLE FORKS UP TO 5 TIMES: forks = 5, --forks=5 272 | TEMPORARILY REDUCE THE NUMBER OF MACHINES RUNNING IN PARALLEL FROM THE FORK COUNT: serial: 2 273 | RUN THE JOB IN THE BACKGROUND AND CHECK BACK LATER: 274 | async: 3600 <-- wait 3600s = 1 hour. async can be '1' 275 | poll: 10 <-- monitor once every 10 seconds for completion. poll can be '0' 276 | e.g. #1 277 | - wait_for: 278 | host: "{{ inventory_hostname }}" 279 | state: started 280 | delay: 30 281 | timeout: 300 282 | port: 22 283 | delegate_to: localhost 284 | become: false 285 | e.g. #2 286 | - async_status: 287 | jid: "{{ register_async.ansible_job_id }}" 288 | register: JOB_RESULT 289 | until: JOB_RESULT.finished 290 | retries: 30 <-- this is not in seconds! 291 | e.g. #3 292 | - name: run the script ... 293 | register: SCRIPT_SLEEPER 294 | - name: check the script 295 | async_status: "jid={{ item.ansible_job_id }}" 296 | register: JOB_RESULT 297 | until: JOB_RESULT.finished 298 | retries: 30 299 | with_items: "{{ SCRIPT_SLEEPER.results }}" 300 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 301 | 302 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 303 | Vault 304 | ansible-vault create 305 | ansible-vault edit 306 | ansible-vault rekey 307 | ansible-vault encrypt ... 308 | ansible-vault view 309 | ansible-vault decrypt --output=decrypted_file.yml 310 | ansible-playbook --ask-vault-pass <.yml> 311 | ansible-playbook --vault-password-file= 312 | $ANSIBLE_VAULT_PASSWORD_FILE 313 | PLAINTEXT PASSWORDS SHOULD GENERALLY BE CONVERTED TO HASH: 314 | "{{ 'passwd' | password_hash('sha512') }}" 315 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 316 | 317 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 318 | Troubleshooting 319 | 'log_path' in 'ansible.cfg' 320 | $ANSIBLE_LOG_PATH 321 | 'verbosity:' can be added to the individual 'debug' task 322 | ansible-playbook ... --step 323 | ansible-playbook ... --start-at-task 324 | ansible-playbook ... --check / -C 325 | ansible-playbook ... --diff 326 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 327 | 328 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 329 | Tower 330 | Customized 'ansible.cfg' should be in the root of project folder and Tower will automatically use it. 331 | You should put 'ansible_become=true' in the 'inventory' if you are installing it on mutiple machines. 332 | If you are using localhost, do NOT change the default setting of 'rabbitmq_use_long_name=false' to 'true'. 333 | For the HA environment: 334 | - minimum of three nodes (one for housekeeping, one for user jobs, one will statistically fail). 335 | - if any service fail, all services are restarted. If this happens multiple times, the entire node will be placed offline. 336 | - license does not care how many Tower nodes you have, but rather how many nodes are you managing with the cluster. 337 | - isolated node(s) are headless Tower(s), only running Ansible(s) in a remote destination acting as a "bastion" 338 | System capacity formula: 339 | No. forks + ((RAM/1024)-2) * 75, but also keep in mind 2 CPUS per 20 forks 340 | e.g.: 5+((4096/1024)-2)*75=155 341 | tower-cli: pip install ansible-tower-cli 342 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 343 | 344 | ####################################### 345 | ## MISTAKES I MADE IN THE FINAL LABS ## 346 | ####################################### 347 | Chapter 13 - Comprehensive review: 348 | - Lab1: Deploying Ansible 349 | - Lab2: Creating playbooks 350 | - Lab3: Creating roles and using dynamic inventory 351 | - Lab4: Optimizing Ansible 352 | - Lab5: Deploying Ansible Tower and executing jobs 353 | 354 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 355 | - firewalld: 356 | immediate: yes 357 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 358 | 359 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 360 | - serial: 1 361 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 362 | 363 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 364 | - haproxy: 365 | state: disabled 366 | backend: app 367 | host: "{{ inventory_hostname }}" 368 | socket: /var/lib/proxy/stats 369 | wait: yes 370 | delegate_to: "{{ item }}" 371 | with_items: "{{ groups.lbservers }}" 372 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 373 | 374 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 375 | - shell: /bin/sleep 5 && shutdown now 376 | async: 1 377 | poll: 0 378 | ignore_errors: true 379 | when: pageupgrade.changed 380 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 381 | 382 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 383 | - 'mount:' module actually edits /etc/fstab 384 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 385 | 386 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 387 | - ansible dev -m copy -a 'content="text" dest=/etc/motd' -b -u devops 388 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 389 | 390 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 391 | - when installing Ansible Tower, do it as root on tower.lab.example.com, not from the workstation (if it's a single node install) 392 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 393 | 394 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 395 | - 'unarchive:' module can use http:// as the 'src:' 396 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 397 | 398 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 399 | - IF YOU USE TRAILING SLASH AFTER THE SOURCE DIR IN 'copy:' MODULE, IT WILL COPY THE ENTIRE CONTENT: 400 | - name: ... 401 | copy: 402 | src: html/ 403 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 404 | 405 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 406 | SOME LESS KNOWN ANSIBLE MODULES WORTH EXPLORING 407 | ansible-doc uri <-- interacts with HTTP/HTTPS endpoints 408 | ansible-doc fail <-- fails the progress with a custom message 409 | ansible-doc script <-- fails if rc != 0 410 | ansible-doc stat <-- retrieves facts for a file similar to the linux/unix 'stat' command 411 | ansible-doc assert <-- asserts that given expressions are true with an optional custom message 412 | ansible-doc replace <-- replace all instances of a pattern within a file 413 | !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 414 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ansible-cheatsheet 2 | 3 | ## Filelist 4 | 5 | - `DO407-exam-notes.txt` - only few notes for the Ansible exam that are hard to remember 6 | - `ansible-cheatsheet.txt` - general notes from 2018-12-04 7 | - `ec2.ini` - AWS dynamic inventory configuration file 8 | - `ec2.py` - AWS dynamic inventory script, slightly modified from the default one 9 | 10 | ## Stargazers over time 11 | 12 | [![Stargazers over time](https://starchart.cc/luckylittle/ansible-cheatsheet.svg)](https://starchart.cc/luckylittle/ansible-cheatsheet) 13 | 14 | --- 15 | 16 | _Last update: Thu Feb 20 04:13:02 UTC 2020_ 17 | -------------------------------------------------------------------------------- /ansible-cheatsheet.txt: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ## Lucian Maly 2018-12-04 3 | ############################################################################### 4 | 5 | ## Ansible works best if the passwordless access is configured on all nodes: 6 | $ ssh-keygen 7 | $ ssh-copy-id lmaly@ 8 | $ ssh lmaly@ 9 | 10 | ## Configuration, Inventory default(s): 11 | $ cat /etc/ansible/hosts 12 | $ grep host_file ansible.cfg 13 | Priority in which the config files are processed: 14 | 1) ANSIBLE_CONFIG (an environment variable) 15 | 2) ./ansible.cfg (in the current directory) 16 | 3) ~/.ansible.cfg 17 | 4) /etc/ansible/ansible.cfg 18 | $ ansible --version <-shows what config file is currently being used 19 | 20 | Commonly Modified Settings in 'ansible.cfg': 21 | inventory= Location of Ansible inventory file 22 | remote user= Remote user account used to establish connections to managed hosts 23 | become= Enables/disables privilege escalation for operations on managed hosts (NOT BY DEFAULT!) 24 | become_method= Defines privilege escalation method 25 | become_user= User account for escalating privileges 26 | become_ask_pass= Defines whether privilege escalation prompts for password 27 | 28 | # Default module (command, NOT SHELL!) is defined in 'ansible.cfg' under 'defaults' section: 29 | $ grep module_name /etc/ansible/ansible.cfg 30 | !If no modules defined, predefined command module used 31 | 32 | ############################################################################### 33 | ## Run the playbook(s): 34 | $ ansible -i localhost, .yaml 35 | !When using the '-i' parameter, if the value is a 'list' (it contains at least one comma) it will be used as the inventory LIST 36 | !It can be either FQDN or IP address 37 | $ ansible -i /etc/ansible/hosts .yaml 38 | !While the variable is a string, it will be used as the inventory PATH 39 | 40 | ## Ansible verbosity: 41 | $ ansible-playbook -v -i .yaml <-displays output data 42 | $ ansible-playbook -vv -i .yaml <-displays output & input data 43 | $ ansible-playbook -vvv -i .yaml <-information about connections 44 | $ ansible-playbook -vvvv -i .yaml <-information about plugins 45 | 46 | ############################################################################### 47 | ## Inventory file examples: 48 | [webserver] <-both hosts can be reduced to ws[01:02].lmaly.io using regexp 49 | ws01.lmaly.io 50 | ws02.lmaly.io:1234 <-configured to listen on port 1234 51 | 52 | [database] 53 | db01.lmaly.io 54 | 192.168.[1:5].[0:255] <-includes 192.168.1.0/24 through 192.168.5.0/24 55 | 56 | [nsw_cities:children] <-nested groups use the keyword :children 57 | webserver 58 | database 59 | others 60 | 61 | [others] 62 | localhost ansible_connection=local <-it will use local connection, without SSH 63 | 192.168.3.7 ansible_connection=ssh ansible_user=ftaylor <-use different settings example 64 | 65 | # Dynamic inventory - AWS custom script 66 | !Script must support '--list' an '--host' parameters, e.g.: 67 | $ ./inventiryscript --list 68 | $ sudo cp ec2.py /etc/ansible/ 69 | $ sudo cp ec2.ini /etc/ansible/ 70 | $ sudo chmod +x /etc/ansible/ec2.py 71 | $ ansible -i ec2.py all -m ping 72 | 73 | !Multiple inventory files in one dir = All the files in the directory are merged and treated as one inventory 74 | 75 | ############################################################################### 76 | ## Which hosts will the playbook run against? 77 | $ ansible-playbook .yaml --list-hosts 78 | 79 | ## Which hosts are in the hosts file? 80 | $ ansible all --list-hosts 81 | $ ansible '192.168.2.*' -i --list-hosts <-everything that matches the wildcard 82 | $ ansible lab:datacenter1 -i --list-hosts <-members of lab OR datacenter1 83 | $ ansible 'lab:&datacenter1' -i --list-hosts <-members of lab AND datacenter1 84 | $ ansible 'datacenter:!test2.example.com' -i --list-hosts <-exclude host 85 | 86 | ## Print all the tasks in short form that playbook would perform: 87 | $ ansible-playbook .yaml --list-tasks 88 | 89 | ############################################################################### 90 | ## See the metadata obtained during Ansible setup (module setup): 91 | $ ansible all -i , -m setup 92 | $ ansible -m setup 93 | $ ansible -m setup --tree facts 94 | # Show the specific metadata: 95 | $ ansible -m setup -a 'filter=*ipv4*' 96 | # Examples: 97 | {{ ansible_hostname }} 98 | {{ ansible_default_ipv4.address }} 99 | {{ ansible_devices.vda.partitions.vda1.size }} 100 | {{ ansible_dns.nameservers }} 101 | {{ ansible_kernel }} 102 | # To disable facts: 103 | gather_facts: no 104 | # Custom facts are saved under ansible_local: 105 | $ cat /etc/ansible/facts.d/new.fact <-ini or json file 106 | $ ansible demo1.example.com -m setup -a 'filter=ansible_local' 107 | # Accessing facts of another node: 108 | {{ hostvars['demo2.example.com']['ansible_os_family'] }} 109 | 110 | ############################################################################### 111 | # Run arbitrary command on all hosts as sudo: 112 | $ ansible all -s -a "cat /var/log/messages" 113 | 114 | # Run Ad Hoc module with arguments: 115 | $ ansible host pattern -m module -a 'argument1 argument2' [-i inventory] 116 | 117 | # Run ad hoc command that generates one line input for each operation: 118 | $ ansible myhosts -m command -a /usr/bin/hostname -o 119 | 120 | # Ad hoc Example: yum module checks if httpd installed on demohost: 121 | $ ansible demohost -u devops -b -m yum -a 'name=httpd state=present' 122 | 123 | # Ad hoc Example: Find available space on disks configured on demohost: 124 | $ ansible demohost -a "df -h" 125 | 126 | # Ad hoc Example: Find available free memory on demohost: 127 | $ ansible demohost -a "free -m" 128 | 129 | ############################################################################### 130 | ## Playbook(s) basics: 131 | --- <-indicates YAML, optional document marker 132 | - hosts: all <-host or host group against we will run the task (example - hosts: webserver) 133 | remote_user: lmaly <-as what user will Ansible log in to the machine(s) 134 | tasks: <-list of actions 135 | - name: Whatever you want to name it <-name of the first task 136 | yum: 137 | name: httpd 138 | state: present <-same as single line 'yum: name=httpd state=present become=True' 139 | become: True <-task will be executed with sudo 140 | ... <-optional document marker indicating end of YAML 141 | 142 | Notes: 143 | !Space characters used for indentation 144 | !Pipe (|) preservers line returns within string 145 | !Arrow (>) converts line returns to spaces, removes leading space in lines 146 | !Indentation rules: 147 | Elements at same level in hierarchy must have same indentation 148 | Child elements must be indented further than parents 149 | No rules about exact number of spaces to use 150 | !Optional: Insert blank lines for readability 151 | !Use YAML Lint http://yamllint.com to check the syntax correctness 152 | !Use 'ansible-playbook --syntax-check ' 153 | !Use 'ansible-playbook -C ' for dry run (what changes would occur if playbook is executed?) 154 | !Use 'always_run: true' to execute only some tasks in check mode (or 'always_run: false' for the opposite) 155 | !Use 'ansible-playbook --step ' to prompt each task (y=yes/n=no/c=exit and execute remaining without asking) 156 | !Use 'ansible-playbook --start-at-task="start httpd service" to start execution from given task 157 | !For complex playbooks, use 'include' to include separate files in main playbook (include: tasks/env.yml) 158 | 159 | ############################################################################### 160 | ## Variables: 161 | --- # Usually comment field describing what it does 162 | - hosts: all 163 | remote_user: lmaly 164 | tasks: 165 | - name: Set variable 'name' 166 | set_fact: 167 | name: Test machine 168 | - name: Print variable 'name' 169 | debug: 170 | msg: '{{ name }}' 171 | 172 | a/ In vars block: 173 | vars: 174 | set_fact: 'Test machine' 175 | 176 | b/ Passed as arguments: 177 | include_vars: vars/extra_args.yml 178 | 179 | # Other alternative ways: 180 | (1) pass variable(s) in the CLI: 181 | $ ansible-playbook -i , .yaml -e 'name=test01' 182 | 183 | (2) pass variable(s) to an inventory file: 184 | a/ 185 | [webserver] <- all playbooks running on webservers will be able to refer to the domain name variable 186 | ws01.lmaly.io domainname=example1.lmaly.io 187 | ws02.lmaly.io domainname=example2.lmaly.io 188 | 189 | b/ 190 | [webserver:vars] <- ! Host variables will override group variables in case tge same variable is used in both 191 | https_enabled=True 192 | 193 | (3) in variable file(s): 194 | a/ host_vars: 195 | $ cat host_vars/ws01.lmaly.io 196 | domainname=example1.lmaly.io 197 | 198 | b/ group_vars: 199 | $ cat group_vars/webserver 200 | https_enabled=True 201 | 202 | !Variable must start with letter; valid characters are: letters, numbers, underscores 203 | # Array definition example: 204 | users: 205 | bjones: 206 | first_name: Bob 207 | last_name: Jones 208 | acook: 209 | first_name: Anne 210 | last_name: Cook 211 | # Array accessing/reading example: 212 | users.bjones.first_name # Returns 'Bob' 213 | users['bjones']['first_name'] # Returns 'Bob' <-preferable method 214 | 215 | # Inventory variables hierarchy/scope 216 | !Three levels: Global, Play, Host 217 | !If Ansible finds variables with the same name, it uses chain of preference (highest priority on the top): 218 | ^ A) Common variable file - overrides everything, host/group/inventory variable files 219 | | 01) 'extra' variables via CLI (-e) 220 | | 02) Task variables (only for task itself) 221 | | 03) Defined in 'block' statement (- block:) 222 | | 04) 'role' and 'include' variables 223 | | 05) Included using 'vars_files' (vars_files: - /vars/env.yml) 224 | | 06) 'vars_prompt' variables (vars_prompt:) 225 | | 07) Defined with '-a' or '--args' command line 226 | | 08) Defined via 'set_facts' (- set_fact: user: joe) 227 | | 09) Registered variables with 'register' keyword for debugging 228 | | 10) Host facts discovered by Ansible (ansible_facts) 229 | | B) Host variables - overrides group variable files 230 | | 11) 'host_vars' in host_vars directory 231 | | C) Group variables - overrides inventory variable files 232 | | 12) 'group_vars' in group_vars directory 233 | | D) Inventory variable files - lowest priority 234 | | 13) 'host_vars' in the inventory file ([hostgroup:vars]) 235 | | 14) 'group_vars' in the inventory file ([hostgroup:children]) 236 | | 15) Inventory file variables - global 237 | | 16) Role default variables - Set in roles vars directory 238 | 239 | !Variables passed to the CLI have priority over any other variable. 240 | !You can override the following parameters from an inventory file: 241 | ansible_user, ansible_port, ansible_host, ansible_connection, ansible_private_key_file, ansible_shell_type 242 | 243 | ############################################################################### 244 | ## Iterates (http://docs.ansible.com/ansible/playbooks_loops.html) 245 | (1) Standard - 'with_items' example: 246 | a/ Simple loop 247 | firewall 248 | firewalld: 249 | service: '{{ item }}' 250 | state: enabled 251 | permanent: True 252 | immediate: True 253 | become: True 254 | with_items: 255 | - http 256 | - https 257 | 258 | b/ List of hashes 259 | - user: 260 | name: {{ item.name }} 261 | state: present 262 | groups: {{ item.groups }} 263 | with_items: 264 | - { name: 'jane', groups: 'wheel' } 265 | - { name: 'joe', groups: 'root' } 266 | 267 | (2) Nested loops (iterate all elements of a list with all items from other lists) - 'with_nested' example: 268 | user: 269 | name: '{{ item }}' 270 | become: True 271 | with_items: 272 | - '{{ users }}' 273 | file: 274 | path: '/home/{{ item.0 }}/{{ item.1 }}' 275 | state: directory 276 | become: True 277 | with_nested: 278 | - '{{ users }}' 279 | - '{{ folders }}' 280 | 281 | (3) Fileglobs loop (action on every file present in a certain folder) - 'with_fileglobs' example: 282 | copy: 283 | src: '{{ item }}' 284 | dest: '/tmp/iproute2' 285 | remote_src: True 286 | become: True 287 | with_fileglob: 288 | - '/etc/iproute2/rt_*' 289 | 290 | (4) Integer loop (iterate over the integer numbers) - 'with_sequence' example: 291 | file: 292 | dest: '/tmp/dir{{ item }}' 293 | state: directory 294 | with_sequence: start=1 end=10 295 | become: True 296 | 297 | Notes: 298 | with_file - Takes list of file names; 'item' set to content of each file in sequence 299 | with_fileglob - Takes file name globbing pattern; 'item' set to each file in directory that matches pattern; in sequence, nonrecursively 300 | with_sequence - Generates sequence of items in increasing numerical order; Can take 'start' and 'end' arguments 301 | - Supports decimal, octal, hexadecimal integer values 302 | with_random_choices - Takes list; 'item' set to one list item at random 303 | 304 | Note - Conditionals: 305 | Equal "{{ max_memory }} == 512" 306 | Less than "{{ min_memory }} < 128" 307 | Greater than "{{ min_memory }} > 256" 308 | Less than or equal "{{ min_memory }} <= 256" 309 | Greater than or equal "{{ min_memory }} >= 512" 310 | Not equal "{{ min_memory }} != 512" 311 | Variable exists "{{ min_memory }} is defined" 312 | Variable does not exist "{{ min_memory }} is not defined" 313 | Variable set to 1, True, or yes "{{ available_memory }}" 314 | Variable set to 0, False, or no "not {{ available_memory }}" 315 | Value present in variable or array "{{ users }} in users["db_admins"]" 316 | AND ansible_kernel == 3.10.el7.x86_64 and inventory_hostname in groups['staging'] 317 | OR ansible_distribution == "RedHat" or ansible_distribution == "Fedora" 318 | 319 | (5) 'when' statement example 320 | - name: Create the database admin 321 | user: 322 | name: db_admin 323 | when: inventory_hostname in groups["databases"] <-when is not a module variable, it must be placed outside of the module 324 | 325 | (6) When combining 'when' with 'with_items', the when statement is processed for each item: 326 | with_items: "{ ansible_mounts }" 327 | when: item.mount == "/" and item.size_available > 300000000 328 | 329 | ############################################################################### 330 | ## Register - To capture output of module that uses array 331 | - shell: echo "{{ item }}" 332 | with_items: 333 | - one 334 | - two 335 | register: echo 336 | 337 | ############################################################################### 338 | ## Handlers - Task that responds to notification triggered by other tasks: 339 | notify: 340 | - restart_apache 341 | handlers: 342 | -name: restart_apache 343 | service: 344 | name: httpd: 345 | state: restarted 346 | !Task may call more than one handler in notify 347 | !Ansible treats notify as an array and iterates over handler names 348 | !Handlers run in order they are written in play, not in order listed by notify task 349 | !If two handlers have same name, only one runs 350 | !Handler runs only once, even if notified by more than one task 351 | !Handlers defined inside 'include' cannot be notified 352 | !--force-handlers 353 | 354 | ############################################################################### 355 | ## Resource tagging 356 | a/ Tasks 357 | tags: 358 | - packages 359 | b/ Roles - all tasks they define also tagged 360 | roles: 361 | - { role: user, tags: [production] } 362 | c/ Include - all tasks they define also tagged 363 | - include: common.yml 364 | tags: [testing] 365 | $ ansible-playbook --tags "production" 366 | $ ansible-playbook --skip-tags "development" 367 | Special tags: always, tagged, untagged, all (default) 368 | 369 | ############################################################################### 370 | ## Ignore and task errors 371 | Default: If task fails, play is aborted immediatelly, same with handlers 372 | To skip failed tasks, use 'ignore_errors: yes' 373 | To force handler to be called even if task fails, use 'force_handlers: yes' 374 | To mark task as failed even when it succeeds, use 'failed_when: 375 | cmd: /usr/local/bin/create_user.show 376 | register: command_result 377 | failed_when: "'Password missing' in command_result.stdout" 378 | Default: Task acquires 'changed' state after updating managed host, which causes handlers skipped if task does not make changed 379 | changed_when: "'Success' in command_result.stdout" 380 | 381 | ############################################################################### 382 | ## Environmental variables 383 | 384 | a/ Set env: 385 | environment: 386 | http_proxy: http://demo.lab.example.com:8080 387 | 388 | b/ Reference vars inside playbook: 389 | - block: 390 | environment: 391 | name: "{{ myname }}" 392 | 393 | ############################################################################### 394 | ## Blocks - logical grouping of tasks, controls how are they executed 395 | a/ block: <-main tasks to run 396 | b/ rescue: <-tasks to run if block tasks fail 397 | c/ always: <-tasks to always run, independent on a/ or b/ 398 | tasks: 399 | - block: 400 | - shell: 401 | cmd: /usr/local/lib/upgrade-database 402 | rescue: 403 | - shell: 404 | cmd: /usr/local/lib/create-user 405 | always: 406 | - service: 407 | name: mariadb 408 | state: restarted 409 | 410 | ############################################################################### 411 | ## Delegation using 'delegate_to' and 'local_action' 412 | 413 | a/ To localhost using delegate_to: 414 | - name: Running Local Process 415 | command: ps 416 | delegate_to: localhost 417 | register: local_process 418 | 419 | b/ To localhost using local_action: 420 | - name: Running Local Process 421 | local_action: command 'ps' 422 | register: local_process 423 | 424 | c/ Host outside play in inventory - example of removing managed hosts from ELB, upgrading and the adding them back: 425 | - hosts: webservers 426 | tasks: 427 | - name: Remove server from load balancer 428 | command: remove-from-lb {{ inventory_hostname }} 429 | delegate_to: localhost 430 | - name: Deploy the latest version of Webstack 431 | git:repo=git://git.example.com/path/repo.git dest=/srv/www 432 | - name: Add server back to ELB pool 433 | command: add-to-lb {{ inventory_hostname }} 434 | delegate_to: localhost 435 | Inventory data used when creating connection to target: 436 | ansible_connection 437 | ansible_host 438 | ansible_port 439 | ansible_user 440 | 441 | d/ Host outside play NOT in inventory using 'add_host': 442 | - name: Add delegation host 443 | add_host: name=demo ansible_host=172.25.250.10 ansible_user=devops 444 | - name: Silly echo 445 | command: echo {{ inventory_hostname }} 446 | delegate_to: demo 447 | args: <-complex arguments can be added 448 | chdir: somedir/ 449 | creates: /path/to/database 450 | 451 | e/ 'run_once: True' 452 | 453 | !Facts gathered by delegated tasks assigned to 'delegate_to' host, not host that produced facts 454 | !To gather facts from delegated host, set 'delegate_facts: True' in the play 455 | 456 | ############################################################################### 457 | ## Jinja2 Template(s) - no need for file extension: 458 | --- 459 | - hosts: all 460 | remote_user: lmaly 461 | tasks: 462 | - name: Ensure the website is present and updated 463 | template: 464 | src: index.html.j2 <-the J2 template source 465 | dest: /var/www/html/index.html <-destination 466 | owner: root 467 | group: root 468 | mode: 0644 469 | become: True 470 | 471 | # Variables in J2 template(s): 472 | '{{ VARIABLE_NAME }}' 473 | '{{ ARRAY_NAME['KEY'] }}' 474 | '{{ OBJECT_NAME.PROPERTY_NAME }}' 475 | 476 | # Example of index.html.j2 using variable: 477 | # {{ ansible_managed }} <-recommended to include it in each template 478 | 479 | 480 |

Hello World!

481 |

This page was created on {{ ansible_date_time.date }}.

482 | 483 | 484 | 485 | # Comments: 486 | {# ... #} 487 | 488 | # Built-in Filtres in J2 template(s): 489 | '{{ VARIABLE_NAME | capitalize }}' 490 | '{{ output | to_json }} 491 | '{{ output | to_yaml }} 492 | '{{ output | to_nice_json }} 493 | '{{ output | to_nice_yaml }} 494 | '{{ output | from_json }} 495 | '{{ output | from_yaml }} 496 | '{{ forest_blockers|split('-') }}' 497 | 498 | # Conditionals: 499 | { % ... % } 500 | a/ Equal to example A 501 | {% if ansible_eth0.active == True %} 502 |

eth0 address {{ ansible_eth0.ipv4.address }}.

503 | {% endif %} 504 | 505 | b/ Equal to example B 506 | {% if ansible_eth0.active is equalto True %} 507 |

eth0 address {{ ansible_eth0.ipv4.address }}.

508 | {% endif %} 509 | 510 | # Cycles/loops: 511 | {% for address in ansible_all_ipv4_addresses %} 512 |

513 | {% endfor %} 514 | 515 | # Issues 516 | a/ When value after : starts with { you need to " the whole object 517 | app_path: "{{ base_path }}/bin" 518 | b/ When you have nested {{...}} elements, remove the inner set: 519 | msg: Host {{ params[{{ host_ip }}] }} <-wrong 520 | msg: Host {{ params[ host_ip] }} <-fine 521 | 522 | More information: 523 | http://jinja.pocoo.org/docs/dev/templates/#builtin-filters 524 | http://jinja.pocoo.org/docs/dev/templates/#builtin-tests 525 | 526 | ############################################################################### 527 | ## Roles 528 | # Structure: 529 | It looks for different roles at 'roles' subdirectory or 'roles_path' dir in ansible.cfg (default /etc/ansible/roles) 530 | Top-level = specifically named role name 531 | Subdirs = main.yml 532 | files = contains objects referenced by main.yml 533 | templates = contains objects referenced by main.yml 534 | !Role tasks execute before tasks of play in which they appear 535 | !To override default, use 'pre_tasks' (performed before any roles applied) and 'post_tasks' (performed after all roles completed) 536 | 537 | # Example - folder structure: 538 | $ tree user.example 539 | user.example/ 540 | ├── defaults 541 | │ └── main.yml <-define default variables, easily overriden 542 | ├── files <-fixed-content files, empty subdir is ignored 543 | ├── handlers 544 | │ └── main.yml 545 | ├── meta 546 | │ └── main.yml <-define dependency roles ('allow_duplicates: yes') 547 | ├── README.md 548 | ├── tasks 549 | │ └── main.yml <-role content (defines modules to call on managed hosts where the role is applied 550 | ├── templates <-contain templates 551 | ├── tests 552 | │ ├── inventory 553 | │ └── test.yml 554 | └── vars 555 | │ └── main.yml <-vars for this module (best practice) 556 | └── main.yml 557 | 558 | # Use roles in play example: 559 | --- 560 | - hosts: remote.example.com 561 | roles: 562 | - role1 563 | - role2 564 | - davidkarban.git <-role from Ansible Galaxy using _AUTHOR.NAME_ 565 | 566 | # Override default variables example: 567 | --- 568 | - hosts: remote.example.com 569 | roles: 570 | - { role: role1 } 571 | - { role: role2, var1: val1, var2: val2 } 572 | 573 | # Define dependency roles in meta/main.yml example: 574 | --- 575 | dependencies: 576 | - { role: apache, port: 8080 } 577 | - { role: postgress, dbname: serverlist, admin_user: felix } 578 | !Role added as dependency to play once, to override default, set 'allow_duplicates=yes' in meta/main.yml 579 | 580 | # Content example of MOTD role (tasks/main.yml) 581 | --- 582 | # tasks file for MOTD 583 | - name: deliver motd file 584 | template: 585 | src: templates/motd.j2 586 | dest: /etc/motd 587 | owner: root 588 | group: root 589 | mode: 0444 590 | 591 | # Role sources 592 | $ cat roles2install.yml 593 | # From Galaxy: 594 | - src: author.rolename 595 | 596 | # From different source: 597 | - src: https://webserver.example.com/files.tgz 598 | name: ftpserver-role 599 | $ ansible-galaxy init -r roles2install.yml 600 | 601 | 602 | ############################################################################### 603 | ## Modules 604 | # Custom modules 605 | Priority in which the custom module is being processed: 606 | 1, ANSIBLE_LIBRARY environment variable 607 | 2, 'library' in the ansible.cfg 608 | 3, ./library/ relative to location of playbook in use 609 | # Default modules 610 | $ cd /usr/lib/python2.7/site-packages/ansible/modules 611 | $ ansible-doc -l <-list all modules 612 | $ ansible-doc <-help for the module 613 | $ ansible-doc -s <-simple view for the module 614 | 615 | ############################################################################### 616 | ## Optimizations 617 | (1) Default 'smart' settings (transport=smart in ansible.cfg): 618 | a/ check if locally installed SSH supports 'ControlPersist', if not, 'paramiko' is used 619 | b/ ControlPersist=60s (listed as comment in /etc/ansible/ansible.cfg) 620 | (2) Other settings include: 621 | a/ paramiko (Python implementation of SSHv2, does not have ControlPersist) 622 | b/ local (runs locally and not over SSH) 623 | c/ ssh (uses OpenSSH) 624 | d/ docker (uses docker exec) 625 | e/ plug-ins (not based on SSH, e.g. chroot, libvirt_lxc...) 626 | (3) Change on-the-fly settings with 627 | a/ $ ansible-playbook -c 628 | b/ $ ansible -c 629 | (4) SSH settings are under [ssh_connection] 630 | (5) Paramiko settings are under [paramiko_connection] 631 | (6) To specify connection type in the inventory file, use 'ansible_connection': 632 | [targets] 633 | localgost ansible_connection=local 634 | demo.lab.example.com ansible_connection=ssh 635 | (7) To specify connection type in play: 636 | --- 637 | - name: Connection type 638 | hosts: 127.0.0.01 639 | connection: local 640 | (8) To limit concurrent connection, use SSH server's 'MaxStartups' option 641 | (9) Parallelism - by default 5 different machines at once: 642 | a/ setting 'forks' in 'ansible.cfg' 643 | b/ $ ansible-playbook --forks 644 | c/ 'serial' in the play overrides 'ansible.cfg' - either number or % 645 | d/ 'async' & 'async_status' - value is time that Ansible waits for command to complete [default 3600s, long tasks 0] 646 | e/ 'pool' - sets how often Ansible checks if command has completed [default 10s, long tasks 0] 647 | f/ 'wait_for' 648 | g/ pause module 649 | 650 | ############################################################################### 651 | ## Ansible vault 652 | (1a) Create encrypted file: 653 | $ ansible-vault create 654 | $ ansible-vault create --vault-password-file=.secret_file 655 | (2a) Enter and confirm new vault password 656 | 657 | (1b) Or encrypt and existing file(s): 658 | $ ansible-vault encrypt ... 659 | $ ansible-vault encrypt --output=NEW_FILE 660 | (2b) Enter and confirm new vault password 661 | 662 | (3) View file: 663 | $ ansible-vault view 664 | (4) Edit file: 665 | $ ansible-vault edit 666 | (5) Change password: 667 | $ ansible-vault rekey 668 | $ ansible-vault rekey --new-vault-password-file=.secret_file 669 | (6) Decrypt file: 670 | $ ansible-vault decrypt --output= 671 | Variable types: 672 | a/ Defined in 'group_vars' or 'host_vars' 673 | b/ Loaded by 'include_vars' or 'vars_files' 674 | c/ Passed on 'ansible-playbook -e @file.yml' 675 | d/ Defined as role variables & defaults 676 | 677 | $ ansible-playbook --ask-vault-pass 678 | $ ansible-playbook --vault-password-file=.secret_file 679 | or 'EXPORT ANSIBLE_VAULT_PASSWORD_FILE=~/.secret_file' 680 | 681 | ############################################################################### 682 | ## Troubleshooting 683 | By default no log, but you can enable it: 684 | a/ 'log_path' parameter under [default] in 'ansible.cfg' 685 | b/ ANSIBLE_LOG_PATH environment variable 686 | 687 | # Debug mode examples: 688 | - debug: msg="The free memory for this system is {{ ansible_memfree_mb }}" 689 | - debug: var=output verbosity=2 690 | 691 | # Report changes made to templated files: 692 | $ ansible-playbook --check --diff 693 | 694 | # 'URI' module to check if RESTfuk API is returning required content: 695 | tasks: 696 | - action: uri url=http://api.myapp.com return_content=yes 697 | register: apiresponse 698 | - fail: msg='version was not provided' 699 | when: "'version' not in apiresponse.content" 700 | 701 | # 'script' module to execute script on managed host (module fails if $? is other then 0): 702 | taks: 703 | - script: check_free_memory 704 | 705 | # 'stat' module to see if files/dirs not managed by Ansible are present 706 | 'assert' module to see if file exists in managed host 707 | tasks: 708 | - stat: path=/var/run/app.lock 709 | register: lock 710 | - assert: 711 | that: 712 | - lock.stat.exists 713 | 714 | ############################################################################### 715 | ## Ansible Tower 716 | Web-based interface 717 | Enterprise solution for IT automation 718 | Dashboard for managing deployments and monitoring resources 719 | Adds automation, visual management, monitoring capabilities to Ansible 720 | Gives administrators control over user access 721 | Uses SSH credentials 722 | Blocks access to or transfer of credentials 723 | Implements continuous delivery and configuration management 724 | Integrates management in single tool 725 | 726 | # Installation - Configuration file: 727 | tower_setup_conf.yml under setup bundle directory 728 | 729 | Installation - 'configure' options: 730 | -l <-install on local machine with internal PostgreSQL 731 | --no-secondary-prompt <-skip prompts regarding secondary Tower nodes to be added 732 | -A <-Disable aut-generation of PostgreSQL password, prompt user for passwords 733 | -o <-source for configuration answers 734 | 735 | Installation - after the config file is ready: 736 | ./setup.sh 737 | -c <-specify file that stores the configuration 738 | -i <- 739 | -p <-specify file to use for host inventory 740 | -s <-require Ansible to prompt for SSH passwords 741 | -u <-require Ansible to prompt for sudo passwords 742 | -e <-set additional variables during installation 743 | -b <-perform database backup instead of installing Tower 744 | -r <-perform database restore instead of installing Tower 745 | 746 | Changing your password: 747 | $ sudo tower-manage changepassword admin 748 | 749 | # SSL certificate: 750 | /etc/tower/awx.cert 751 | /etc/tower/awx.key 752 | 753 | # REST API from CLI example: 754 | $ curl -s http://demo.lab.example.com/api/v1/ping | json_reformat 755 | 756 | # User types: 757 | normal 758 | organization admin 759 | superuser 760 | 761 | # User permissions: 762 | read 763 | write 764 | admin 765 | execute commands 766 | check 767 | run 768 | create 769 | 770 | # Projects: 771 | /var/lib/awx/projects 772 | $ sudo mkdir /var/lib/awx/projects/demoproject 773 | $ sudo cp demo.yml /var/lib/awx/projects/demoproject 774 | $ sudo chown -R awx /var/lib/awx/projects/demoproject 775 | 776 | ############################################################################### 777 | ## CLI: 778 | (1) Run a command somewhere else using Ansible 779 | $ ansible 780 | Usage: ansible [options] 781 | Options: 782 | -a MODULE_ARGS, --args=MODULE_ARGS 783 | module arguments 784 | --ask-vault-pass ask for vault password 785 | -B SECONDS, --background=SECONDS 786 | run asynchronously, failing after X seconds 787 | (default=N/A) 788 | -C, --check don't make any changes; instead, try to predict some 789 | of the changes that may occur 790 | -D, --diff when changing (small) files and templates, show the 791 | differences in those files; works great with --check 792 | -e EXTRA_VARS, --extra-vars=EXTRA_VARS 793 | set additional variables as key=value or YAML/JSON 794 | -f FORKS, --forks=FORKS 795 | specify number of parallel processes to use 796 | (default=5) 797 | -h, --help show this help message and exit 798 | -i INVENTORY, --inventory-file=INVENTORY 799 | specify inventory host path 800 | (default=/etc/ansible/hosts) or comma separated host 801 | list. 802 | -l SUBSET, --limit=SUBSET 803 | further limit selected hosts to an additional pattern 804 | --list-hosts outputs a list of matching hosts; does not execute 805 | anything else 806 | -m MODULE_NAME, --module-name=MODULE_NAME 807 | module name to execute (default=command) 808 | -M MODULE_PATH, --module-path=MODULE_PATH 809 | specify path(s) to module library (default=None) 810 | --new-vault-password-file=NEW_VAULT_PASSWORD_FILE 811 | new vault password file for rekey 812 | -o, --one-line condense output 813 | --output=OUTPUT_FILE output file name for encrypt or decrypt; use - for 814 | stdout 815 | -P POLL_INTERVAL, --poll=POLL_INTERVAL 816 | set the poll interval if using -B (default=15) 817 | --syntax-check perform a syntax check on the playbook, but do not 818 | execute it 819 | -t TREE, --tree=TREE log output to this directory 820 | --vault-password-file=VAULT_PASSWORD_FILE 821 | vault password file 822 | -v, --verbose verbose mode (-vvv for more, -vvvv to enable 823 | connection debugging) 824 | --version show program's version number and exit 825 | 826 | Connection Options: 827 | control as whom and how to connect to hosts 828 | 829 | -k, --ask-pass ask for connection password 830 | --private-key=PRIVATE_KEY_FILE, --key-file=PRIVATE_KEY_FILE 831 | use this file to authenticate the connection 832 | -u REMOTE_USER, --user=REMOTE_USER 833 | connect as this user (default=None) 834 | -c CONNECTION, --connection=CONNECTION 835 | connection type to use (default=smart) 836 | -T TIMEOUT, --timeout=TIMEOUT 837 | override the connection timeout in seconds 838 | (default=10) 839 | --ssh-common-args=SSH_COMMON_ARGS 840 | specify common arguments to pass to sftp/scp/ssh (e.g. 841 | ProxyCommand) 842 | --sftp-extra-args=SFTP_EXTRA_ARGS 843 | specify extra arguments to pass to sftp only (e.g. -f, 844 | -l) 845 | --scp-extra-args=SCP_EXTRA_ARGS 846 | specify extra arguments to pass to scp only (e.g. -l) 847 | --ssh-extra-args=SSH_EXTRA_ARGS 848 | specify extra arguments to pass to ssh only (e.g. -R) 849 | 850 | Privilege Escalation Options: 851 | control how and which user you become as on target hosts 852 | 853 | -s, --sudo run operations with sudo (nopasswd) (deprecated, use 854 | become) 855 | -U SUDO_USER, --sudo-user=SUDO_USER 856 | desired sudo user (default=root) (deprecated, use 857 | become) 858 | -S, --su run operations with su (deprecated, use become) 859 | -R SU_USER, --su-user=SU_USER 860 | run operations with su as this user (default=root) 861 | (deprecated, use become) 862 | -b, --become run operations with become (does not imply password 863 | prompting) 864 | --become-method=BECOME_METHOD 865 | privilege escalation method to use (default=sudo), 866 | valid choices: [ sudo | su | pbrun | pfexec | runas | 867 | doas | dzdo ] 868 | --become-user=BECOME_USER 869 | run operations as this user (default=root) 870 | --ask-sudo-pass ask for sudo password (deprecated, use become) 871 | --ask-su-pass ask for su password (deprecated, use become) 872 | -K, --ask-become-pass 873 | ask for privilege escalation password 874 | 875 | 876 | ############################################################################### 877 | (2) Run Ansible playbook 878 | $ ansible-playbook 879 | Usage: ansible-playbook playbook.yml 880 | Options: 881 | --ask-vault-pass ask for vault password 882 | -C, --check don't make any changes; instead, try to predict some 883 | of the changes that may occur 884 | -D, --diff when changing (small) files and templates, show the 885 | differences in those files; works great with --check 886 | -e EXTRA_VARS, --extra-vars=EXTRA_VARS 887 | set additional variables as key=value or YAML/JSON 888 | --flush-cache clear the fact cache 889 | --force-handlers run handlers even if a task fails 890 | -f FORKS, --forks=FORKS 891 | specify number of parallel processes to use 892 | (default=5) 893 | -h, --help show this help message and exit 894 | -i INVENTORY, --inventory-file=INVENTORY 895 | specify inventory host path 896 | (default=/etc/ansible/hosts) or comma separated host 897 | list. 898 | -l SUBSET, --limit=SUBSET 899 | further limit selected hosts to an additional pattern 900 | --list-hosts outputs a list of matching hosts; does not execute 901 | anything else 902 | --list-tags list all available tags 903 | --list-tasks list all tasks that would be executed 904 | -M MODULE_PATH, --module-path=MODULE_PATH 905 | specify path(s) to module library (default=None) 906 | --new-vault-password-file=NEW_VAULT_PASSWORD_FILE 907 | new vault password file for rekey 908 | --output=OUTPUT_FILE output file name for encrypt or decrypt; use - for 909 | stdout 910 | --skip-tags=SKIP_TAGS 911 | only run plays and tasks whose tags do not match these 912 | values 913 | --start-at-task=START_AT_TASK 914 | start the playbook at the task matching this name 915 | --step one-step-at-a-time: confirm each task before running 916 | --syntax-check perform a syntax check on the playbook, but do not 917 | execute it 918 | -t TAGS, --tags=TAGS only run plays and tasks tagged with these values 919 | --vault-password-file=VAULT_PASSWORD_FILE 920 | vault password file 921 | -v, --verbose verbose mode (-vvv for more, -vvvv to enable 922 | connection debugging) 923 | --version show program's version number and exit 924 | 925 | Connection Options: 926 | control as whom and how to connect to hosts 927 | 928 | -k, --ask-pass ask for connection password 929 | --private-key=PRIVATE_KEY_FILE, --key-file=PRIVATE_KEY_FILE 930 | use this file to authenticate the connection 931 | -u REMOTE_USER, --user=REMOTE_USER 932 | connect as this user (default=None) 933 | -c CONNECTION, --connection=CONNECTION 934 | connection type to use (default=smart) 935 | -T TIMEOUT, --timeout=TIMEOUT 936 | override the connection timeout in seconds 937 | (default=10) 938 | --ssh-common-args=SSH_COMMON_ARGS 939 | specify common arguments to pass to sftp/scp/ssh (e.g. 940 | ProxyCommand) 941 | --sftp-extra-args=SFTP_EXTRA_ARGS 942 | specify extra arguments to pass to sftp only (e.g. -f, 943 | -l) 944 | --scp-extra-args=SCP_EXTRA_ARGS 945 | specify extra arguments to pass to scp only (e.g. -l) 946 | --ssh-extra-args=SSH_EXTRA_ARGS 947 | specify extra arguments to pass to ssh only (e.g. -R) 948 | 949 | Privilege Escalation Options: 950 | control how and which user you become as on target hosts 951 | 952 | -s, --sudo run operations with sudo (nopasswd) (deprecated, use 953 | become) 954 | -U SUDO_USER, --sudo-user=SUDO_USER 955 | desired sudo user (default=root) (deprecated, use 956 | become) 957 | -S, --su run operations with su (deprecated, use become) 958 | -R SU_USER, --su-user=SU_USER 959 | run operations with su as this user (default=root) 960 | (deprecated, use become) 961 | -b, --become run operations with become (does not imply password 962 | prompting) 963 | --become-method=BECOME_METHOD 964 | privilege escalation method to use (default=sudo), 965 | valid choices: [ sudo | su | pbrun | pfexec | runas | 966 | doas | dzdo ] 967 | --become-user=BECOME_USER 968 | run operations as this user (default=root) 969 | --ask-sudo-pass ask for sudo password (deprecated, use become) 970 | --ask-su-pass ask for su password (deprecated, use become) 971 | -K, --ask-become-pass 972 | ask for privilege escalation password 973 | 974 | 975 | ############################################################################### 976 | (3) Set up a remote copy of ansible on each managed node (clone Ansible configuration files from Git repository) 977 | $ ansible-pull 978 | Usage: ansible-pull -U [options] 979 | 980 | Options: 981 | --accept-host-key adds the hostkey for the repo url if not already added 982 | --ask-vault-pass ask for vault password 983 | -C CHECKOUT, --checkout=CHECKOUT 984 | branch/tag/commit to checkout. Defaults to behavior 985 | of repository module. 986 | -d DEST, --directory=DEST 987 | directory to checkout repository to 988 | -e EXTRA_VARS, --extra-vars=EXTRA_VARS 989 | set additional variables as key=value or YAML/JSON 990 | -f, --force run the playbook even if the repository could not be 991 | updated 992 | --full Do a full clone, instead of a shallow one. 993 | -h, --help show this help message and exit 994 | -i INVENTORY, --inventory-file=INVENTORY 995 | specify inventory host path 996 | (default=/etc/ansible/hosts) or comma separated host 997 | list. 998 | -l SUBSET, --limit=SUBSET 999 | further limit selected hosts to an additional pattern 1000 | --list-hosts outputs a list of matching hosts; does not execute 1001 | anything else 1002 | -m MODULE_NAME, --module-name=MODULE_NAME 1003 | Repository module name, which ansible will use to 1004 | check out the repo. Default is git. 1005 | -M MODULE_PATH, --module-path=MODULE_PATH 1006 | specify path(s) to module library (default=None) 1007 | --new-vault-password-file=NEW_VAULT_PASSWORD_FILE 1008 | new vault password file for rekey 1009 | -o, --only-if-changed 1010 | only run the playbook if the repository has been 1011 | updated 1012 | --output=OUTPUT_FILE output file name for encrypt or decrypt; use - for 1013 | stdout 1014 | --purge purge checkout after playbook run 1015 | --skip-tags=SKIP_TAGS 1016 | only run plays and tasks whose tags do not match these 1017 | values 1018 | -s SLEEP, --sleep=SLEEP 1019 | sleep for random interval (between 0 and n number of 1020 | seconds) before starting. This is a useful way to 1021 | disperse git requests 1022 | -t TAGS, --tags=TAGS only run plays and tasks tagged with these values 1023 | -U URL, --url=URL URL of the playbook repository 1024 | --vault-password-file=VAULT_PASSWORD_FILE 1025 | vault password file 1026 | -v, --verbose verbose mode (-vvv for more, -vvvv to enable 1027 | connection debugging) 1028 | --verify-commit verify GPG signature of checked out commit, if it 1029 | fails abort running the playbook. This needs the 1030 | corresponding VCS module to support such an operation 1031 | --version show program's version number and exit 1032 | 1033 | Connection Options: 1034 | control as whom and how to connect to hosts 1035 | 1036 | -k, --ask-pass ask for connection password 1037 | --private-key=PRIVATE_KEY_FILE, --key-file=PRIVATE_KEY_FILE 1038 | use this file to authenticate the connection 1039 | -u REMOTE_USER, --user=REMOTE_USER 1040 | connect as this user (default=None) 1041 | -c CONNECTION, --connection=CONNECTION 1042 | connection type to use (default=smart) 1043 | -T TIMEOUT, --timeout=TIMEOUT 1044 | override the connection timeout in seconds 1045 | (default=10) 1046 | --ssh-common-args=SSH_COMMON_ARGS 1047 | specify common arguments to pass to sftp/scp/ssh (e.g. 1048 | ProxyCommand) 1049 | --sftp-extra-args=SFTP_EXTRA_ARGS 1050 | specify extra arguments to pass to sftp only (e.g. -f, 1051 | -l) 1052 | --scp-extra-args=SCP_EXTRA_ARGS 1053 | specify extra arguments to pass to scp only (e.g. -l) 1054 | --ssh-extra-args=SSH_EXTRA_ARGS 1055 | specify extra arguments to pass to ssh only (e.g. -R) 1056 | 1057 | Privilege Escalation Options: 1058 | control how and which user you become as on target hosts 1059 | 1060 | --ask-sudo-pass ask for sudo password (deprecated, use become) 1061 | --ask-su-pass ask for su password (deprecated, use become) 1062 | -K, --ask-become-pass 1063 | ask for privilege escalation password 1064 | 1065 | ############################################################################### 1066 | (4) Accessing documentation locally 1067 | $ ansible-doc 1068 | Usage: ansible-doc [options] [module...] 1069 | 1070 | Options: 1071 | -h, --help show this help message and exit 1072 | -l, --list List available modules 1073 | -M MODULE_PATH, --module-path=MODULE_PATH 1074 | specify path(s) to module library (default=None) 1075 | -s, --snippet Show playbook snippet for specified module(s) 1076 | -v, --verbose verbose mode (-vvv for more, -vvvv to enable 1077 | connection debugging) 1078 | --version show program's version number and exit 1079 | 1080 | ############################################################################### 1081 | (5) Ansible Galaxy tool 1082 | $ ansible-galaxy 1083 | 1084 | Usage: ansible-galaxy [delete|import|info|init|install|list|login|remove|search|setup] [--help] [options] ... 1085 | 1086 | Options: 1087 | -h, --help show this help message and exit 1088 | -v, --verbose verbose mode (-vvv for more, -vvvv to enable connection 1089 | debugging) 1090 | --version show program's version number and exit 1091 | Examples: 1092 | ansible-galaxy search --author 1093 | ansible-galaxy search --platforms 1094 | ansible-galaxy search --galaxy-tags 1095 | ansible-galaxy info 1096 | ansible-galaxy install -p 1097 | ansible-galaxy install -r ... 1098 | ansible-galaxy list 1099 | ansible-galaxy remove 1100 | ansible-galaxy init 1101 | ansible-galaxy init --offline 1102 | 1103 | ############################################################################### 1104 | (6) Hiding secrets 1105 | $ ansible-vault 1106 | 1107 | Usage: ansible-vault [create|decrypt|edit|encrypt|rekey|view] [--help] [options] vaultfile.yml 1108 | 1109 | Options: 1110 | --ask-vault-pass ask for vault password 1111 | -h, --help show this help message and exit 1112 | --new-vault-password-file=NEW_VAULT_PASSWORD_FILE 1113 | new vault password file for rekey 1114 | --output=OUTPUT_FILE output file name for encrypt or decrypt; use - for 1115 | stdout 1116 | --vault-password-file=VAULT_PASSWORD_FILE 1117 | vault password file 1118 | -v, --verbose verbose mode (-vvv for more, -vvvv to enable 1119 | connection debugging) 1120 | --version show program's version number and exit 1121 | 1122 | ############################################################################### 1123 | (7) Windows 1124 | $ pip install pywinrm <-- on the control machine 1125 | 1126 | Authentication: 1127 | a/ Certificate: authentication similar to SSH 1128 | b/ Kerberos: python-kerberos 1129 | c/ CredSSP: for local and domain accounts 1130 | 1131 | On the client(s): 1132 | a/ PowerShell 3.0 or higher 1133 | b/ Enable PowerShell 1134 | c/ Set up WinRM: 1135 | https://github.com/ansible/ansible/blob/devel/examples/scripts/ConfigureRemotingForAnsible.ps1 1136 | 1137 | For Kerberos, on the client(s): 1138 | python-devel, krb5-devel, krb5-libs, krb5-workstation, pywinrm 1139 | /etc/krb5.conf.d/ansible.conf 1140 | [realms] 1141 | ad1.${GUID}.example.com = { 1142 | kdc = ad1.${GUID}.example.opentlc.com 1143 | } 1144 | 1145 | 'ansible.cfg' example: 1146 | ansible_connection=winrm 1147 | ansible_user=Administrator 1148 | 1149 | Windows Ansible modules examples: 1150 | win_ping 1151 | win_chocolatey 1152 | win_service 1153 | win_firewall 1154 | win_firewall_rule 1155 | win_user 1156 | win_domain_user 1157 | win_domain_controller 1158 | -------------------------------------------------------------------------------- /ec2.ini: -------------------------------------------------------------------------------- 1 | # Ansible EC2 external inventory script settings 2 | # 3 | 4 | [ec2] 5 | 6 | # to talk to a private eucalyptus instance uncomment these lines 7 | # and edit edit eucalyptus_host to be the host name of your cloud controller 8 | #eucalyptus = True 9 | #eucalyptus_host = clc.cloud.domain.org 10 | 11 | # AWS regions to make calls to. Set this to 'all' to make request to all regions 12 | # in AWS and merge the results together. Alternatively, set this to a comma 13 | # separated list of regions. E.g. 'us-east-1,us-west-1,us-west-2' 14 | regions = all 15 | regions_exclude = us-gov-west-1,cn-north-1 16 | 17 | # When generating inventory, Ansible needs to know how to address a server. 18 | # Each EC2 instance has a lot of variables associated with it. Here is the list: 19 | # http://docs.pythonboto.org/en/latest/ref/ec2.html#module-boto.ec2.instance 20 | # Below are 2 variables that are used as the address of a server: 21 | # - destination_variable 22 | # - vpc_destination_variable 23 | 24 | # This is the normal destination variable to use. If you are running Ansible 25 | # from outside EC2, then 'public_dns_name' makes the most sense. If you are 26 | # running Ansible from within EC2, then perhaps you want to use the internal 27 | # address, and should set this to 'private_dns_name'. The key of an EC2 tag 28 | # may optionally be used; however the boto instance variables hold precedence 29 | # in the event of a collision. 30 | destination_variable = public_dns_name 31 | 32 | # This allows you to override the inventory_name with an ec2 variable, instead 33 | # of using the destination_variable above. Addressing (aka ansible_ssh_host) 34 | # will still use destination_variable. Tags should be written as 'tag_TAGNAME'. 35 | #hostname_variable = tag_Name 36 | 37 | # For server inside a VPC, using DNS names may not make sense. When an instance 38 | # has 'subnet_id' set, this variable is used. If the subnet is public, setting 39 | # this to 'ip_address' will return the public IP address. For instances in a 40 | # private subnet, this should be set to 'private_ip_address', and Ansible must 41 | # be run from within EC2. The key of an EC2 tag may optionally be used; however 42 | # the boto instance variables hold precedence in the event of a collision. 43 | # WARNING: - instances that are in the private vpc, _without_ public ip address 44 | # will not be listed in the inventory until You set: 45 | # vpc_destination_variable = private_ip_address 46 | vpc_destination_variable = ip_address 47 | 48 | # The following two settings allow flexible ansible host naming based on a 49 | # python format string and a comma-separated list of ec2 tags. Note that: 50 | # 51 | # 1) If the tags referenced are not present for some instances, empty strings 52 | # will be substituted in the format string. 53 | # 2) This overrides both destination_variable and vpc_destination_variable. 54 | # 55 | #destination_format = {0}.{1}.example.com 56 | #destination_format_tags = Name,environment 57 | 58 | # To tag instances on EC2 with the resource records that point to them from 59 | # Route53, uncomment and set 'route53' to True. 60 | route53 = False 61 | 62 | # To exclude RDS instances from the inventory, uncomment and set to False. 63 | #rds = False 64 | 65 | # To exclude ElastiCache instances from the inventory, uncomment and set to False. 66 | #elasticache = False 67 | 68 | # Additionally, you can specify the list of zones to exclude looking up in 69 | # 'route53_excluded_zones' as a comma-separated list. 70 | # route53_excluded_zones = samplezone1.com, samplezone2.com 71 | 72 | # By default, only EC2 instances in the 'running' state are returned. Set 73 | # 'all_instances' to True to return all instances regardless of state. 74 | all_instances = False 75 | 76 | # By default, only EC2 instances in the 'running' state are returned. Specify 77 | # EC2 instance states to return as a comma-separated list. This 78 | # option is overriden when 'all_instances' is True. 79 | # instance_states = pending, running, shutting-down, terminated, stopping, stopped 80 | 81 | # By default, only RDS instances in the 'available' state are returned. Set 82 | # 'all_rds_instances' to True return all RDS instances regardless of state. 83 | all_rds_instances = False 84 | 85 | # Include RDS cluster information (Aurora etc.) 86 | include_rds_clusters = False 87 | 88 | # By default, only ElastiCache clusters and nodes in the 'available' state 89 | # are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes' 90 | # to True return all ElastiCache clusters and nodes, regardless of state. 91 | # 92 | # Note that all_elasticache_nodes only applies to listed clusters. That means 93 | # if you set all_elastic_clusters to false, no node will be return from 94 | # unavailable clusters, regardless of the state and to what you set for 95 | # all_elasticache_nodes. 96 | all_elasticache_replication_groups = False 97 | all_elasticache_clusters = False 98 | all_elasticache_nodes = False 99 | 100 | # API calls to EC2 are slow. For this reason, we cache the results of an API 101 | # call. Set this to the path you want cache files to be written to. Two files 102 | # will be written to this directory: 103 | # - ansible-ec2.cache 104 | # - ansible-ec2.index 105 | cache_path = ~/.ansible/tmp 106 | 107 | # The number of seconds a cache file is considered valid. After this many 108 | # seconds, a new API call will be made, and the cache file will be updated. 109 | # To disable the cache, set this value to 0 110 | cache_max_age = 300 111 | 112 | # Organize groups into a nested/hierarchy instead of a flat namespace. 113 | nested_groups = False 114 | 115 | # Replace - tags when creating groups to avoid issues with ansible 116 | replace_dash_in_groups = True 117 | 118 | # If set to true, any tag of the form "a,b,c" is expanded into a list 119 | # and the results are used to create additional tag_* inventory groups. 120 | expand_csv_tags = False 121 | 122 | # The EC2 inventory output can become very large. To manage its size, 123 | # configure which groups should be created. 124 | group_by_instance_id = True 125 | group_by_region = True 126 | group_by_availability_zone = True 127 | group_by_ami_id = True 128 | group_by_instance_type = True 129 | group_by_key_pair = True 130 | group_by_vpc_id = True 131 | group_by_security_group = True 132 | group_by_tag_keys = True 133 | group_by_tag_none = True 134 | group_by_route53_names = True 135 | group_by_rds_engine = True 136 | group_by_rds_parameter_group = True 137 | group_by_elasticache_engine = True 138 | group_by_elasticache_cluster = True 139 | group_by_elasticache_parameter_group = True 140 | group_by_elasticache_replication_group = True 141 | 142 | # If you only want to include hosts that match a certain regular expression 143 | # pattern_include = staging-* 144 | 145 | # If you want to exclude any hosts that match a certain regular expression 146 | # pattern_exclude = staging-* 147 | 148 | # Instance filters can be used to control which instances are retrieved for 149 | # inventory. For the full list of possible filters, please read the EC2 API 150 | # docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters 151 | # Filters are key/value pairs separated by '=', to list multiple filters use 152 | # a list separated by commas. See examples below. 153 | 154 | # Retrieve only instances with (key=value) env=staging tag 155 | # instance_filters = tag:env=staging 156 | 157 | # Retrieve only instances with role=webservers OR role=dbservers tag 158 | # instance_filters = tag:role=webservers,tag:role=dbservers 159 | 160 | # Retrieve only t1.micro instances OR instances with tag env=staging 161 | # instance_filters = instance-type=t1.micro,tag:env=staging 162 | 163 | # You can use wildcards in filter values also. Below will list instances which 164 | # tag Name value matches webservers1* 165 | # (ex. webservers15, webservers1a, webservers123 etc) 166 | # instance_filters = tag:Name=webservers1* 167 | 168 | # A boto configuration profile may be used to separate out credentials 169 | # see http://boto.readthedocs.org/en/latest/boto_config_tut.html 170 | # boto_profile = some-boto-profile-name 171 | 172 | 173 | [credentials] 174 | 175 | # The AWS credentials can optionally be specified here. Credentials specified 176 | # here are ignored if the environment variable AWS_ACCESS_KEY_ID or 177 | # AWS_PROFILE is set, or if the boto_profile property above is set. 178 | # 179 | # Supplying AWS credentials here is not recommended, as it introduces 180 | # non-trivial security concerns. When going down this route, please make sure 181 | # to set access permissions for this file correctly, e.g. handle it the same 182 | # way as you would a private SSH key. 183 | # 184 | # Unlike the boto and AWS configure files, this section does not support 185 | # profiles. 186 | # 187 | # aws_access_key_id = AXXXXXXXXXXXXXX 188 | # aws_secret_access_key = XXXXXXXXXXXXXXXXXXX 189 | # aws_security_token = XXXXXXXXXXXXXXXXXXXXXXXXXXXX 190 | -------------------------------------------------------------------------------- /ec2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | EC2 external inventory script 5 | ================================= 6 | 7 | Generates inventory that Ansible can understand by making API request to 8 | AWS EC2 using the Boto library. 9 | 10 | NOTE: This script assumes Ansible is being executed where the environment 11 | variables needed for Boto have already been set: 12 | export AWS_ACCESS_KEY_ID='AK123' 13 | export AWS_SECRET_ACCESS_KEY='abc123' 14 | 15 | This script also assumes there is an ec2.ini file alongside it. To specify a 16 | different path to ec2.ini, define the EC2_INI_PATH environment variable: 17 | 18 | export EC2_INI_PATH=/path/to/my_ec2.ini 19 | 20 | If you're using eucalyptus you need to set the above variables and 21 | you need to define: 22 | 23 | export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus 24 | 25 | If you're using boto profiles (requires boto>=2.24.0) you can choose a profile 26 | using the --boto-profile command line argument (e.g. ec2.py --boto-profile prod) or using 27 | the AWS_PROFILE variable: 28 | 29 | AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml 30 | 31 | For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html 32 | 33 | When run against a specific host, this script returns the following variables: 34 | - ec2_ami_launch_index 35 | - ec2_architecture 36 | - ec2_association 37 | - ec2_attachTime 38 | - ec2_attachment 39 | - ec2_attachmentId 40 | - ec2_block_devices 41 | - ec2_client_token 42 | - ec2_deleteOnTermination 43 | - ec2_description 44 | - ec2_deviceIndex 45 | - ec2_dns_name 46 | - ec2_eventsSet 47 | - ec2_group_name 48 | - ec2_hypervisor 49 | - ec2_id 50 | - ec2_image_id 51 | - ec2_instanceState 52 | - ec2_instance_type 53 | - ec2_ipOwnerId 54 | - ec2_ip_address 55 | - ec2_item 56 | - ec2_kernel 57 | - ec2_key_name 58 | - ec2_launch_time 59 | - ec2_monitored 60 | - ec2_monitoring 61 | - ec2_networkInterfaceId 62 | - ec2_ownerId 63 | - ec2_persistent 64 | - ec2_placement 65 | - ec2_platform 66 | - ec2_previous_state 67 | - ec2_private_dns_name 68 | - ec2_private_ip_address 69 | - ec2_publicIp 70 | - ec2_public_dns_name 71 | - ec2_ramdisk 72 | - ec2_reason 73 | - ec2_region 74 | - ec2_requester_id 75 | - ec2_root_device_name 76 | - ec2_root_device_type 77 | - ec2_security_group_ids 78 | - ec2_security_group_names 79 | - ec2_shutdown_state 80 | - ec2_sourceDestCheck 81 | - ec2_spot_instance_request_id 82 | - ec2_state 83 | - ec2_state_code 84 | - ec2_state_reason 85 | - ec2_status 86 | - ec2_subnet_id 87 | - ec2_tenancy 88 | - ec2_virtualization_type 89 | - ec2_vpc_id 90 | 91 | These variables are pulled out of a boto.ec2.instance object. There is a lack of 92 | consistency with variable spellings (camelCase and underscores) since this 93 | just loops through all variables the object exposes. It is preferred to use the 94 | ones with underscores when multiple exist. 95 | 96 | In addition, if an instance has AWS Tags associated with it, each tag is a new 97 | variable named: 98 | - ec2_tag_[Key] = [Value] 99 | 100 | Security groups are comma-separated in 'ec2_security_group_ids' and 101 | 'ec2_security_group_names'. 102 | ''' 103 | 104 | # (c) 2012, Peter Sankauskas 105 | # 106 | # This file is part of Ansible, 107 | # 108 | # Ansible is free software: you can redistribute it and/or modify 109 | # it under the terms of the GNU General Public License as published by 110 | # the Free Software Foundation, either version 3 of the License, or 111 | # (at your option) any later version. 112 | # 113 | # Ansible is distributed in the hope that it will be useful, 114 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 115 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 116 | # GNU General Public License for more details. 117 | # 118 | # You should have received a copy of the GNU General Public License 119 | # along with Ansible. If not, see . 120 | 121 | ###################################################################### 122 | 123 | import sys 124 | import os 125 | import argparse 126 | import re 127 | from time import time 128 | import boto 129 | from boto import ec2 130 | from boto import rds 131 | from boto import elasticache 132 | from boto import route53 133 | import six 134 | 135 | from ansible.module_utils import ec2 as ec2_utils 136 | 137 | HAS_BOTO3 = False 138 | try: 139 | import boto3 140 | HAS_BOTO3 = True 141 | except ImportError: 142 | pass 143 | 144 | from six.moves import configparser 145 | from collections import defaultdict 146 | 147 | try: 148 | import json 149 | except ImportError: 150 | import simplejson as json 151 | 152 | 153 | class Ec2Inventory(object): 154 | 155 | def _empty_inventory(self): 156 | return {"_meta" : {"hostvars" : {}}} 157 | 158 | def __init__(self): 159 | ''' Main execution path ''' 160 | 161 | # Inventory grouped by instance IDs, tags, security groups, regions, 162 | # and availability zones 163 | self.inventory = self._empty_inventory() 164 | 165 | # Index of hostname (address) to instance ID 166 | self.index = {} 167 | 168 | # Boto profile to use (if any) 169 | self.boto_profile = None 170 | 171 | # AWS credentials. 172 | self.credentials = {} 173 | 174 | # Read settings and parse CLI arguments 175 | self.parse_cli_args() 176 | self.read_settings() 177 | 178 | # Make sure that profile_name is not passed at all if not set 179 | # as pre 2.24 boto will fall over otherwise 180 | if self.boto_profile: 181 | if not hasattr(boto.ec2.EC2Connection, 'profile_name'): 182 | self.fail_with_error("boto version must be >= 2.24 to use profile") 183 | 184 | # Cache 185 | if self.args.refresh_cache: 186 | self.do_api_calls_update_cache() 187 | elif not self.is_cache_valid(): 188 | self.do_api_calls_update_cache() 189 | 190 | # Data to print 191 | if self.args.host: 192 | data_to_print = self.get_host_info() 193 | 194 | elif self.args.list: 195 | # Display list of instances for inventory 196 | if self.inventory == self._empty_inventory(): 197 | data_to_print = self.get_inventory_from_cache() 198 | else: 199 | data_to_print = self.json_format_dict(self.inventory, True) 200 | 201 | print(data_to_print) 202 | 203 | 204 | def is_cache_valid(self): 205 | ''' Determines if the cache files have expired, or if it is still valid ''' 206 | 207 | if os.path.isfile(self.cache_path_cache): 208 | mod_time = os.path.getmtime(self.cache_path_cache) 209 | current_time = time() 210 | if (mod_time + self.cache_max_age) > current_time: 211 | if os.path.isfile(self.cache_path_index): 212 | return True 213 | 214 | return False 215 | 216 | 217 | def read_settings(self): 218 | ''' Reads the settings from the ec2.ini file ''' 219 | if six.PY3: 220 | config = configparser.ConfigParser() 221 | else: 222 | config = configparser.SafeConfigParser() 223 | ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini') 224 | ec2_ini_path = os.path.expanduser(os.path.expandvars(os.environ.get('EC2_INI_PATH', ec2_default_ini_path))) 225 | config.read(ec2_ini_path) 226 | 227 | # is eucalyptus? 228 | self.eucalyptus_host = None 229 | self.eucalyptus = False 230 | if config.has_option('ec2', 'eucalyptus'): 231 | self.eucalyptus = config.getboolean('ec2', 'eucalyptus') 232 | if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): 233 | self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') 234 | 235 | # Regions 236 | self.regions = [] 237 | configRegions = config.get('ec2', 'regions') 238 | configRegions_exclude = config.get('ec2', 'regions_exclude') 239 | if (configRegions == 'all'): 240 | if self.eucalyptus_host: 241 | self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials) 242 | else: 243 | for regionInfo in ec2.regions(): 244 | if regionInfo.name not in configRegions_exclude: 245 | self.regions.append(regionInfo.name) 246 | else: 247 | self.regions = configRegions.split(",") 248 | 249 | # Destination addresses 250 | self.destination_variable = config.get('ec2', 'destination_variable') 251 | self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') 252 | 253 | if config.has_option('ec2', 'hostname_variable'): 254 | self.hostname_variable = config.get('ec2', 'hostname_variable') 255 | else: 256 | self.hostname_variable = None 257 | 258 | if config.has_option('ec2', 'destination_format') and \ 259 | config.has_option('ec2', 'destination_format_tags'): 260 | self.destination_format = config.get('ec2', 'destination_format') 261 | self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',') 262 | else: 263 | self.destination_format = None 264 | self.destination_format_tags = None 265 | 266 | # Route53 267 | self.route53_enabled = config.getboolean('ec2', 'route53') 268 | self.route53_excluded_zones = [] 269 | if config.has_option('ec2', 'route53_excluded_zones'): 270 | self.route53_excluded_zones.extend( 271 | config.get('ec2', 'route53_excluded_zones', '').split(',')) 272 | 273 | # Include RDS instances? 274 | self.rds_enabled = True 275 | if config.has_option('ec2', 'rds'): 276 | self.rds_enabled = config.getboolean('ec2', 'rds') 277 | 278 | # Include RDS cluster instances? 279 | if config.has_option('ec2', 'include_rds_clusters'): 280 | self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters') 281 | else: 282 | self.include_rds_clusters = False 283 | 284 | # Include ElastiCache instances? 285 | self.elasticache_enabled = True 286 | if config.has_option('ec2', 'elasticache'): 287 | self.elasticache_enabled = config.getboolean('ec2', 'elasticache') 288 | 289 | # Return all EC2 instances? 290 | if config.has_option('ec2', 'all_instances'): 291 | self.all_instances = config.getboolean('ec2', 'all_instances') 292 | else: 293 | self.all_instances = False 294 | 295 | # Instance states to be gathered in inventory. Default is 'running'. 296 | # Setting 'all_instances' to 'yes' overrides this option. 297 | ec2_valid_instance_states = [ 298 | 'pending', 299 | 'running', 300 | 'shutting-down', 301 | 'terminated', 302 | 'stopping', 303 | 'stopped' 304 | ] 305 | self.ec2_instance_states = [] 306 | if self.all_instances: 307 | self.ec2_instance_states = ec2_valid_instance_states 308 | elif config.has_option('ec2', 'instance_states'): 309 | for instance_state in config.get('ec2', 'instance_states').split(','): 310 | instance_state = instance_state.strip() 311 | if instance_state not in ec2_valid_instance_states: 312 | continue 313 | self.ec2_instance_states.append(instance_state) 314 | else: 315 | self.ec2_instance_states = ['running'] 316 | 317 | # Return all RDS instances? (if RDS is enabled) 318 | if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled: 319 | self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances') 320 | else: 321 | self.all_rds_instances = False 322 | 323 | # Return all ElastiCache replication groups? (if ElastiCache is enabled) 324 | if config.has_option('ec2', 'all_elasticache_replication_groups') and self.elasticache_enabled: 325 | self.all_elasticache_replication_groups = config.getboolean('ec2', 'all_elasticache_replication_groups') 326 | else: 327 | self.all_elasticache_replication_groups = False 328 | 329 | # Return all ElastiCache clusters? (if ElastiCache is enabled) 330 | if config.has_option('ec2', 'all_elasticache_clusters') and self.elasticache_enabled: 331 | self.all_elasticache_clusters = config.getboolean('ec2', 'all_elasticache_clusters') 332 | else: 333 | self.all_elasticache_clusters = False 334 | 335 | # Return all ElastiCache nodes? (if ElastiCache is enabled) 336 | if config.has_option('ec2', 'all_elasticache_nodes') and self.elasticache_enabled: 337 | self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes') 338 | else: 339 | self.all_elasticache_nodes = False 340 | 341 | # boto configuration profile (prefer CLI argument) 342 | self.boto_profile = self.args.boto_profile 343 | if config.has_option('ec2', 'boto_profile') and not self.boto_profile: 344 | self.boto_profile = config.get('ec2', 'boto_profile') 345 | 346 | # AWS credentials (prefer environment variables) 347 | if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or 348 | os.environ.get('AWS_PROFILE')): 349 | if config.has_option('credentials', 'aws_access_key_id'): 350 | aws_access_key_id = config.get('credentials', 'aws_access_key_id') 351 | else: 352 | aws_access_key_id = None 353 | if config.has_option('credentials', 'aws_secret_access_key'): 354 | aws_secret_access_key = config.get('credentials', 'aws_secret_access_key') 355 | else: 356 | aws_secret_access_key = None 357 | if config.has_option('credentials', 'aws_security_token'): 358 | aws_security_token = config.get('credentials', 'aws_security_token') 359 | else: 360 | aws_security_token = None 361 | if aws_access_key_id: 362 | self.credentials = { 363 | 'aws_access_key_id': aws_access_key_id, 364 | 'aws_secret_access_key': aws_secret_access_key 365 | } 366 | if aws_security_token: 367 | self.credentials['security_token'] = aws_security_token 368 | 369 | # Cache related 370 | cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) 371 | if self.boto_profile: 372 | cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile) 373 | if not os.path.exists(cache_dir): 374 | os.makedirs(cache_dir) 375 | 376 | cache_name = 'ansible-ec2' 377 | aws_profile = lambda: (self.boto_profile or 378 | os.environ.get('AWS_PROFILE') or 379 | os.environ.get('AWS_ACCESS_KEY_ID') or 380 | self.credentials.get('aws_access_key_id', None)) 381 | if aws_profile(): 382 | cache_name = '%s-%s' % (cache_name, aws_profile()) 383 | self.cache_path_cache = cache_dir + "/%s.cache" % cache_name 384 | self.cache_path_index = cache_dir + "/%s.index" % cache_name 385 | self.cache_max_age = config.getint('ec2', 'cache_max_age') 386 | 387 | if config.has_option('ec2', 'expand_csv_tags'): 388 | self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags') 389 | else: 390 | self.expand_csv_tags = False 391 | 392 | # Configure nested groups instead of flat namespace. 393 | if config.has_option('ec2', 'nested_groups'): 394 | self.nested_groups = config.getboolean('ec2', 'nested_groups') 395 | else: 396 | self.nested_groups = False 397 | 398 | # Replace dash or not in group names 399 | if config.has_option('ec2', 'replace_dash_in_groups'): 400 | self.replace_dash_in_groups = config.getboolean('ec2', 'replace_dash_in_groups') 401 | else: 402 | self.replace_dash_in_groups = True 403 | 404 | # Configure which groups should be created. 405 | group_by_options = [ 406 | 'group_by_instance_id', 407 | 'group_by_region', 408 | 'group_by_availability_zone', 409 | 'group_by_ami_id', 410 | 'group_by_instance_type', 411 | 'group_by_key_pair', 412 | 'group_by_vpc_id', 413 | 'group_by_security_group', 414 | 'group_by_tag_keys', 415 | 'group_by_tag_none', 416 | 'group_by_route53_names', 417 | 'group_by_rds_engine', 418 | 'group_by_rds_parameter_group', 419 | 'group_by_elasticache_engine', 420 | 'group_by_elasticache_cluster', 421 | 'group_by_elasticache_parameter_group', 422 | 'group_by_elasticache_replication_group', 423 | ] 424 | for option in group_by_options: 425 | if config.has_option('ec2', option): 426 | setattr(self, option, config.getboolean('ec2', option)) 427 | else: 428 | setattr(self, option, True) 429 | 430 | # Do we need to just include hosts that match a pattern? 431 | try: 432 | pattern_include = config.get('ec2', 'pattern_include') 433 | if pattern_include and len(pattern_include) > 0: 434 | self.pattern_include = re.compile(pattern_include) 435 | else: 436 | self.pattern_include = None 437 | except configparser.NoOptionError: 438 | self.pattern_include = None 439 | 440 | # Do we need to exclude hosts that match a pattern? 441 | try: 442 | pattern_exclude = config.get('ec2', 'pattern_exclude'); 443 | if pattern_exclude and len(pattern_exclude) > 0: 444 | self.pattern_exclude = re.compile(pattern_exclude) 445 | else: 446 | self.pattern_exclude = None 447 | except configparser.NoOptionError: 448 | self.pattern_exclude = None 449 | 450 | # Instance filters (see boto and EC2 API docs). Ignore invalid filters. 451 | self.ec2_instance_filters = defaultdict(list) 452 | if config.has_option('ec2', 'instance_filters'): 453 | 454 | filters = [f for f in config.get('ec2', 'instance_filters').split(',') if f] 455 | 456 | for instance_filter in filters: 457 | instance_filter = instance_filter.strip() 458 | if not instance_filter or '=' not in instance_filter: 459 | continue 460 | filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] 461 | if not filter_key: 462 | continue 463 | self.ec2_instance_filters[filter_key].append(filter_value) 464 | 465 | def parse_cli_args(self): 466 | ''' Command line argument processing ''' 467 | 468 | parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2') 469 | parser.add_argument('--list', action='store_true', default=True, 470 | help='List instances (default: True)') 471 | parser.add_argument('--host', action='store', 472 | help='Get all the variables about a specific instance') 473 | parser.add_argument('--refresh-cache', action='store_true', default=False, 474 | help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') 475 | parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile', 476 | help='Use boto profile for connections to EC2') 477 | self.args = parser.parse_args() 478 | 479 | 480 | def do_api_calls_update_cache(self): 481 | ''' Do API calls to each region, and save data in cache files ''' 482 | 483 | if self.route53_enabled: 484 | self.get_route53_records() 485 | 486 | for region in self.regions: 487 | self.get_instances_by_region(region) 488 | if self.rds_enabled: 489 | self.get_rds_instances_by_region(region) 490 | if self.elasticache_enabled: 491 | self.get_elasticache_clusters_by_region(region) 492 | self.get_elasticache_replication_groups_by_region(region) 493 | if self.include_rds_clusters: 494 | self.include_rds_clusters_by_region(region) 495 | 496 | self.write_to_cache(self.inventory, self.cache_path_cache) 497 | self.write_to_cache(self.index, self.cache_path_index) 498 | 499 | def connect(self, region): 500 | ''' create connection to api server''' 501 | if self.eucalyptus: 502 | conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials) 503 | conn.APIVersion = '2010-08-31' 504 | else: 505 | conn = self.connect_to_aws(ec2, region) 506 | return conn 507 | 508 | def boto_fix_security_token_in_profile(self, connect_args): 509 | ''' monkey patch for boto issue boto/boto#2100 ''' 510 | profile = 'profile ' + self.boto_profile 511 | if boto.config.has_option(profile, 'aws_security_token'): 512 | connect_args['security_token'] = boto.config.get(profile, 'aws_security_token') 513 | return connect_args 514 | 515 | def connect_to_aws(self, module, region): 516 | connect_args = self.credentials 517 | 518 | # only pass the profile name if it's set (as it is not supported by older boto versions) 519 | if self.boto_profile: 520 | connect_args['profile_name'] = self.boto_profile 521 | self.boto_fix_security_token_in_profile(connect_args) 522 | 523 | conn = module.connect_to_region(region, **connect_args) 524 | # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported 525 | if conn is None: 526 | self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) 527 | return conn 528 | 529 | def get_instances_by_region(self, region): 530 | ''' Makes an AWS EC2 API call to the list of instances in a particular 531 | region ''' 532 | 533 | try: 534 | conn = self.connect(region) 535 | reservations = [] 536 | if self.ec2_instance_filters: 537 | for filter_key, filter_values in self.ec2_instance_filters.items(): 538 | reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values })) 539 | else: 540 | reservations = conn.get_all_instances() 541 | 542 | # Pull the tags back in a second step 543 | # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not 544 | # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags` 545 | instance_ids = [] 546 | for reservation in reservations: 547 | instance_ids.extend([instance.id for instance in reservation.instances]) 548 | 549 | max_filter_value = 199 550 | tags = [] 551 | for i in range(0, len(instance_ids), max_filter_value): 552 | tags.extend(conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids[i:i+max_filter_value]})) 553 | 554 | tags_by_instance_id = defaultdict(dict) 555 | for tag in tags: 556 | tags_by_instance_id[tag.res_id][tag.name] = tag.value 557 | 558 | for reservation in reservations: 559 | for instance in reservation.instances: 560 | instance.tags = tags_by_instance_id[instance.id] 561 | self.add_instance(instance, region) 562 | 563 | except boto.exception.BotoServerError as e: 564 | if e.error_code == 'AuthFailure': 565 | error = self.get_auth_error_message() 566 | else: 567 | backend = 'Eucalyptus' if self.eucalyptus else 'AWS' 568 | error = "Error connecting to %s backend.\n%s" % (backend, e.message) 569 | self.fail_with_error(error, 'getting EC2 instances') 570 | 571 | def get_rds_instances_by_region(self, region): 572 | ''' Makes an AWS API call to the list of RDS instances in a particular 573 | region ''' 574 | 575 | try: 576 | conn = self.connect_to_aws(rds, region) 577 | if conn: 578 | marker = None 579 | while True: 580 | instances = conn.get_all_dbinstances(marker=marker) 581 | marker = instances.marker 582 | for instance in instances: 583 | self.add_rds_instance(instance, region) 584 | if not marker: 585 | break 586 | except boto.exception.BotoServerError as e: 587 | error = e.reason 588 | 589 | if e.error_code == 'AuthFailure': 590 | error = self.get_auth_error_message() 591 | if not e.reason == "Forbidden": 592 | error = "Looks like AWS RDS is down:\n%s" % e.message 593 | self.fail_with_error(error, 'getting RDS instances') 594 | 595 | def include_rds_clusters_by_region(self, region): 596 | if not HAS_BOTO3: 597 | self.fail_with_error("Working with RDS clusters requires boto3 - please install boto3 and try again", 598 | "getting RDS clusters") 599 | 600 | client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials) 601 | 602 | marker, clusters = '', [] 603 | while marker is not None: 604 | resp = client.describe_db_clusters(Marker=marker) 605 | clusters.extend(resp["DBClusters"]) 606 | marker = resp.get('Marker', None) 607 | 608 | account_id = boto.connect_iam().get_user().arn.split(':')[4] 609 | c_dict = {} 610 | for c in clusters: 611 | # remove these datetime objects as there is no serialisation to json 612 | # currently in place and we don't need the data yet 613 | if 'EarliestRestorableTime' in c: 614 | del c['EarliestRestorableTime'] 615 | if 'LatestRestorableTime' in c: 616 | del c['LatestRestorableTime'] 617 | 618 | if self.ec2_instance_filters == {}: 619 | matches_filter = True 620 | else: 621 | matches_filter = False 622 | 623 | try: 624 | # arn:aws:rds:::: 625 | tags = client.list_tags_for_resource( 626 | ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier']) 627 | c['Tags'] = tags['TagList'] 628 | 629 | if self.ec2_instance_filters: 630 | for filter_key, filter_values in self.ec2_instance_filters.items(): 631 | # get AWS tag key e.g. tag:env will be 'env' 632 | tag_name = filter_key.split(":", 1)[1] 633 | # Filter values is a list (if you put multiple values for the same tag name) 634 | matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags']) 635 | 636 | if matches_filter: 637 | # it matches a filter, so stop looking for further matches 638 | break 639 | 640 | except Exception as e: 641 | if e.message.find('DBInstanceNotFound') >= 0: 642 | # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster. 643 | # Ignore errors when trying to find tags for these 644 | pass 645 | 646 | # ignore empty clusters caused by AWS bug 647 | if len(c['DBClusterMembers']) == 0: 648 | continue 649 | elif matches_filter: 650 | c_dict[c['DBClusterIdentifier']] = c 651 | 652 | self.inventory['db_clusters'] = c_dict 653 | 654 | def get_elasticache_clusters_by_region(self, region): 655 | ''' Makes an AWS API call to the list of ElastiCache clusters (with 656 | nodes' info) in a particular region.''' 657 | 658 | # ElastiCache boto module doesn't provide a get_all_intances method, 659 | # that's why we need to call describe directly (it would be called by 660 | # the shorthand method anyway...) 661 | try: 662 | conn = self.connect_to_aws(elasticache, region) 663 | if conn: 664 | # show_cache_node_info = True 665 | # because we also want nodes' information 666 | response = conn.describe_cache_clusters(None, None, None, True) 667 | 668 | except boto.exception.BotoServerError as e: 669 | error = e.reason 670 | 671 | if e.error_code == 'AuthFailure': 672 | error = self.get_auth_error_message() 673 | if not e.reason == "Forbidden": 674 | error = "Looks like AWS ElastiCache is down:\n%s" % e.message 675 | self.fail_with_error(error, 'getting ElastiCache clusters') 676 | 677 | try: 678 | # Boto also doesn't provide wrapper classes to CacheClusters or 679 | # CacheNodes. Because of that wo can't make use of the get_list 680 | # method in the AWSQueryConnection. Let's do the work manually 681 | clusters = response['DescribeCacheClustersResponse']['DescribeCacheClustersResult']['CacheClusters'] 682 | 683 | except KeyError as e: 684 | error = "ElastiCache query to AWS failed (unexpected format)." 685 | self.fail_with_error(error, 'getting ElastiCache clusters') 686 | 687 | for cluster in clusters: 688 | self.add_elasticache_cluster(cluster, region) 689 | 690 | def get_elasticache_replication_groups_by_region(self, region): 691 | ''' Makes an AWS API call to the list of ElastiCache replication groups 692 | in a particular region.''' 693 | 694 | # ElastiCache boto module doesn't provide a get_all_intances method, 695 | # that's why we need to call describe directly (it would be called by 696 | # the shorthand method anyway...) 697 | try: 698 | conn = self.connect_to_aws(elasticache, region) 699 | if conn: 700 | response = conn.describe_replication_groups() 701 | 702 | except boto.exception.BotoServerError as e: 703 | error = e.reason 704 | 705 | if e.error_code == 'AuthFailure': 706 | error = self.get_auth_error_message() 707 | if not e.reason == "Forbidden": 708 | error = "Looks like AWS ElastiCache [Replication Groups] is down:\n%s" % e.message 709 | self.fail_with_error(error, 'getting ElastiCache clusters') 710 | 711 | try: 712 | # Boto also doesn't provide wrapper classes to ReplicationGroups 713 | # Because of that wo can't make use of the get_list method in the 714 | # AWSQueryConnection. Let's do the work manually 715 | replication_groups = response['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']['ReplicationGroups'] 716 | 717 | except KeyError as e: 718 | error = "ElastiCache [Replication Groups] query to AWS failed (unexpected format)." 719 | self.fail_with_error(error, 'getting ElastiCache clusters') 720 | 721 | for replication_group in replication_groups: 722 | self.add_elasticache_replication_group(replication_group, region) 723 | 724 | def get_auth_error_message(self): 725 | ''' create an informative error message if there is an issue authenticating''' 726 | errors = ["Authentication error retrieving ec2 inventory."] 727 | if None in [os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')]: 728 | errors.append(' - No AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment vars found') 729 | else: 730 | errors.append(' - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment vars found but may not be correct') 731 | 732 | boto_paths = ['/etc/boto.cfg', '~/.boto', '~/.aws/credentials'] 733 | boto_config_found = list(p for p in boto_paths if os.path.isfile(os.path.expanduser(p))) 734 | if len(boto_config_found) > 0: 735 | errors.append(" - Boto configs found at '%s', but the credentials contained may not be correct" % ', '.join(boto_config_found)) 736 | else: 737 | errors.append(" - No Boto config found at any expected location '%s'" % ', '.join(boto_paths)) 738 | 739 | return '\n'.join(errors) 740 | 741 | def fail_with_error(self, err_msg, err_operation=None): 742 | '''log an error to std err for ansible-playbook to consume and exit''' 743 | if err_operation: 744 | err_msg = 'ERROR: "{err_msg}", while: {err_operation}'.format( 745 | err_msg=err_msg, err_operation=err_operation) 746 | sys.stderr.write(err_msg) 747 | sys.exit(1) 748 | 749 | def get_instance(self, region, instance_id): 750 | conn = self.connect(region) 751 | 752 | reservations = conn.get_all_instances([instance_id]) 753 | for reservation in reservations: 754 | for instance in reservation.instances: 755 | return instance 756 | 757 | def add_instance(self, instance, region): 758 | ''' Adds an instance to the inventory and index, as long as it is 759 | addressable ''' 760 | 761 | # Only return instances with desired instance states 762 | if instance.state not in self.ec2_instance_states: 763 | return 764 | 765 | # Select the best destination address 766 | if self.destination_format and self.destination_format_tags: 767 | dest = self.destination_format.format(*[ getattr(instance, 'tags').get(tag, '') for tag in self.destination_format_tags ]) 768 | elif instance.subnet_id: 769 | dest = getattr(instance, self.vpc_destination_variable, None) 770 | if dest is None: 771 | dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None) 772 | else: 773 | dest = getattr(instance, self.destination_variable, None) 774 | if dest is None: 775 | dest = getattr(instance, 'tags').get(self.destination_variable, None) 776 | 777 | if not dest: 778 | # Skip instances we cannot address (e.g. private VPC subnet) 779 | return 780 | 781 | # Set the inventory name 782 | hostname = None 783 | if self.hostname_variable: 784 | if self.hostname_variable.startswith('tag_'): 785 | hostname = instance.tags.get(self.hostname_variable[4:], None) 786 | else: 787 | hostname = getattr(instance, self.hostname_variable) 788 | 789 | # If we can't get a nice hostname, use the destination address 790 | if not hostname: 791 | hostname = dest 792 | else: 793 | hostname = self.to_safe(hostname).lower() 794 | 795 | # if we only want to include hosts that match a pattern, skip those that don't 796 | if self.pattern_include and not self.pattern_include.match(hostname): 797 | return 798 | 799 | # if we need to exclude hosts that match a pattern, skip those 800 | if self.pattern_exclude and self.pattern_exclude.match(hostname): 801 | return 802 | 803 | # Add to index 804 | self.index[hostname] = [region, instance.id] 805 | 806 | # Inventory: Group by instance ID (always a group of 1) 807 | if self.group_by_instance_id: 808 | self.inventory[instance.id] = [hostname] 809 | if self.nested_groups: 810 | self.push_group(self.inventory, 'instances', instance.id) 811 | 812 | # Inventory: Group by region 813 | if self.group_by_region: 814 | self.push(self.inventory, region, hostname) 815 | if self.nested_groups: 816 | self.push_group(self.inventory, 'regions', region) 817 | 818 | # Inventory: Group by availability zone 819 | if self.group_by_availability_zone: 820 | self.push(self.inventory, instance.placement, hostname) 821 | if self.nested_groups: 822 | if self.group_by_region: 823 | self.push_group(self.inventory, region, instance.placement) 824 | self.push_group(self.inventory, 'zones', instance.placement) 825 | 826 | # Inventory: Group by Amazon Machine Image (AMI) ID 827 | if self.group_by_ami_id: 828 | ami_id = self.to_safe(instance.image_id) 829 | self.push(self.inventory, ami_id, hostname) 830 | if self.nested_groups: 831 | self.push_group(self.inventory, 'images', ami_id) 832 | 833 | # Inventory: Group by instance type 834 | if self.group_by_instance_type: 835 | type_name = self.to_safe('type_' + instance.instance_type) 836 | self.push(self.inventory, type_name, hostname) 837 | if self.nested_groups: 838 | self.push_group(self.inventory, 'types', type_name) 839 | 840 | # Inventory: Group by key pair 841 | if self.group_by_key_pair and instance.key_name: 842 | key_name = self.to_safe('key_' + instance.key_name) 843 | self.push(self.inventory, key_name, hostname) 844 | if self.nested_groups: 845 | self.push_group(self.inventory, 'keys', key_name) 846 | 847 | # Inventory: Group by VPC 848 | if self.group_by_vpc_id and instance.vpc_id: 849 | vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id) 850 | self.push(self.inventory, vpc_id_name, hostname) 851 | if self.nested_groups: 852 | self.push_group(self.inventory, 'vpcs', vpc_id_name) 853 | 854 | # Inventory: Group by security group 855 | if self.group_by_security_group: 856 | try: 857 | for group in instance.groups: 858 | key = self.to_safe("security_group_" + group.name) 859 | self.push(self.inventory, key, hostname) 860 | if self.nested_groups: 861 | self.push_group(self.inventory, 'security_groups', key) 862 | except AttributeError: 863 | self.fail_with_error('\n'.join(['Package boto seems a bit older.', 864 | 'Please upgrade boto >= 2.3.0.'])) 865 | 866 | # Inventory: Group by tag keys 867 | if self.group_by_tag_keys: 868 | for k, v in instance.tags.items(): 869 | if self.expand_csv_tags and v and ',' in v: 870 | values = map(lambda x: x.strip(), v.split(',')) 871 | else: 872 | values = [v] 873 | 874 | for v in values: 875 | if v: 876 | key = self.to_safe("tag_" + k + "=" + v) 877 | else: 878 | key = self.to_safe("tag_" + k) 879 | self.push(self.inventory, key, hostname) 880 | if self.nested_groups: 881 | self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) 882 | if v: 883 | self.push_group(self.inventory, self.to_safe("tag_" + k), key) 884 | 885 | # Inventory: Group by Route53 domain names if enabled 886 | if self.route53_enabled and self.group_by_route53_names: 887 | route53_names = self.get_instance_route53_names(instance) 888 | for name in route53_names: 889 | self.push(self.inventory, name, hostname) 890 | if self.nested_groups: 891 | self.push_group(self.inventory, 'route53', name) 892 | 893 | # Global Tag: instances without tags 894 | if self.group_by_tag_none and len(instance.tags) == 0: 895 | self.push(self.inventory, 'tag_none', hostname) 896 | if self.nested_groups: 897 | self.push_group(self.inventory, 'tags', 'tag_none') 898 | 899 | # Global Tag: tag all EC2 instances 900 | self.push(self.inventory, 'ec2', hostname) 901 | 902 | self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) 903 | self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest 904 | 905 | 906 | def add_rds_instance(self, instance, region): 907 | ''' Adds an RDS instance to the inventory and index, as long as it is 908 | addressable ''' 909 | 910 | # Only want available instances unless all_rds_instances is True 911 | if not self.all_rds_instances and instance.status != 'available': 912 | return 913 | 914 | # Select the best destination address 915 | dest = instance.endpoint[0] 916 | 917 | if not dest: 918 | # Skip instances we cannot address (e.g. private VPC subnet) 919 | return 920 | 921 | # Set the inventory name 922 | hostname = None 923 | if self.hostname_variable: 924 | if self.hostname_variable.startswith('tag_'): 925 | hostname = instance.tags.get(self.hostname_variable[4:], None) 926 | else: 927 | hostname = getattr(instance, self.hostname_variable) 928 | 929 | # If we can't get a nice hostname, use the destination address 930 | if not hostname: 931 | hostname = dest 932 | 933 | hostname = self.to_safe(hostname).lower() 934 | 935 | # Add to index 936 | self.index[hostname] = [region, instance.id] 937 | 938 | # Inventory: Group by instance ID (always a group of 1) 939 | if self.group_by_instance_id: 940 | self.inventory[instance.id] = [hostname] 941 | if self.nested_groups: 942 | self.push_group(self.inventory, 'instances', instance.id) 943 | 944 | # Inventory: Group by region 945 | if self.group_by_region: 946 | self.push(self.inventory, region, hostname) 947 | if self.nested_groups: 948 | self.push_group(self.inventory, 'regions', region) 949 | 950 | # Inventory: Group by availability zone 951 | if self.group_by_availability_zone: 952 | self.push(self.inventory, instance.availability_zone, hostname) 953 | if self.nested_groups: 954 | if self.group_by_region: 955 | self.push_group(self.inventory, region, instance.availability_zone) 956 | self.push_group(self.inventory, 'zones', instance.availability_zone) 957 | 958 | # Inventory: Group by instance type 959 | if self.group_by_instance_type: 960 | type_name = self.to_safe('type_' + instance.instance_class) 961 | self.push(self.inventory, type_name, hostname) 962 | if self.nested_groups: 963 | self.push_group(self.inventory, 'types', type_name) 964 | 965 | # Inventory: Group by VPC 966 | if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id: 967 | vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id) 968 | self.push(self.inventory, vpc_id_name, hostname) 969 | if self.nested_groups: 970 | self.push_group(self.inventory, 'vpcs', vpc_id_name) 971 | 972 | # Inventory: Group by security group 973 | if self.group_by_security_group: 974 | try: 975 | if instance.security_group: 976 | key = self.to_safe("security_group_" + instance.security_group.name) 977 | self.push(self.inventory, key, hostname) 978 | if self.nested_groups: 979 | self.push_group(self.inventory, 'security_groups', key) 980 | 981 | except AttributeError: 982 | self.fail_with_error('\n'.join(['Package boto seems a bit older.', 983 | 'Please upgrade boto >= 2.3.0.'])) 984 | 985 | 986 | # Inventory: Group by engine 987 | if self.group_by_rds_engine: 988 | self.push(self.inventory, self.to_safe("rds_" + instance.engine), hostname) 989 | if self.nested_groups: 990 | self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine)) 991 | 992 | # Inventory: Group by parameter group 993 | if self.group_by_rds_parameter_group: 994 | self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), hostname) 995 | if self.nested_groups: 996 | self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name)) 997 | 998 | # Global Tag: all RDS instances 999 | self.push(self.inventory, 'rds', hostname) 1000 | 1001 | self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) 1002 | self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest 1003 | 1004 | def add_elasticache_cluster(self, cluster, region): 1005 | ''' Adds an ElastiCache cluster to the inventory and index, as long as 1006 | it's nodes are addressable ''' 1007 | 1008 | # Only want available clusters unless all_elasticache_clusters is True 1009 | if not self.all_elasticache_clusters and cluster['CacheClusterStatus'] != 'available': 1010 | return 1011 | 1012 | # Select the best destination address 1013 | if 'ConfigurationEndpoint' in cluster and cluster['ConfigurationEndpoint']: 1014 | # Memcached cluster 1015 | dest = cluster['ConfigurationEndpoint']['Address'] 1016 | is_redis = False 1017 | else: 1018 | # Redis sigle node cluster 1019 | # Because all Redis clusters are single nodes, we'll merge the 1020 | # info from the cluster with info about the node 1021 | dest = cluster['CacheNodes'][0]['Endpoint']['Address'] 1022 | is_redis = True 1023 | 1024 | if not dest: 1025 | # Skip clusters we cannot address (e.g. private VPC subnet) 1026 | return 1027 | 1028 | # Add to index 1029 | self.index[dest] = [region, cluster['CacheClusterId']] 1030 | 1031 | # Inventory: Group by instance ID (always a group of 1) 1032 | if self.group_by_instance_id: 1033 | self.inventory[cluster['CacheClusterId']] = [dest] 1034 | if self.nested_groups: 1035 | self.push_group(self.inventory, 'instances', cluster['CacheClusterId']) 1036 | 1037 | # Inventory: Group by region 1038 | if self.group_by_region and not is_redis: 1039 | self.push(self.inventory, region, dest) 1040 | if self.nested_groups: 1041 | self.push_group(self.inventory, 'regions', region) 1042 | 1043 | # Inventory: Group by availability zone 1044 | if self.group_by_availability_zone and not is_redis: 1045 | self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) 1046 | if self.nested_groups: 1047 | if self.group_by_region: 1048 | self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) 1049 | self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) 1050 | 1051 | # Inventory: Group by node type 1052 | if self.group_by_instance_type and not is_redis: 1053 | type_name = self.to_safe('type_' + cluster['CacheNodeType']) 1054 | self.push(self.inventory, type_name, dest) 1055 | if self.nested_groups: 1056 | self.push_group(self.inventory, 'types', type_name) 1057 | 1058 | # Inventory: Group by VPC (information not available in the current 1059 | # AWS API version for ElastiCache) 1060 | 1061 | # Inventory: Group by security group 1062 | if self.group_by_security_group and not is_redis: 1063 | 1064 | # Check for the existence of the 'SecurityGroups' key and also if 1065 | # this key has some value. When the cluster is not placed in a SG 1066 | # the query can return None here and cause an error. 1067 | if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: 1068 | for security_group in cluster['SecurityGroups']: 1069 | key = self.to_safe("security_group_" + security_group['SecurityGroupId']) 1070 | self.push(self.inventory, key, dest) 1071 | if self.nested_groups: 1072 | self.push_group(self.inventory, 'security_groups', key) 1073 | 1074 | # Inventory: Group by engine 1075 | if self.group_by_elasticache_engine and not is_redis: 1076 | self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) 1077 | if self.nested_groups: 1078 | self.push_group(self.inventory, 'elasticache_engines', self.to_safe(cluster['Engine'])) 1079 | 1080 | # Inventory: Group by parameter group 1081 | if self.group_by_elasticache_parameter_group: 1082 | self.push(self.inventory, self.to_safe("elasticache_parameter_group_" + cluster['CacheParameterGroup']['CacheParameterGroupName']), dest) 1083 | if self.nested_groups: 1084 | self.push_group(self.inventory, 'elasticache_parameter_groups', self.to_safe(cluster['CacheParameterGroup']['CacheParameterGroupName'])) 1085 | 1086 | # Inventory: Group by replication group 1087 | if self.group_by_elasticache_replication_group and 'ReplicationGroupId' in cluster and cluster['ReplicationGroupId']: 1088 | self.push(self.inventory, self.to_safe("elasticache_replication_group_" + cluster['ReplicationGroupId']), dest) 1089 | if self.nested_groups: 1090 | self.push_group(self.inventory, 'elasticache_replication_groups', self.to_safe(cluster['ReplicationGroupId'])) 1091 | 1092 | # Global Tag: all ElastiCache clusters 1093 | self.push(self.inventory, 'elasticache_clusters', cluster['CacheClusterId']) 1094 | 1095 | host_info = self.get_host_info_dict_from_describe_dict(cluster) 1096 | 1097 | self.inventory["_meta"]["hostvars"][dest] = host_info 1098 | 1099 | # Add the nodes 1100 | for node in cluster['CacheNodes']: 1101 | self.add_elasticache_node(node, cluster, region) 1102 | 1103 | def add_elasticache_node(self, node, cluster, region): 1104 | ''' Adds an ElastiCache node to the inventory and index, as long as 1105 | it is addressable ''' 1106 | 1107 | # Only want available nodes unless all_elasticache_nodes is True 1108 | if not self.all_elasticache_nodes and node['CacheNodeStatus'] != 'available': 1109 | return 1110 | 1111 | # Select the best destination address 1112 | dest = node['Endpoint']['Address'] 1113 | 1114 | if not dest: 1115 | # Skip nodes we cannot address (e.g. private VPC subnet) 1116 | return 1117 | 1118 | node_id = self.to_safe(cluster['CacheClusterId'] + '_' + node['CacheNodeId']) 1119 | 1120 | # Add to index 1121 | self.index[dest] = [region, node_id] 1122 | 1123 | # Inventory: Group by node ID (always a group of 1) 1124 | if self.group_by_instance_id: 1125 | self.inventory[node_id] = [dest] 1126 | if self.nested_groups: 1127 | self.push_group(self.inventory, 'instances', node_id) 1128 | 1129 | # Inventory: Group by region 1130 | if self.group_by_region: 1131 | self.push(self.inventory, region, dest) 1132 | if self.nested_groups: 1133 | self.push_group(self.inventory, 'regions', region) 1134 | 1135 | # Inventory: Group by availability zone 1136 | if self.group_by_availability_zone: 1137 | self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) 1138 | if self.nested_groups: 1139 | if self.group_by_region: 1140 | self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) 1141 | self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) 1142 | 1143 | # Inventory: Group by node type 1144 | if self.group_by_instance_type: 1145 | type_name = self.to_safe('type_' + cluster['CacheNodeType']) 1146 | self.push(self.inventory, type_name, dest) 1147 | if self.nested_groups: 1148 | self.push_group(self.inventory, 'types', type_name) 1149 | 1150 | # Inventory: Group by VPC (information not available in the current 1151 | # AWS API version for ElastiCache) 1152 | 1153 | # Inventory: Group by security group 1154 | if self.group_by_security_group: 1155 | 1156 | # Check for the existence of the 'SecurityGroups' key and also if 1157 | # this key has some value. When the cluster is not placed in a SG 1158 | # the query can return None here and cause an error. 1159 | if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: 1160 | for security_group in cluster['SecurityGroups']: 1161 | key = self.to_safe("security_group_" + security_group['SecurityGroupId']) 1162 | self.push(self.inventory, key, dest) 1163 | if self.nested_groups: 1164 | self.push_group(self.inventory, 'security_groups', key) 1165 | 1166 | # Inventory: Group by engine 1167 | if self.group_by_elasticache_engine: 1168 | self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) 1169 | if self.nested_groups: 1170 | self.push_group(self.inventory, 'elasticache_engines', self.to_safe("elasticache_" + cluster['Engine'])) 1171 | 1172 | # Inventory: Group by parameter group (done at cluster level) 1173 | 1174 | # Inventory: Group by replication group (done at cluster level) 1175 | 1176 | # Inventory: Group by ElastiCache Cluster 1177 | if self.group_by_elasticache_cluster: 1178 | self.push(self.inventory, self.to_safe("elasticache_cluster_" + cluster['CacheClusterId']), dest) 1179 | 1180 | # Global Tag: all ElastiCache nodes 1181 | self.push(self.inventory, 'elasticache_nodes', dest) 1182 | 1183 | host_info = self.get_host_info_dict_from_describe_dict(node) 1184 | 1185 | if dest in self.inventory["_meta"]["hostvars"]: 1186 | self.inventory["_meta"]["hostvars"][dest].update(host_info) 1187 | else: 1188 | self.inventory["_meta"]["hostvars"][dest] = host_info 1189 | 1190 | def add_elasticache_replication_group(self, replication_group, region): 1191 | ''' Adds an ElastiCache replication group to the inventory and index ''' 1192 | 1193 | # Only want available clusters unless all_elasticache_replication_groups is True 1194 | if not self.all_elasticache_replication_groups and replication_group['Status'] != 'available': 1195 | return 1196 | 1197 | # Select the best destination address (PrimaryEndpoint) 1198 | dest = replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address'] 1199 | 1200 | if not dest: 1201 | # Skip clusters we cannot address (e.g. private VPC subnet) 1202 | return 1203 | 1204 | # Add to index 1205 | self.index[dest] = [region, replication_group['ReplicationGroupId']] 1206 | 1207 | # Inventory: Group by ID (always a group of 1) 1208 | if self.group_by_instance_id: 1209 | self.inventory[replication_group['ReplicationGroupId']] = [dest] 1210 | if self.nested_groups: 1211 | self.push_group(self.inventory, 'instances', replication_group['ReplicationGroupId']) 1212 | 1213 | # Inventory: Group by region 1214 | if self.group_by_region: 1215 | self.push(self.inventory, region, dest) 1216 | if self.nested_groups: 1217 | self.push_group(self.inventory, 'regions', region) 1218 | 1219 | # Inventory: Group by availability zone (doesn't apply to replication groups) 1220 | 1221 | # Inventory: Group by node type (doesn't apply to replication groups) 1222 | 1223 | # Inventory: Group by VPC (information not available in the current 1224 | # AWS API version for replication groups 1225 | 1226 | # Inventory: Group by security group (doesn't apply to replication groups) 1227 | # Check this value in cluster level 1228 | 1229 | # Inventory: Group by engine (replication groups are always Redis) 1230 | if self.group_by_elasticache_engine: 1231 | self.push(self.inventory, 'elasticache_redis', dest) 1232 | if self.nested_groups: 1233 | self.push_group(self.inventory, 'elasticache_engines', 'redis') 1234 | 1235 | # Global Tag: all ElastiCache clusters 1236 | self.push(self.inventory, 'elasticache_replication_groups', replication_group['ReplicationGroupId']) 1237 | 1238 | host_info = self.get_host_info_dict_from_describe_dict(replication_group) 1239 | 1240 | self.inventory["_meta"]["hostvars"][dest] = host_info 1241 | 1242 | def get_route53_records(self): 1243 | ''' Get and store the map of resource records to domain names that 1244 | point to them. ''' 1245 | 1246 | r53_conn = route53.Route53Connection() 1247 | all_zones = r53_conn.get_zones() 1248 | 1249 | route53_zones = [ zone for zone in all_zones if zone.name[:-1] 1250 | not in self.route53_excluded_zones ] 1251 | 1252 | self.route53_records = {} 1253 | 1254 | for zone in route53_zones: 1255 | rrsets = r53_conn.get_all_rrsets(zone.id) 1256 | 1257 | for record_set in rrsets: 1258 | record_name = record_set.name 1259 | 1260 | if record_name.endswith('.'): 1261 | record_name = record_name[:-1] 1262 | 1263 | for resource in record_set.resource_records: 1264 | self.route53_records.setdefault(resource, set()) 1265 | self.route53_records[resource].add(record_name) 1266 | 1267 | 1268 | def get_instance_route53_names(self, instance): 1269 | ''' Check if an instance is referenced in the records we have from 1270 | Route53. If it is, return the list of domain names pointing to said 1271 | instance. If nothing points to it, return an empty list. ''' 1272 | 1273 | instance_attributes = [ 'public_dns_name', 'private_dns_name', 1274 | 'ip_address', 'private_ip_address' ] 1275 | 1276 | name_list = set() 1277 | 1278 | for attrib in instance_attributes: 1279 | try: 1280 | value = getattr(instance, attrib) 1281 | except AttributeError: 1282 | continue 1283 | 1284 | if value in self.route53_records: 1285 | name_list.update(self.route53_records[value]) 1286 | 1287 | return list(name_list) 1288 | 1289 | def get_host_info_dict_from_instance(self, instance): 1290 | instance_vars = {} 1291 | for key in vars(instance): 1292 | value = getattr(instance, key) 1293 | key = self.to_safe('ec2_' + key) 1294 | 1295 | # Handle complex types 1296 | # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518 1297 | if key == 'ec2__state': 1298 | instance_vars['ec2_state'] = instance.state or '' 1299 | instance_vars['ec2_state_code'] = instance.state_code 1300 | elif key == 'ec2__previous_state': 1301 | instance_vars['ec2_previous_state'] = instance.previous_state or '' 1302 | instance_vars['ec2_previous_state_code'] = instance.previous_state_code 1303 | elif type(value) in [int, bool]: 1304 | instance_vars[key] = value 1305 | elif isinstance(value, six.string_types): 1306 | instance_vars[key] = value.strip() 1307 | elif type(value) == type(None): 1308 | instance_vars[key] = '' 1309 | elif key == 'ec2_region': 1310 | instance_vars[key] = value.name 1311 | elif key == 'ec2__placement': 1312 | instance_vars['ec2_placement'] = value.zone 1313 | elif key == 'ec2_tags': 1314 | for k, v in value.items(): 1315 | if self.expand_csv_tags and ',' in v: 1316 | v = list(map(lambda x: x.strip(), v.split(','))) 1317 | key = self.to_safe('ec2_tag_' + k) 1318 | instance_vars[key] = v 1319 | elif key == 'ec2_groups': 1320 | group_ids = [] 1321 | group_names = [] 1322 | for group in value: 1323 | group_ids.append(group.id) 1324 | group_names.append(group.name) 1325 | instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids]) 1326 | instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names]) 1327 | elif key == 'ec2_block_device_mapping': 1328 | instance_vars["ec2_block_devices"] = {} 1329 | for k, v in value.items(): 1330 | instance_vars["ec2_block_devices"][ os.path.basename(k) ] = v.volume_id 1331 | else: 1332 | pass 1333 | # TODO Product codes if someone finds them useful 1334 | #print key 1335 | #print type(value) 1336 | #print value 1337 | 1338 | return instance_vars 1339 | 1340 | def get_host_info_dict_from_describe_dict(self, describe_dict): 1341 | ''' Parses the dictionary returned by the API call into a flat list 1342 | of parameters. This method should be used only when 'describe' is 1343 | used directly because Boto doesn't provide specific classes. ''' 1344 | 1345 | # I really don't agree with prefixing everything with 'ec2' 1346 | # because EC2, RDS and ElastiCache are different services. 1347 | # I'm just following the pattern used until now to not break any 1348 | # compatibility. 1349 | 1350 | host_info = {} 1351 | for key in describe_dict: 1352 | value = describe_dict[key] 1353 | key = self.to_safe('ec2_' + self.uncammelize(key)) 1354 | 1355 | # Handle complex types 1356 | 1357 | # Target: Memcached Cache Clusters 1358 | if key == 'ec2_configuration_endpoint' and value: 1359 | host_info['ec2_configuration_endpoint_address'] = value['Address'] 1360 | host_info['ec2_configuration_endpoint_port'] = value['Port'] 1361 | 1362 | # Target: Cache Nodes and Redis Cache Clusters (single node) 1363 | if key == 'ec2_endpoint' and value: 1364 | host_info['ec2_endpoint_address'] = value['Address'] 1365 | host_info['ec2_endpoint_port'] = value['Port'] 1366 | 1367 | # Target: Redis Replication Groups 1368 | if key == 'ec2_node_groups' and value: 1369 | host_info['ec2_endpoint_address'] = value[0]['PrimaryEndpoint']['Address'] 1370 | host_info['ec2_endpoint_port'] = value[0]['PrimaryEndpoint']['Port'] 1371 | replica_count = 0 1372 | for node in value[0]['NodeGroupMembers']: 1373 | if node['CurrentRole'] == 'primary': 1374 | host_info['ec2_primary_cluster_address'] = node['ReadEndpoint']['Address'] 1375 | host_info['ec2_primary_cluster_port'] = node['ReadEndpoint']['Port'] 1376 | host_info['ec2_primary_cluster_id'] = node['CacheClusterId'] 1377 | elif node['CurrentRole'] == 'replica': 1378 | host_info['ec2_replica_cluster_address_'+ str(replica_count)] = node['ReadEndpoint']['Address'] 1379 | host_info['ec2_replica_cluster_port_'+ str(replica_count)] = node['ReadEndpoint']['Port'] 1380 | host_info['ec2_replica_cluster_id_'+ str(replica_count)] = node['CacheClusterId'] 1381 | replica_count += 1 1382 | 1383 | # Target: Redis Replication Groups 1384 | if key == 'ec2_member_clusters' and value: 1385 | host_info['ec2_member_clusters'] = ','.join([str(i) for i in value]) 1386 | 1387 | # Target: All Cache Clusters 1388 | elif key == 'ec2_cache_parameter_group': 1389 | host_info["ec2_cache_node_ids_to_reboot"] = ','.join([str(i) for i in value['CacheNodeIdsToReboot']]) 1390 | host_info['ec2_cache_parameter_group_name'] = value['CacheParameterGroupName'] 1391 | host_info['ec2_cache_parameter_apply_status'] = value['ParameterApplyStatus'] 1392 | 1393 | # Target: Almost everything 1394 | elif key == 'ec2_security_groups': 1395 | 1396 | # Skip if SecurityGroups is None 1397 | # (it is possible to have the key defined but no value in it). 1398 | if value is not None: 1399 | sg_ids = [] 1400 | for sg in value: 1401 | sg_ids.append(sg['SecurityGroupId']) 1402 | host_info["ec2_security_group_ids"] = ','.join([str(i) for i in sg_ids]) 1403 | 1404 | # Target: Everything 1405 | # Preserve booleans and integers 1406 | elif type(value) in [int, bool]: 1407 | host_info[key] = value 1408 | 1409 | # Target: Everything 1410 | # Sanitize string values 1411 | elif isinstance(value, six.string_types): 1412 | host_info[key] = value.strip() 1413 | 1414 | # Target: Everything 1415 | # Replace None by an empty string 1416 | elif type(value) == type(None): 1417 | host_info[key] = '' 1418 | 1419 | else: 1420 | # Remove non-processed complex types 1421 | pass 1422 | 1423 | return host_info 1424 | 1425 | def get_host_info(self): 1426 | ''' Get variables about a specific host ''' 1427 | 1428 | if len(self.index) == 0: 1429 | # Need to load index from cache 1430 | self.load_index_from_cache() 1431 | 1432 | if not self.args.host in self.index: 1433 | # try updating the cache 1434 | self.do_api_calls_update_cache() 1435 | if not self.args.host in self.index: 1436 | # host might not exist anymore 1437 | return self.json_format_dict({}, True) 1438 | 1439 | (region, instance_id) = self.index[self.args.host] 1440 | 1441 | instance = self.get_instance(region, instance_id) 1442 | return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) 1443 | 1444 | def push(self, my_dict, key, element): 1445 | ''' Push an element onto an array that may not have been defined in 1446 | the dict ''' 1447 | group_info = my_dict.setdefault(key, []) 1448 | if isinstance(group_info, dict): 1449 | host_list = group_info.setdefault('hosts', []) 1450 | host_list.append(element) 1451 | else: 1452 | group_info.append(element) 1453 | 1454 | def push_group(self, my_dict, key, element): 1455 | ''' Push a group as a child of another group. ''' 1456 | parent_group = my_dict.setdefault(key, {}) 1457 | if not isinstance(parent_group, dict): 1458 | parent_group = my_dict[key] = {'hosts': parent_group} 1459 | child_groups = parent_group.setdefault('children', []) 1460 | if element not in child_groups: 1461 | child_groups.append(element) 1462 | 1463 | def get_inventory_from_cache(self): 1464 | ''' Reads the inventory from the cache file and returns it as a JSON 1465 | object ''' 1466 | 1467 | cache = open(self.cache_path_cache, 'r') 1468 | json_inventory = cache.read() 1469 | return json_inventory 1470 | 1471 | 1472 | def load_index_from_cache(self): 1473 | ''' Reads the index from the cache file sets self.index ''' 1474 | 1475 | cache = open(self.cache_path_index, 'r') 1476 | json_index = cache.read() 1477 | self.index = json.loads(json_index) 1478 | 1479 | 1480 | def write_to_cache(self, data, filename): 1481 | ''' Writes data in JSON format to a file ''' 1482 | 1483 | json_data = self.json_format_dict(data, True) 1484 | cache = open(filename, 'w') 1485 | cache.write(json_data) 1486 | cache.close() 1487 | 1488 | def uncammelize(self, key): 1489 | temp = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', key) 1490 | return re.sub('([a-z0-9])([A-Z])', r'\1_\2', temp).lower() 1491 | 1492 | def to_safe(self, word): 1493 | ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' 1494 | regex = "[^A-Za-z0-9\_" 1495 | if not self.replace_dash_in_groups: 1496 | regex += "\-" 1497 | return re.sub(regex + "]", "_", word) 1498 | 1499 | def json_format_dict(self, data, pretty=False): 1500 | ''' Converts a dict to a JSON object and dumps it as a formatted 1501 | string ''' 1502 | 1503 | if pretty: 1504 | return json.dumps(data, sort_keys=True, indent=2) 1505 | else: 1506 | return json.dumps(data) 1507 | 1508 | 1509 | # Run the script 1510 | Ec2Inventory() 1511 | --------------------------------------------------------------------------------