├── .gitignore ├── README.md ├── images └── consul │ ├── README.md │ ├── v1 │ ├── build.sh │ ├── files │ │ ├── bin │ │ │ └── consul-config.sh │ │ └── systemd │ │ │ ├── consul.service │ │ │ ├── format-data-volume.service │ │ │ ├── generate-consul-config.service │ │ │ └── mnt-data.mount │ ├── scripts │ │ ├── base.sh │ │ ├── configure_dnsmasq.sh │ │ ├── configure_network.sh │ │ ├── copy_files.sh │ │ ├── disable_selinux.sh │ │ ├── enable_services.sh │ │ ├── install_awscli.sh │ │ ├── install_consul.sh │ │ ├── prepare_staging.sh │ │ └── update.sh │ └── template.json │ └── v2 │ ├── build.sh │ ├── files │ ├── bin │ │ └── consul-config.sh │ └── systemd │ │ ├── consul.service │ │ ├── format-data-volume.service │ │ ├── generate-consul-config.service │ │ └── mnt-data.mount │ ├── scripts │ ├── base.sh │ ├── configure_dnsmasq.sh │ ├── configure_network.sh │ ├── copy_files.sh │ ├── disable_selinux.sh │ ├── enable_services.sh │ ├── install_awscli.sh │ ├── install_consul.sh │ ├── prepare_staging.sh │ └── update.sh │ └── template.json ├── infra ├── README.md ├── v1 │ ├── Dockerfile │ ├── ansible │ │ ├── test_consul_servers_active.yml │ │ ├── wait_instance_stopped.yml │ │ └── wait_instance_up.yml │ ├── config │ ├── main.tf │ ├── run-docker.sh │ └── scripts │ │ ├── create.sh │ │ ├── destroy.sh │ │ ├── run.sh │ │ ├── terraform_to_ansible_inventory.sh │ │ ├── test_create.sh │ │ ├── test_upgrade.sh │ │ ├── upgrade.sh │ │ └── utils.sh └── v2 │ ├── Dockerfile │ ├── ansible │ ├── test_consul_servers_active.yml │ ├── wait_instance_stopped.yml │ └── wait_instance_up.yml │ ├── config │ ├── main.tf │ ├── run-docker.sh │ └── scripts │ ├── create.sh │ ├── destroy.sh │ ├── run.sh │ ├── terraform_to_ansible_inventory.sh │ ├── test_create.sh │ ├── test_upgrade.sh │ ├── upgrade.sh │ └── utils.sh └── setup └── iam_roles.tf /.gitignore: -------------------------------------------------------------------------------- 1 | [._]*.s[a-w][a-z] 2 | [._]s[a-w][a-z] 3 | *.un~ 4 | Session.vim 5 | .netrwhist 6 | *~ 7 | .terraform 8 | terraform.tfstate* 9 | build.log 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Example terraform immutable infrastructure and rolling upgrades of stateful services 2 | 3 | For a detailed explanation see [this](https://sgotti.me/post/terraform-immutable-infrastructure-stateful-rolling-upgrades/) or [this](https://www.sorint.it/osblog/-/blogs/immutable-infrastructure-with-terraform-and-rolling-upgrades-of-stateful-services) posts. 4 | 5 | This is an immutable infrastructure example. It demonstrates how to handle an immutable infrastructure creation, rolling upgrade, testing on an IaaS (in this case AWS in the us-east-1 region) on a service using persistent data. It uses [packer](https://www.packer.io/) and [terraform](https://www.terraform.io/) to respectively create the AMIs and to create/upgrade the infrastructure (and test all of this before doing it on the real environment). 6 | 7 | In this case we are creating/upgrading an infrastructure made by a consul cluster on 3 instances. Each instance has an EBS volume used for consul data. This volume is persisted across instance recreation (to avoid the need to remove a member from the cluster leaving it in a partial 2 node cluster and to avoid data resynchronization that will take some time 8 | 9 | Then we are going to upgrade the instances image with a new consul version and do a rolling upgrade of an instance at a time testing every step to ensure that the consul cluster remains healthy. 10 | 11 | We used consul just as a common example but this can be applied to any stateful application that achieves high availability using replication (like an etcd/mongodb/cassandra/postgresql etc...) and supports rolling upgrades (in case where rolling upgrade isn't supported, like a major version upgrade of postgresql the additional upgrade steps are needed but the base remains the same). 12 | 13 | In a real development workflow there'll be two repos: one for the image and one for the infrastructure and all of the below steps will be done with a continuous integration/deployment process. 14 | 15 | To make this as simple as possible using an unique repository we split the various repositories and their versions inside different directories (**images/consul/vX** and **infra/vX**). 16 | 17 | These are the steps that we are going to do below: 18 | 19 | ### Creation 20 | * Build an AMI with packer for the consul instances (images/consul/v1) 21 | * Build the infra/v1 project. Its build artifact is a docker image called infra:v1. 22 | * With this docker image we can test the infrastructure and then deploy (create) the production infrastructure 23 | 24 | ### Rolling upgrade 25 | * Build an updated AMI (images/consul/v2) 26 | * Build the infra/v2 project. Its build artifact is a docker image called infra:v2. 27 | * Use this build artifact to test the rolling upgrade and then do this on the production environment. 28 | 29 | 30 | ## Prerequisites 31 | 32 | You'll need: 33 | 34 | * Docker installed and running on your machine (with docker socket available under `/var/run/docker.sock`). 35 | * [Packer](https://www.packer.io/) 36 | * An aws access and secret key. Everything will be created inside a new vpc but we suggest to not use a "production" account. 37 | * Since the created aws instances have to call the aws APIs, an instance profile is needed. The terraform `main.tf` file requires this instance profile to be named `default_instance_profile` 38 | Inside the `setup` directory there's a terraform definition to create it (just execute a `terraform apply` inside this directory). 39 | * An ssh key pair configured in your AWS account. Additionally the ssh private key needs to be added to your ssh agent before executing the next steps. These are needed since the various tests use ssh to connect to the instances. The SSH_AUTH_SOCK path is bind mounted to the executed docker containers. 40 | * An S3 bucket to save terraform state files. The created files will start with `terraform/` 41 | 42 | 43 | ### Export the required environment variables 44 | 45 | ``` bash 46 | export AWS_ACCESS_KEY_ID= # The aws access key 47 | export AWS_SECRET_ACCESS_KEY= # The aws secret key 48 | export AWS_DEFAULT_REGION=us-east-1 49 | export S3_BUCKET= # The s3 bucket for saving terraform state files 50 | export SSH_KEYPAIR= # The ssh keypair name created in your aws account 51 | ``` 52 | 53 | ## Immutable infrastructure creation 54 | ### Build consul v1 ami 55 | 56 | ``` bash 57 | cd images/consul/v1 58 | VERSION=v1 ./build.sh 59 | ``` 60 | 61 | ### Build infra v1 docker image 62 | 63 | This image will be used to manage the infrastructure. It contains all the needed tools and scripts. 64 | 65 | ``` bash 66 | cd ../../../infra/v1 67 | ``` 68 | 69 | * Edit the `config` file and put in the `CONSUL_AMI_ID` variable the ami id generated from the previous step. 70 | 71 | ``` bash 72 | docker build -t infra:v1 . 73 | ``` 74 | 75 | ### Test the v1 infrastructure creation 76 | 77 | We can test the infrastructure creation in a temporary environment. 78 | 79 | Since the docker command exports different environment variables and two volumes (`/var/run/docker.sock` for letting the container execute another docker container and the ssh authentications socket), there an helper script called `run-docker.sh` to simplify this: 80 | 81 | ``` bash 82 | VERSION=v1 ./run-docker.sh test-create 83 | ``` 84 | 85 | ### Create the real v1 infrastructure 86 | If all goes ok we can create the "real" environment: 87 | 88 | ``` bash 89 | ENV=prod VERSION=v1 ./run-docker.sh create 90 | ``` 91 | 92 | ## Immutable infrastructure rolling upgrade 93 | Now that we have our infrastructure at v1 ready we can do a rolling upgrade. 94 | 95 | ### Build consul v2 ami 96 | 97 | ``` bash 98 | cd ../../images/consul/v2 99 | VERSION=v2 ./build.sh 100 | ``` 101 | 102 | ### Build infra v2 docker image 103 | 104 | ``` bash 105 | cd ../../../infra/v2 106 | ``` 107 | 108 | * Edit the `config` file and put in the `CONSUL_AMI_ID` variable the ami id generated from the previous step. 109 | 110 | ``` bash 111 | docker build -t infra:v2 . 112 | ``` 113 | 114 | ### Test the v2 infrastructure creation 115 | 116 | ``` bash 117 | VERSION=v2 ./run-docker.sh test-create 118 | ``` 119 | 120 | ### Test the v1 -> v2 infrastructure upgrade 121 | 122 | ``` bash 123 | VERSION=v2 ./run-docker.sh test-upgrade 124 | ``` 125 | 126 | 127 | ### Upgrade the real infrastructure from v1 -> v2 128 | If all goes ok we can upgrade the "real" environment: 129 | 130 | ``` bash 131 | ENV=prod VERSION=v2 ./run-docker.sh upgrade 132 | ``` 133 | 134 | 135 | ## Cleanup 136 | 137 | If you want to remove the real infrastructure you can do it with: 138 | 139 | ``` bash 140 | ENV=prod VERSION=v2 ./run-docker.sh destroy 141 | ``` 142 | -------------------------------------------------------------------------------- /images/consul/README.md: -------------------------------------------------------------------------------- 1 | The directories `v1` and `v2` represents the history of the consul image repository. To make them as simple as possible their content are identical with exception of the `install_consul.sh` script that uses a different consul release version (0.6.2 in v1 and 0.6.3 in v2). 2 | -------------------------------------------------------------------------------- /images/consul/v1/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | if [ "${1}" == "test" ]; then 7 | TEST=1 8 | else 9 | if [ -z ${VERSION} ]; then 10 | echo "VERSION env variable is missing!" 11 | exit 1 12 | fi 13 | fi 14 | 15 | 16 | packer build -machine-readable template.json | tee build.log 17 | AMI_ID=$(grep 'artifact,0,id' build.log | cut -d, -f6 | cut -d: -f2) 18 | 19 | echo "AMI_ID: ${AMI_ID}" 20 | if [ -z "${AMI_ID}" ]; then 21 | exit 1 22 | fi 23 | 24 | if [ -n "${TEST}" ]; then 25 | # Get ami's snapshot id 26 | SNAPSHOT_ID=$(aws ec2 describe-images --image-id ${AMI_ID} --query 'Images[*].BlockDeviceMappings[0].Ebs.SnapshotId' --output text) 27 | 28 | aws ec2 deregister-image --image-id ${AMI_ID} 29 | aws ec2 delete-snapshot --snapshot-id ${SNAPSHOT_ID} 30 | else 31 | # Remove test tag added by packer template 32 | aws ec2 delete-tags --resources ${AMI_ID} --tags Key=test,Value= 33 | aws ec2 create-tags --resources ${AMI_ID} --tags Key=version,Value=${VERSION} 34 | fi 35 | -------------------------------------------------------------------------------- /images/consul/v1/files/bin/consul-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | CONSUL_CONFIG_FILE="/etc/consul.d/config.json" 6 | 7 | MY_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 8 | MY_PRIVATE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) 9 | 10 | REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | sed 's/ "region" : "\(.*\)",\?/\1/') 11 | 12 | VPC=$(aws ec2 describe-instances --region ${REGION} --instance ${MY_INSTANCE_ID} --query 'Reservations[0].Instances[0].VpcId' --output text) 13 | 14 | CONSUL_SERVERS=$(aws ec2 describe-instances --region us-east-1 --filters "Name=vpc-id,Values=${VPC}" "Name=tag:consul-type,Values=server" --query 'Reservations[].Instances[].PrivateIpAddress' --output text) 15 | CONSUL_SERVERS_JSON=$(jq -n -c -M --arg s "${CONSUL_SERVERS}" '($s|split("\\s+"; "" ))') 16 | 17 | CONSUL_TYPE=$(aws ec2 describe-instances --region ${REGION} --instance ${MY_INSTANCE_ID} --query 'Reservations[0].Instances[0].Tags[?Key==`consul-type`].Value' --output text) 18 | 19 | if [ ${CONSUL_TYPE} == "server" ]; then 20 | cat < ${CONSUL_CONFIG_FILE} 21 | { 22 | "bind_addr": "${MY_PRIVATE_IP}", 23 | "server": true, 24 | "bootstrap_expect": 3, 25 | "retry_join": ${CONSUL_SERVERS_JSON}, 26 | "skip_leave_on_interrupt": true 27 | } 28 | EOF 29 | else 30 | cat < ${CONSUL_CONFIG_FILE} 31 | { 32 | "bind_addr": "${MY_PRIVATE_IP}", 33 | "retry_join": ${CONSUL_SERVERS_JSON} 34 | } 35 | EOF 36 | fi 37 | 38 | -------------------------------------------------------------------------------- /images/consul/v1/files/systemd/consul.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=consul agent 3 | Requires=network-online.target 4 | Requires=mnt-data.mount 5 | Requires=generate-consul-config.service 6 | After=network-online.target 7 | After=mnt-data.mount 8 | After=generate-consul-config.service 9 | 10 | [Service] 11 | Environment=GOMAXPROCS=2 12 | Restart=on-failure 13 | ExecStartPre=/usr/bin/mkdir -p /mnt/data/consul 14 | ExecStart=/usr/local/bin/consul agent -data-dir=/mnt/data/consul -config-dir=/etc/consul.d -ui 15 | ExecReload=/bin/kill -HUP $MAINPID 16 | KillSignal=SIGINT 17 | 18 | [Install] 19 | WantedBy=multi-user.target 20 | 21 | -------------------------------------------------------------------------------- /images/consul/v1/files/systemd/format-data-volume.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Format EBS volume if needed 3 | 4 | [Service] 5 | Type=oneshot 6 | RemainAfterExit=yes 7 | ExecStart=/bin/bash -c \ 8 | '(/usr/sbin/blkid /dev/xvdb) || \ 9 | (/usr/sbin/wipefs -fa /dev/xvdb && /usr/sbin/mkfs.ext4 /dev/xvdb)' 10 | 11 | -------------------------------------------------------------------------------- /images/consul/v1/files/systemd/generate-consul-config.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Generate consul config 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | Type=oneshot 8 | RemainAfterExit=yes 9 | ExecStart=/usr/local/bin/consul-config.sh 10 | 11 | -------------------------------------------------------------------------------- /images/consul/v1/files/systemd/mnt-data.mount: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Mount EBS volume to /mnt/data 3 | BindsTo=dev-xvdb.device 4 | Requires=format-data-volume.service 5 | After=format-data-volume.service 6 | 7 | [Mount] 8 | What=/dev/xvdb 9 | Where=/mnt/data 10 | DirectoryMode=0777 11 | Type=ext4 12 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/base.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | dnf install -y curl wget unzip dnsmasq jq bind-utils 7 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/configure_dnsmasq.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | cat << 'EOF' > /etc/dhcp/dhclient-enter-hooks 7 | make_resolv_conf() { 8 | RESOLV_CONF_DHCP=/etc/resolv.conf.dhcp 9 | > ${RESOLV_CONF_DHCP} 10 | 11 | if [ -n "${new_domain_name_servers}" ]; then 12 | for nameserver in ${new_domain_name_servers} ; do 13 | echo "nameserver ${nameserver}" >> "/etc/resolv.conf.dhcp" 14 | done 15 | fi 16 | } 17 | EOF 18 | 19 | echo "server=/consul/127.0.0.1#8600" > /etc/dnsmasq.d/10-consul 20 | # TODO retrieve/calculate this? 21 | echo "resolv-file=/etc/resolv.conf.dhcp" > /etc/dnsmasq.d/0-default 22 | 23 | systemctl enable dnsmasq 24 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/configure_network.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # Add PEERDNS=no to ifcfg-eth0 since we don't want the dhcp client to rewrite /etc/resolv.conf 7 | echo "PEERDNS=no" >> /etc/sysconfig/network-scripts/ifcfg-eth0 8 | 9 | cat < /etc/resolv.conf 10 | search ec2.internal 11 | nameserver 127.0.0.1 12 | EOF 13 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/copy_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | STAGING_DIR="/tmp/packer/files/" 7 | mkdir -p ${STAGING_DIR} 8 | chown -R fedora:fedora ${STAGING_DIR} 9 | 10 | chown root:root ${STAGING_DIR}/systemd/* 11 | chown root:root ${STAGING_DIR}/bin/* 12 | chmod +x ${STAGING_DIR}/bin/* 13 | 14 | cp ${STAGING_DIR}/systemd/* /etc/systemd/system/ 15 | cp ${STAGING_DIR}/bin/* /usr/local/bin 16 | 17 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/disable_selinux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | sed -i --follow-symlinks 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config 7 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/enable_services.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | systemctl enable consul.service 7 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/install_awscli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | dnf install -y python-pip 7 | pip install awscli 8 | 9 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/install_consul.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | echo "Fetching Consul..." 7 | cd /tmp 8 | wget https://releases.hashicorp.com/consul/0.6.2/consul_0.6.2_linux_amd64.zip -O consul.zip 9 | 10 | echo "Installing Consul..." 11 | unzip consul.zip >/dev/null 12 | chown root:root consul 13 | chmod +x consul 14 | mv consul /usr/local/bin/consul 15 | mkdir -p /etc/consul.d 16 | mkdir -p /mnt/consul 17 | mkdir -p /etc/service 18 | 19 | rm -f consul.zip 20 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/prepare_staging.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | mkdir -p /tmp/packer/files 7 | chown -R fedora:fedora /tmp/packer/ 8 | -------------------------------------------------------------------------------- /images/consul/v1/scripts/update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | dnf -y update 7 | -------------------------------------------------------------------------------- /images/consul/v1/template.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "access_key": "{{env `AWS_ACCESS_KEY_ID`}}", 4 | "secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", 5 | "region": "{{env `AWS_DEFAULT_REGION`}}" 6 | }, 7 | "builders": [{ 8 | "type": "amazon-ebs", 9 | "access_key": "{{user `access_key`}}", 10 | "secret_key": "{{user `secret_key`}}", 11 | "region": "{{user `region`}}", 12 | "source_ami": "ami-518bfb3b", 13 | "instance_type": "t2.micro", 14 | "ssh_username": "fedora", 15 | "ami_name": "consul ami {{timestamp}}", 16 | "tags": { 17 | "test": "" 18 | } 19 | }], 20 | "provisioners": [ 21 | { 22 | "type": "shell", 23 | "execute_command": "{{ .Vars }} sudo -E sh '{{ .Path }}'", 24 | "scripts": [ 25 | "./scripts/base.sh", 26 | "./scripts/disable_selinux.sh", 27 | "./scripts/update.sh", 28 | "./scripts/install_awscli.sh", 29 | "./scripts/install_consul.sh", 30 | "./scripts/configure_network.sh", 31 | "./scripts/configure_dnsmasq.sh" 32 | ] 33 | }, 34 | { 35 | "type": "shell", 36 | "execute_command": "{{ .Vars }} sudo -E sh '{{ .Path }}'", 37 | "scripts": [ 38 | "./scripts/prepare_staging.sh" 39 | ] 40 | }, 41 | { 42 | "type": "file", 43 | "source": "files/", 44 | "destination": "/tmp/packer/files/" 45 | }, 46 | { 47 | "type": "shell", 48 | "execute_command": "{{ .Vars }} sudo -E sh '{{ .Path }}'", 49 | "scripts": [ 50 | "./scripts/copy_files.sh", 51 | "./scripts/enable_services.sh" 52 | ] 53 | } 54 | 55 | ] 56 | } 57 | -------------------------------------------------------------------------------- /images/consul/v2/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | if [ "${1}" == "test" ]; then 7 | TEST=1 8 | else 9 | if [ -z ${VERSION} ]; then 10 | echo "VERSION env variable is missing!" 11 | exit 1 12 | fi 13 | fi 14 | 15 | 16 | packer build -machine-readable template.json | tee build.log 17 | AMI_ID=$(grep 'artifact,0,id' build.log | cut -d, -f6 | cut -d: -f2) 18 | 19 | echo "AMI_ID: ${AMI_ID}" 20 | if [ -z "${AMI_ID}" ]; then 21 | exit 1 22 | fi 23 | 24 | if [ -n "${TEST}" ]; then 25 | # Get ami's snapshot id 26 | SNAPSHOT_ID=$(aws ec2 describe-images --image-id ${AMI_ID} --query 'Images[*].BlockDeviceMappings[0].Ebs.SnapshotId' --output text) 27 | 28 | aws ec2 deregister-image --image-id ${AMI_ID} 29 | aws ec2 delete-snapshot --snapshot-id ${SNAPSHOT_ID} 30 | else 31 | # Remove test tag added by packer template 32 | aws ec2 delete-tags --resources ${AMI_ID} --tags Key=test,Value= 33 | aws ec2 create-tags --resources ${AMI_ID} --tags Key=version,Value=${VERSION} 34 | fi 35 | -------------------------------------------------------------------------------- /images/consul/v2/files/bin/consul-config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | CONSUL_CONFIG_FILE="/etc/consul.d/config.json" 6 | 7 | MY_INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) 8 | MY_PRIVATE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) 9 | 10 | REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep region | sed 's/ "region" : "\(.*\)",\?/\1/') 11 | 12 | VPC=$(aws ec2 describe-instances --region ${REGION} --instance ${MY_INSTANCE_ID} --query 'Reservations[0].Instances[0].VpcId' --output text) 13 | 14 | CONSUL_SERVERS=$(aws ec2 describe-instances --region us-east-1 --filters "Name=vpc-id,Values=${VPC}" "Name=tag:consul-type,Values=server" --query 'Reservations[].Instances[].PrivateIpAddress' --output text) 15 | CONSUL_SERVERS_JSON=$(jq -n -c -M --arg s "${CONSUL_SERVERS}" '($s|split("\\s+"; "" ))') 16 | 17 | CONSUL_TYPE=$(aws ec2 describe-instances --region ${REGION} --instance ${MY_INSTANCE_ID} --query 'Reservations[0].Instances[0].Tags[?Key==`consul-type`].Value' --output text) 18 | 19 | if [ ${CONSUL_TYPE} == "server" ]; then 20 | cat < ${CONSUL_CONFIG_FILE} 21 | { 22 | "bind_addr": "${MY_PRIVATE_IP}", 23 | "server": true, 24 | "bootstrap_expect": 3, 25 | "retry_join": ${CONSUL_SERVERS_JSON}, 26 | "skip_leave_on_interrupt": true 27 | } 28 | EOF 29 | else 30 | cat < ${CONSUL_CONFIG_FILE} 31 | { 32 | "bind_addr": "${MY_PRIVATE_IP}", 33 | "retry_join": ${CONSUL_SERVERS_JSON} 34 | } 35 | EOF 36 | fi 37 | 38 | -------------------------------------------------------------------------------- /images/consul/v2/files/systemd/consul.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=consul agent 3 | Requires=network-online.target 4 | Requires=mnt-data.mount 5 | Requires=generate-consul-config.service 6 | After=network-online.target 7 | After=mnt-data.mount 8 | After=generate-consul-config.service 9 | 10 | [Service] 11 | Environment=GOMAXPROCS=2 12 | Restart=on-failure 13 | ExecStartPre=/usr/bin/mkdir -p /mnt/data/consul 14 | ExecStart=/usr/local/bin/consul agent -data-dir=/mnt/data/consul -config-dir=/etc/consul.d -ui 15 | ExecReload=/bin/kill -HUP $MAINPID 16 | KillSignal=SIGINT 17 | 18 | [Install] 19 | WantedBy=multi-user.target 20 | 21 | -------------------------------------------------------------------------------- /images/consul/v2/files/systemd/format-data-volume.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Format EBS volume if needed 3 | 4 | [Service] 5 | Type=oneshot 6 | RemainAfterExit=yes 7 | ExecStart=/bin/bash -c \ 8 | '(/usr/sbin/blkid /dev/xvdb) || \ 9 | (/usr/sbin/wipefs -fa /dev/xvdb && /usr/sbin/mkfs.ext4 /dev/xvdb)' 10 | 11 | -------------------------------------------------------------------------------- /images/consul/v2/files/systemd/generate-consul-config.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Generate consul config 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | Type=oneshot 8 | RemainAfterExit=yes 9 | ExecStart=/usr/local/bin/consul-config.sh 10 | 11 | -------------------------------------------------------------------------------- /images/consul/v2/files/systemd/mnt-data.mount: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Mount EBS volume to /mnt/data 3 | BindsTo=dev-xvdb.device 4 | Requires=format-data-volume.service 5 | After=format-data-volume.service 6 | 7 | [Mount] 8 | What=/dev/xvdb 9 | Where=/mnt/data 10 | DirectoryMode=0777 11 | Type=ext4 12 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/base.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | dnf install -y curl wget unzip dnsmasq jq bind-utils 7 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/configure_dnsmasq.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | cat << 'EOF' > /etc/dhcp/dhclient-enter-hooks 7 | make_resolv_conf() { 8 | RESOLV_CONF_DHCP=/etc/resolv.conf.dhcp 9 | > ${RESOLV_CONF_DHCP} 10 | 11 | if [ -n "${new_domain_name_servers}" ]; then 12 | for nameserver in ${new_domain_name_servers} ; do 13 | echo "nameserver ${nameserver}" >> "/etc/resolv.conf.dhcp" 14 | done 15 | fi 16 | } 17 | EOF 18 | 19 | echo "server=/consul/127.0.0.1#8600" > /etc/dnsmasq.d/10-consul 20 | # TODO retrieve/calculate this? 21 | echo "resolv-file=/etc/resolv.conf.dhcp" > /etc/dnsmasq.d/0-default 22 | 23 | systemctl enable dnsmasq 24 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/configure_network.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # Add PEERDNS=no to ifcfg-eth0 since we don't want the dhcp client to rewrite /etc/resolv.conf 7 | echo "PEERDNS=no" >> /etc/sysconfig/network-scripts/ifcfg-eth0 8 | 9 | cat < /etc/resolv.conf 10 | search ec2.internal 11 | nameserver 127.0.0.1 12 | EOF 13 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/copy_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | STAGING_DIR="/tmp/packer/files/" 7 | mkdir -p ${STAGING_DIR} 8 | chown -R fedora:fedora ${STAGING_DIR} 9 | 10 | chown root:root ${STAGING_DIR}/systemd/* 11 | chown root:root ${STAGING_DIR}/bin/* 12 | chmod +x ${STAGING_DIR}/bin/* 13 | 14 | cp ${STAGING_DIR}/systemd/* /etc/systemd/system/ 15 | cp ${STAGING_DIR}/bin/* /usr/local/bin 16 | 17 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/disable_selinux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | sed -i --follow-symlinks 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config 7 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/enable_services.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | systemctl enable consul.service 7 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/install_awscli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | dnf install -y python-pip 7 | pip install awscli 8 | 9 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/install_consul.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | echo "Fetching Consul..." 7 | cd /tmp 8 | wget https://releases.hashicorp.com/consul/0.6.3/consul_0.6.3_linux_amd64.zip -O consul.zip 9 | 10 | echo "Installing Consul..." 11 | unzip consul.zip >/dev/null 12 | chown root:root consul 13 | chmod +x consul 14 | mv consul /usr/local/bin/consul 15 | mkdir -p /etc/consul.d 16 | mkdir -p /mnt/consul 17 | mkdir -p /etc/service 18 | 19 | rm -f consul.zip 20 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/prepare_staging.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | mkdir -p /tmp/packer/files 7 | chown -R fedora:fedora /tmp/packer/ 8 | -------------------------------------------------------------------------------- /images/consul/v2/scripts/update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | dnf -y update 7 | -------------------------------------------------------------------------------- /images/consul/v2/template.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "access_key": "{{env `AWS_ACCESS_KEY_ID`}}", 4 | "secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", 5 | "region": "{{env `AWS_DEFAULT_REGION`}}" 6 | }, 7 | "builders": [{ 8 | "type": "amazon-ebs", 9 | "access_key": "{{user `access_key`}}", 10 | "secret_key": "{{user `secret_key`}}", 11 | "region": "{{user `region`}}", 12 | "source_ami": "ami-518bfb3b", 13 | "instance_type": "t2.micro", 14 | "ssh_username": "fedora", 15 | "ami_name": "consul ami {{timestamp}}", 16 | "tags": { 17 | "test": "" 18 | } 19 | }], 20 | "provisioners": [ 21 | { 22 | "type": "shell", 23 | "execute_command": "{{ .Vars }} sudo -E sh '{{ .Path }}'", 24 | "scripts": [ 25 | "./scripts/base.sh", 26 | "./scripts/disable_selinux.sh", 27 | "./scripts/update.sh", 28 | "./scripts/install_awscli.sh", 29 | "./scripts/install_consul.sh", 30 | "./scripts/configure_network.sh", 31 | "./scripts/configure_dnsmasq.sh" 32 | ] 33 | }, 34 | { 35 | "type": "shell", 36 | "execute_command": "{{ .Vars }} sudo -E sh '{{ .Path }}'", 37 | "scripts": [ 38 | "./scripts/prepare_staging.sh" 39 | ] 40 | }, 41 | { 42 | "type": "file", 43 | "source": "files/", 44 | "destination": "/tmp/packer/files/" 45 | }, 46 | { 47 | "type": "shell", 48 | "execute_command": "{{ .Vars }} sudo -E sh '{{ .Path }}'", 49 | "scripts": [ 50 | "./scripts/copy_files.sh", 51 | "./scripts/enable_services.sh" 52 | ] 53 | } 54 | 55 | ] 56 | } 57 | -------------------------------------------------------------------------------- /infra/README.md: -------------------------------------------------------------------------------- 1 | The directories `v1` and `v2` represents the history of the infra repository. To make them as simple as possible their content (terraform config, scripts etc...) is identical with exception of the `config` file that defines which consul AMI to use and, in `v1`, which previous infra version to use for testing rolling upgrades. 2 | -------------------------------------------------------------------------------- /infra/v1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM fedora:23 2 | 3 | RUN dnf -y install wget curl git unzip jq python python-pip python-boto3 awscli ansible docker 4 | 5 | RUN curl https://releases.hashicorp.com/terraform/0.6.12/terraform_0.6.12_linux_amd64.zip -o /tmp/terraform_0.6.12_linux_amd64.zip 6 | 7 | WORKDIR /usr/local/bin/ 8 | RUN unzip /tmp/terraform_0.6.12_linux_amd64.zip 9 | 10 | RUN mkdir -p /root/.ssh 11 | RUN chmod 700 /root/.ssh 12 | RUN echo -e "Host *\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config 13 | 14 | WORKDIR /deploy 15 | ADD main.tf config ./ 16 | ADD scripts scripts/ 17 | ADD ansible ansible/ 18 | 19 | ENTRYPOINT [ "/deploy/scripts/run.sh" ] 20 | -------------------------------------------------------------------------------- /infra/v1/ansible/test_consul_servers_active.yml: -------------------------------------------------------------------------------- 1 | 2 | - hosts: all 3 | remote_user: fedora 4 | become_user: root 5 | gather_facts: false 6 | tasks: 7 | - name: Check for 3 active consul servers 8 | shell: /usr/local/bin/consul members | grep alive | grep server | wc -l 9 | register: result 10 | until: result.stdout.find("3") != -1 11 | retries: 5 12 | delay: 10 13 | 14 | - name: Fail if we cannot find 3 active consul server 15 | fail: 16 | msg: "Only {{ result.stdout }} consul servers active" 17 | when: result.stdout.find("3") == -1 18 | 19 | -------------------------------------------------------------------------------- /infra/v1/ansible/wait_instance_stopped.yml: -------------------------------------------------------------------------------- 1 | 2 | - hosts: all 3 | remote_user: fedora 4 | become_user: root 5 | gather_facts: false 6 | tasks: 7 | - name: waiting for server to be stopped 8 | local_action: wait_for host={{ inventory_hostname }} port=22 state=stopped timeout=300 9 | become: false 10 | -------------------------------------------------------------------------------- /infra/v1/ansible/wait_instance_up.yml: -------------------------------------------------------------------------------- 1 | 2 | - hosts: all 3 | remote_user: fedora 4 | become_user: root 5 | gather_facts: false 6 | tasks: 7 | - name: waiting for server to become reachable via ssh 8 | local_action: wait_for host={{ inventory_hostname }} port=22 state=started timeout=300 9 | become: false 10 | -------------------------------------------------------------------------------- /infra/v1/config: -------------------------------------------------------------------------------- 1 | CONSUL_AMI_ID= # Put here the ami id reported at the end of the build of `images/consul/v1` 2 | -------------------------------------------------------------------------------- /infra/v1/main.tf: -------------------------------------------------------------------------------- 1 | variable "access_key" {} 2 | variable "secret_key" {} 3 | 4 | variable "region" { 5 | default = "us-east-1" 6 | } 7 | 8 | variable "env" {} 9 | variable "consul_ami" {} 10 | variable "ssh_keypair" {} 11 | 12 | variable "azs" { 13 | default = "us-east-1a,us-east-1b,us-east-1c,us-east-1e" 14 | } 15 | 16 | provider "aws" { 17 | access_key = "${var.access_key}" 18 | secret_key = "${var.secret_key}" 19 | region = "${var.region}" 20 | } 21 | 22 | # The next resources define a new vpc with a public subnet for every 23 | # availabilty zone and a default security group completely open. This isn't 24 | # meant to be used in a production environment, it's just to make the vpc 25 | # definition the most concise as possible. 26 | 27 | resource "aws_vpc" "vpc" { 28 | cidr_block = "10.0.0.0/16" 29 | enable_dns_support = true 30 | enable_dns_hostnames = true 31 | tags { 32 | Name = "vpc ${var.env} ${var.region}" 33 | } 34 | } 35 | 36 | resource "aws_internet_gateway" "igw" { 37 | vpc_id = "${aws_vpc.vpc.id}" 38 | 39 | tags { 40 | Name = "igw ${var.env} ${var.region}" 41 | } 42 | } 43 | 44 | # Create a subnet for every availability zone 45 | resource "aws_subnet" "front" { 46 | count = "${length(split(\",\", var.azs))}" 47 | vpc_id = "${aws_vpc.vpc.id}" 48 | cidr_block = "10.0.${count.index * 16}.0/20" 49 | map_public_ip_on_launch = true 50 | availability_zone = "${element(split(\",\", var.azs), count.index)}" 51 | 52 | tags { 53 | Name = "subnet ${count.index} ${element(split(\",\", var.azs), count.index)}" 54 | } 55 | } 56 | 57 | resource "aws_route_table" "public" { 58 | vpc_id = "${aws_vpc.vpc.id}" 59 | route { 60 | cidr_block = "0.0.0.0/0" 61 | gateway_id = "${aws_internet_gateway.igw.id}" 62 | } 63 | 64 | tags { 65 | Name = "${var.region} ${var.env} public" 66 | } 67 | } 68 | 69 | resource "aws_route_table_association" "front" { 70 | count = "${length(split(\",\", var.azs))}" 71 | subnet_id = "${element(aws_subnet.front.*.id, count.index)}" 72 | route_table_id = "${aws_route_table.public.id}" 73 | } 74 | 75 | # An (in)security_group :D 76 | resource "aws_security_group" "allow_all" { 77 | name = "allow_all" 78 | description = "Allow all inbound traffic" 79 | 80 | vpc_id = "${aws_vpc.vpc.id}" 81 | 82 | ingress { 83 | from_port = 0 84 | to_port = 0 85 | protocol = "-1" 86 | cidr_blocks = ["0.0.0.0/0"] 87 | } 88 | 89 | egress { 90 | from_port = 0 91 | to_port = 0 92 | protocol = "-1" 93 | cidr_blocks = ["0.0.0.0/0"] 94 | } 95 | } 96 | 97 | # Here we define out instances, volumes etc. 98 | # Some notes: 99 | # * Do not use an unique resource with a count, since we want to distribute 100 | # them on different availability zones 101 | # * We have to avoid an hard dependency between the ebs and the instances since 102 | # it will make terraform recreate the ebs when an instance is recreated. 103 | 104 | resource "aws_volume_attachment" "consul_server01_ebs_attachment" { 105 | device_name = "/dev/xvdb" 106 | volume_id = "${aws_ebs_volume.consul_server01_ebs.id}" 107 | instance_id = "${aws_instance.consul_server01.id}" 108 | } 109 | 110 | resource "aws_ebs_volume" "consul_server01_ebs" { 111 | availability_zone = "us-east-1a" 112 | size = 1 113 | } 114 | 115 | resource "aws_volume_attachment" "consul_server02_ebs_attachment" { 116 | device_name = "/dev/xvdb" 117 | volume_id = "${aws_ebs_volume.consul_server02_ebs.id}" 118 | instance_id = "${aws_instance.consul_server02.id}" 119 | } 120 | 121 | resource "aws_ebs_volume" "consul_server02_ebs" { 122 | availability_zone = "us-east-1b" 123 | size = 1 124 | } 125 | 126 | resource "aws_volume_attachment" "consul_server03_ebs_attachment" { 127 | device_name = "/dev/xvdb" 128 | volume_id = "${aws_ebs_volume.consul_server03_ebs.id}" 129 | instance_id = "${aws_instance.consul_server03.id}" 130 | } 131 | 132 | resource "aws_ebs_volume" "consul_server03_ebs" { 133 | availability_zone = "us-east-1c" 134 | size = 1 135 | } 136 | 137 | 138 | # Use a fixed ip address for various reasons: 139 | # * Avoid consul raft logic problems with changing ips when instance is 140 | # recreated (https://github.com/hashicorp/consul/issues/457). Another solution 141 | # will be to let consul server leave the raft cluster but this will make a 142 | # temporary 2 node cluster and, if another node fails, it will lose the quorum 143 | # requiring a manual operation to restore it in a working state. 144 | resource "aws_instance" "consul_server01" { 145 | ami = "${var.consul_ami}" 146 | instance_type = "t2.micro" 147 | iam_instance_profile = "default_instance_profile" 148 | key_name = "${var.ssh_keypair}" 149 | subnet_id = "${aws_subnet.front.0.id}" 150 | vpc_security_group_ids = ["${aws_security_group.allow_all.id}"] 151 | private_ip = "${cidrhost(aws_subnet.front.0.cidr_block, 10)}" 152 | tags = { 153 | "consul-type" = "server" 154 | } 155 | } 156 | 157 | resource "aws_instance" "consul_server02" { 158 | ami = "${var.consul_ami}" 159 | instance_type = "t2.micro" 160 | iam_instance_profile = "default_instance_profile" 161 | key_name = "${var.ssh_keypair}" 162 | subnet_id = "${aws_subnet.front.1.id}" 163 | vpc_security_group_ids = ["${aws_security_group.allow_all.id}"] 164 | private_ip = "${cidrhost(aws_subnet.front.1.cidr_block, 10)}" 165 | tags = { 166 | "consul-type" = "server" 167 | } 168 | } 169 | 170 | resource "aws_instance" "consul_server03" { 171 | ami = "${var.consul_ami}" 172 | instance_type = "t2.micro" 173 | iam_instance_profile = "default_instance_profile" 174 | key_name = "${var.ssh_keypair}" 175 | subnet_id = "${aws_subnet.front.2.id}" 176 | vpc_security_group_ids = ["${aws_security_group.allow_all.id}"] 177 | private_ip = "${cidrhost(aws_subnet.front.2.cidr_block, 10)}" 178 | tags = { 179 | "consul-type" = "server" 180 | } 181 | } 182 | 183 | ### Outputs ### 184 | 185 | output region { 186 | value = "${var.region}" 187 | } 188 | 189 | output vpc_id { 190 | value = "${aws_vpc.vpc.id}" 191 | } 192 | 193 | output security_group_allow_all_id { 194 | value = "${aws_security_group.allow_all.id}" 195 | } 196 | 197 | output subnets { 198 | value = "${join(",", aws_subnet.front.*.id)}" 199 | } 200 | -------------------------------------------------------------------------------- /infra/v1/run-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker run -it -v /var/run/docker.sock:/var/run/docker.sock -v "$SSH_AUTH_SOCK:/tmp/ssh_auth_sock" -e "SSH_AUTH_SOCK=/tmp/ssh_auth_sock" -e "ORIG_SSH_AUTH_SOCK=$SSH_AUTH_SOCK" -e ENV=${ENV} -e S3_BUCKET=${S3_BUCKET} -e SSH_KEYPAIR=${SSH_KEYPAIR} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION infra:${VERSION} ${1} 4 | -------------------------------------------------------------------------------- /infra/v1/scripts/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | [ -z ${ENV} ] && error "missing ENV environment variable" 15 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 16 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 17 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 18 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 19 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 20 | 21 | export TF_VAR_access_key=${AWS_ACCESS_KEY_ID} 22 | export TF_VAR_secret_key=${AWS_SECRET_ACCESS_KEY} 23 | export TF_VAR_region=${AWS_DEFAULT_REGION} 24 | export TF_VAR_ssh_keypair=${SSH_KEYPAIR} 25 | 26 | TFSTATE_KEY="terraform/$ENV/base" 27 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 28 | 29 | [ -z ${CONSUL_AMI_ID} ] && error "undefined CONSUL_AMI_ID" 30 | export TF_VAR_consul_ami=${CONSUL_AMI_ID} 31 | 32 | 33 | pushd ${__root} 34 | 35 | # Remove local cached terraform.tfstate file. This is to avoid having a cached state file referencing another environment due to manual tests or wrong operations. 36 | rm -f ${TFSTATE_FILE} 37 | terraform remote config -backend=s3 --backend-config="bucket=${S3_BUCKET}" --backend-config="key=$TFSTATE_KEY" --backend-config="region=${AWS_DEFAULT_REGION}" 38 | 39 | terraform plan -input=false -var "env=$ENV" || error "terraform plan failed" 40 | 41 | terraform apply -input=false -var "env=$ENV" || error "terraform apply failed" 42 | 43 | ALL_INSTANCE_IDS=$(tf_get_all_instance_ids ${TFSTATE_FILE}) 44 | aws ec2 wait instance-running --instance-ids ${ALL_INSTANCE_IDS} || error "some instances not active" 45 | 46 | # Wait all instances are reachable via ssh 47 | ansible-playbook -i ${__root}/scripts/terraform_to_ansible_inventory.sh ${__ansible}/wait_instance_up.yml 48 | 49 | # Wait for all the consul server being active. Check this using the first consul server. 50 | consul01ip=$(tf_get_instance_public_ip ${TFSTATE_FILE} "consul_server01") 51 | ansible-playbook -i ${consul01ip}, ${__ansible}/test_consul_servers_active.yml 52 | -------------------------------------------------------------------------------- /infra/v1/scripts/destroy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | [ -z ${ENV} ] && error "missing ENV environment variable" 15 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 16 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 17 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 18 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 19 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 20 | 21 | export TF_VAR_access_key=${AWS_ACCESS_KEY_ID} 22 | export TF_VAR_secret_key=${AWS_SECRET_ACCESS_KEY} 23 | export TF_VAR_region=${AWS_DEFAULT_REGION} 24 | export TF_VAR_ssh_keypair=${SSH_KEYPAIR} 25 | 26 | TFSTATE_KEY="terraform/$ENV/base" 27 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 28 | 29 | export TF_VAR_consul_ami="IDONTCARE" 30 | 31 | # Remove local cached terraform.tfstate file. This is to avoid having a cached state file referencing another environment due to manual tests or wrong operations. 32 | rm -f ${TFSTATE_FILE} 33 | terraform remote config -backend=s3 --backend-config="bucket=${S3_BUCKET}" --backend-config="key=$TFSTATE_KEY" --backend-config="region=${AWS_DEFAULT_REGION}" 34 | 35 | # shutdown instance before doing terraform apply or it will fail to remove the aws_volume_attachment since it's mounted. See also https://github.com/hashicorp/terraform/issues/2957 36 | INSTANCE_IDS=$(tf_get_all_instance_ids ${TFSTATE_FILE}) 37 | if [ ${INSTANCE_IDS} != "[]" ]; then 38 | aws ec2 stop-instances --instance-ids ${INSTANCE_IDS} 39 | aws ec2 wait instance-stopped --instance-ids ${INSTANCE_IDS} || error "some instance are not stopped" 40 | fi 41 | 42 | terraform destroy -input=false -force -var "env=$ENV" || error "terraform plan failed" 43 | -------------------------------------------------------------------------------- /infra/v1/scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__dir}/utils.sh 11 | 12 | case ${1} in 13 | "create") 14 | ${__root}/scripts/create.sh 15 | ;; 16 | "upgrade") 17 | ${__root}/scripts/upgrade.sh 18 | ;; 19 | "destroy") 20 | ${__root}/scripts/destroy.sh 21 | ;; 22 | "test-create") 23 | ${__root}/scripts/test_create.sh 24 | ;; 25 | "test-upgrade") 26 | ${__root}/scripts/test_upgrade.sh 27 | ;; 28 | *) 29 | error "Usage: ${0} {create|upgrade|destroy|test-create|test-upgrade}" 30 | esac 31 | 32 | -------------------------------------------------------------------------------- /infra/v1/scripts/terraform_to_ansible_inventory.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # A simple script that generates an ansible inventory from a terraform state file 4 | set -x 5 | set -e 6 | 7 | __dir="$(readlink -f $(dirname ${0}))" 8 | __root="$(readlink -f ${__dir}/../)" 9 | 10 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 11 | 12 | cat ${TFSTATE_FILE} | jq -c -e -r -M '{ all: .modules[0].resources | to_entries | map(select(.key | test("aws_instance\\..*"))) | map(.value.primary.attributes.public_ip) }' 13 | -------------------------------------------------------------------------------- /infra/v1/scripts/test_create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__dir}/utils.sh 11 | 12 | function cleanup() { 13 | ${__dir}/destroy.sh 14 | delete_s3_object ${S3_BUCKET} ${TFSTATE_KEY} 15 | } 16 | 17 | env 18 | 19 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 20 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 21 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 22 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 23 | 24 | # Generate a randon environment name 25 | export ENV="test-$(uuidgen | cut -c1-8)" 26 | 27 | TFSTATE_KEY="terraform/$ENV/base" 28 | 29 | trap cleanup EXIT 30 | 31 | pushd ${__root} 32 | 33 | # Remove local cached terraform.tfstate file. Shouldn't exist since test 34 | # environment is new but useful for local test runs. 35 | rm -f .terraform/terraform.tfstate 36 | 37 | ${__dir}/create.sh || error "create failed" 38 | -------------------------------------------------------------------------------- /infra/v1/scripts/test_upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | function cleanup() { 15 | ${__dir}/destroy.sh 16 | delete_s3_object ${S3_BUCKET} ${TFSTATE_KEY} 17 | } 18 | 19 | env 20 | 21 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 22 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 23 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 24 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 25 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 26 | 27 | [ -z ${PREV_VERSION} ] && error "PREV_VERSION undefined, cannot test upgrade" 28 | 29 | # Generate a randon environment name 30 | export ENV="test-$(uuidgen | cut -c1-8)" 31 | 32 | TFSTATE_KEY="terraform/$ENV/base" 33 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 34 | 35 | trap cleanup EXIT 36 | 37 | pushd ${__root} 38 | 39 | # Remove old remote tfstate 40 | delete_s3_object ${S3_BUCKET} ${TFSTATE_KEY} 41 | 42 | # Remove local cached terraform.tfstate file. Shouldn't exist since test environment is new. 43 | rm -f ${TFSTATE_FILE} 44 | 45 | # Create ourself from previous version 46 | docker run -it -v "$ORIG_SSH_AUTH_SOCK:/tmp/ssh_auth_sock" -e "SSH_AUTH_SOCK=/tmp/ssh_auth_sock" -e ENV=${ENV} -e S3_BUCKET=${S3_BUCKET} -e SSH_KEYPAIR=${SSH_KEYPAIR} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION infra:${PREV_VERSION} create 47 | 48 | # Run upgrade on current version 49 | 50 | ${__dir}/upgrade.sh || die "upgrade failed" 51 | -------------------------------------------------------------------------------- /infra/v1/scripts/upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | [ -z ${ENV} ] && error "missing ENV environment variable" 15 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 16 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 17 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 18 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 19 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 20 | 21 | export TF_VAR_access_key=${AWS_ACCESS_KEY_ID} 22 | export TF_VAR_secret_key=${AWS_SECRET_ACCESS_KEY} 23 | export TF_VAR_region=${AWS_DEFAULT_REGION} 24 | export TF_VAR_ssh_keypair=${SSH_KEYPAIR} 25 | 26 | TFSTATE_KEY="terraform/$ENV/base" 27 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 28 | 29 | [ -z ${CONSUL_AMI_ID} ] && error "undefined CONSUL_AMI_ID" 30 | export TF_VAR_consul_ami=${CONSUL_AMI_ID} 31 | 32 | 33 | pushd ${__root} 34 | 35 | # Remove local cached terraform.tfstate file. This is to avoid having a cached state file referencing another environment due to manual tests or wrong operations. 36 | rm -f ${TFSTATE_FILE} 37 | terraform remote config -backend=s3 --backend-config="bucket=${S3_BUCKET}" --backend-config="key=$TFSTATE_KEY" --backend-config="region=${AWS_DEFAULT_REGION}" 38 | 39 | # Consul server upgrade 40 | # Test all consul servers are active. Check this using the first consul server. 41 | consul01ip=$(tf_get_instance_public_ip ${TFSTATE_FILE} "consul_server01") 42 | ansible-playbook -i ${consul01ip}, ${__ansible}/test_consul_servers_active.yml 43 | 44 | ## Rolling upgrade of consul server 45 | for id in 01 02 03; do 46 | INSTANCE_ID=$(tf_get_instance_id ${TFSTATE_FILE} "consul_server${id}") 47 | if [ -z ${INSTANCE_ID} ]; then 48 | error "empty instance id" 49 | fi 50 | 51 | # check for changes 52 | set +e 53 | terraform plan -detailed-exitcode -input=false -var "env=$ENV" -target aws_instance.consul_server${id} -target aws_volume_attachment.consul_server${id}_ebs_attachment 54 | if [ $? -eq 0 ]; then 55 | echo "no changes for instance ${instance}" 56 | continue 57 | fi 58 | set -e 59 | 60 | # shutdown instance before doing terraform apply or it will fail to remove the aws_volume_attachment since it's mounted. See also https://github.com/hashicorp/terraform/issues/2957 61 | aws ec2 stop-instances --instance-ids ${INSTANCE_ID} 62 | aws ec2 wait instance-stopped --instance-ids ${INSTANCE_ID} || error "instance ${INSTANCE_ID} is not stopped" 63 | 64 | # recreate instance 65 | terraform apply -input=false -var "env=$ENV" -target aws_instance.consul_server${id} -target aws_volume_attachment.consul_server${id}_ebs_attachment 66 | 67 | # Get the new instance id 68 | INSTANCE_ID=$(tf_get_instance_id ${TFSTATE_FILE} consul_server${id}) 69 | if [ -z ${INSTANCE_ID} ]; then 70 | error "empty instance id" 71 | fi 72 | aws ec2 wait instance-running --instance-ids ${INSTANCE_ID} || error "instance ${INSTANCE_ID} not running" 73 | 74 | INSTANCE_PUBLIC_IP=$(tf_get_instance_public_ip ${TFSTATE_FILE} consul_server${id}) 75 | # Wait for the consul server instance being reachable via ssh 76 | ansible-playbook -i ${INSTANCE_PUBLIC_IP}, ${__ansible}/wait_instance_up.yml 77 | 78 | # Wait for all the consul server being active 79 | ansible-playbook -i ${INSTANCE_PUBLIC_IP}, ${__ansible}/test_consul_servers_active.yml 80 | 81 | done 82 | 83 | 84 | # Finally check that all the changes were applied 85 | # If there're some changes left behind, then we forgot to do something. 86 | echo "Checking that no changes were left behind" 87 | set +e 88 | terraform plan -detailed-exitcode -input=false -var "env=$ENV" 89 | ret=$? 90 | if [ ${ret} -eq 1 ]; then 91 | error "terraform plan error!" 92 | fi 93 | if [ ${ret} -eq 2 ]; then 94 | error "some changes were left behind!" 95 | fi 96 | set -e 97 | -------------------------------------------------------------------------------- /infra/v1/scripts/utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | function error() { 7 | echo $1 8 | exit 1 9 | } 10 | 11 | function tf_get_instance_id() { 12 | local tfstatefile=${1} 13 | local instance=${2} 14 | local id 15 | id=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".primary.id') 16 | if [ $? -ne 0 ]; then 17 | # if someone has tainted the resource try with tainted instead of primary 18 | id=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".tainted[0].id') 19 | if [ $? -ne 0 ]; then 20 | echo "" 21 | return 22 | fi 23 | fi 24 | echo $id 25 | } 26 | 27 | function tf_get_instance_public_ip() { 28 | local tfstatefile=${1} 29 | local instance=${2} 30 | local ip 31 | ip=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".primary.attributes.public_ip') 32 | if [ $? -ne 0 ]; then 33 | # if someone has tainted the resource try with tainted instead of primary 34 | ip=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".tainted[0].attributes.public_ip') 35 | if [ $? -ne 0 ]; then 36 | echo "" 37 | return 38 | fi 39 | fi 40 | echo $ip 41 | } 42 | 43 | function tf_get_all_instance_ids() { 44 | local tfstatefile=${1} 45 | local ids 46 | ids=$(cat ${tfstatefile} | jq -c -e -r -M '.modules[0].resources | to_entries | map(select(.key | test("aws_instance\\..*"))) | map(.value.primary.id)') 47 | if [ $? -ne 0 ]; then 48 | echo "" 49 | return 50 | fi 51 | echo $ids 52 | } 53 | 54 | function tf_get_all_instance_public_ips() { 55 | local tfstatefile=${1} 56 | local ids 57 | ids=$(cat ${tfstatefile} | jq -c -e -r -M '.modules[0].resources | to_entries | map(select(.key | test("aws_instance\\..*"))) | map(.value.primary.attributes.public_ip)') 58 | if [ $? -ne 0 ]; then 59 | echo "" 60 | return 61 | fi 62 | echo $ids 63 | } 64 | 65 | # Get the latest (by creation date) ami id with specified version tag value 66 | function get_ami_id_by_version() { 67 | local ami_id=$(aws ec2 describe-images --filters "Name=tag:version,Values=${1}" --query 'Images[].[ImageId,CreationDate]' --output text | sort -n -k2 | head -1 | awk '{ print $1 }') 68 | echo $ami_id 69 | } 70 | 71 | function delete_s3_object() { 72 | # Ignore errors if file doesn't exists 73 | set +e 74 | aws s3 rm "s3://${1}/${2}" 75 | set -e 76 | } 77 | -------------------------------------------------------------------------------- /infra/v2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM fedora:23 2 | 3 | RUN dnf -y install wget curl git unzip jq python python-pip python-boto3 awscli ansible docker 4 | 5 | RUN curl https://releases.hashicorp.com/terraform/0.6.12/terraform_0.6.12_linux_amd64.zip -o /tmp/terraform_0.6.12_linux_amd64.zip 6 | 7 | WORKDIR /usr/local/bin/ 8 | RUN unzip /tmp/terraform_0.6.12_linux_amd64.zip 9 | 10 | RUN mkdir -p /root/.ssh 11 | RUN chmod 700 /root/.ssh 12 | RUN echo -e "Host *\n\tStrictHostKeyChecking no\n" >> /root/.ssh/config 13 | 14 | WORKDIR /deploy 15 | ADD main.tf config ./ 16 | ADD scripts scripts/ 17 | ADD ansible ansible/ 18 | 19 | ENTRYPOINT [ "/deploy/scripts/run.sh" ] 20 | -------------------------------------------------------------------------------- /infra/v2/ansible/test_consul_servers_active.yml: -------------------------------------------------------------------------------- 1 | 2 | - hosts: all 3 | remote_user: fedora 4 | become_user: root 5 | gather_facts: false 6 | tasks: 7 | - name: Check for 3 active consul servers 8 | shell: /usr/local/bin/consul members | grep alive | grep server | wc -l 9 | register: result 10 | until: result.stdout.find("3") != -1 11 | retries: 5 12 | delay: 10 13 | 14 | - name: Fail if we cannot find 3 active consul server 15 | fail: 16 | msg: "Only {{ result.stdout }} consul servers active" 17 | when: result.stdout.find("3") == -1 18 | 19 | -------------------------------------------------------------------------------- /infra/v2/ansible/wait_instance_stopped.yml: -------------------------------------------------------------------------------- 1 | 2 | - hosts: all 3 | remote_user: fedora 4 | become_user: root 5 | gather_facts: false 6 | tasks: 7 | - name: waiting for server to be stopped 8 | local_action: wait_for host={{ inventory_hostname }} port=22 state=stopped timeout=300 9 | become: false 10 | -------------------------------------------------------------------------------- /infra/v2/ansible/wait_instance_up.yml: -------------------------------------------------------------------------------- 1 | 2 | - hosts: all 3 | remote_user: fedora 4 | become_user: root 5 | gather_facts: false 6 | tasks: 7 | - name: waiting for server to become reachable via ssh 8 | local_action: wait_for host={{ inventory_hostname }} port=22 state=started timeout=300 9 | become: false 10 | -------------------------------------------------------------------------------- /infra/v2/config: -------------------------------------------------------------------------------- 1 | CONSUL_AMI_ID= # Put here the ami id reported at the end of the build of `images/consul/v2` 2 | 3 | PREV_VERSION="v1" 4 | -------------------------------------------------------------------------------- /infra/v2/main.tf: -------------------------------------------------------------------------------- 1 | variable "access_key" {} 2 | variable "secret_key" {} 3 | 4 | variable "region" { 5 | default = "us-east-1" 6 | } 7 | 8 | variable "env" {} 9 | variable "consul_ami" {} 10 | variable "ssh_keypair" {} 11 | 12 | variable "azs" { 13 | default = "us-east-1a,us-east-1b,us-east-1c,us-east-1e" 14 | } 15 | 16 | provider "aws" { 17 | access_key = "${var.access_key}" 18 | secret_key = "${var.secret_key}" 19 | region = "${var.region}" 20 | } 21 | 22 | # The next resources define a new vpc with a public subnet for every 23 | # availabilty zone and a default security group completely open. This isn't 24 | # meant to be used in a production environment, it's just to make the vpc 25 | # definition the most concise as possible. 26 | 27 | resource "aws_vpc" "vpc" { 28 | cidr_block = "10.0.0.0/16" 29 | enable_dns_support = true 30 | enable_dns_hostnames = true 31 | tags { 32 | Name = "vpc ${var.env} ${var.region}" 33 | } 34 | } 35 | 36 | resource "aws_internet_gateway" "igw" { 37 | vpc_id = "${aws_vpc.vpc.id}" 38 | 39 | tags { 40 | Name = "igw ${var.env} ${var.region}" 41 | } 42 | } 43 | 44 | # Create a subnet for every availability zone 45 | resource "aws_subnet" "front" { 46 | count = "${length(split(\",\", var.azs))}" 47 | vpc_id = "${aws_vpc.vpc.id}" 48 | cidr_block = "10.0.${count.index * 16}.0/20" 49 | map_public_ip_on_launch = true 50 | availability_zone = "${element(split(\",\", var.azs), count.index)}" 51 | 52 | tags { 53 | Name = "subnet ${count.index} ${element(split(\",\", var.azs), count.index)}" 54 | } 55 | } 56 | 57 | resource "aws_route_table" "public" { 58 | vpc_id = "${aws_vpc.vpc.id}" 59 | route { 60 | cidr_block = "0.0.0.0/0" 61 | gateway_id = "${aws_internet_gateway.igw.id}" 62 | } 63 | 64 | tags { 65 | Name = "${var.region} ${var.env} public" 66 | } 67 | } 68 | 69 | resource "aws_route_table_association" "front" { 70 | count = "${length(split(\",\", var.azs))}" 71 | subnet_id = "${element(aws_subnet.front.*.id, count.index)}" 72 | route_table_id = "${aws_route_table.public.id}" 73 | } 74 | 75 | # An (in)security_group :D 76 | resource "aws_security_group" "allow_all" { 77 | name = "allow_all" 78 | description = "Allow all inbound traffic" 79 | 80 | vpc_id = "${aws_vpc.vpc.id}" 81 | 82 | ingress { 83 | from_port = 0 84 | to_port = 0 85 | protocol = "-1" 86 | cidr_blocks = ["0.0.0.0/0"] 87 | } 88 | 89 | egress { 90 | from_port = 0 91 | to_port = 0 92 | protocol = "-1" 93 | cidr_blocks = ["0.0.0.0/0"] 94 | } 95 | } 96 | 97 | # Here we define out instances, volumes etc. 98 | # Some notes: 99 | # * Do not use an unique resource with a count, since we want to distribute 100 | # them on different availability zones 101 | # * We have to avoid an hard dependency between the ebs and the instances since 102 | # it will make terraform recreate the ebs when an instance is recreated. 103 | 104 | resource "aws_volume_attachment" "consul_server01_ebs_attachment" { 105 | device_name = "/dev/xvdb" 106 | volume_id = "${aws_ebs_volume.consul_server01_ebs.id}" 107 | instance_id = "${aws_instance.consul_server01.id}" 108 | } 109 | 110 | resource "aws_ebs_volume" "consul_server01_ebs" { 111 | availability_zone = "us-east-1a" 112 | size = 1 113 | } 114 | 115 | resource "aws_volume_attachment" "consul_server02_ebs_attachment" { 116 | device_name = "/dev/xvdb" 117 | volume_id = "${aws_ebs_volume.consul_server02_ebs.id}" 118 | instance_id = "${aws_instance.consul_server02.id}" 119 | } 120 | 121 | resource "aws_ebs_volume" "consul_server02_ebs" { 122 | availability_zone = "us-east-1b" 123 | size = 1 124 | } 125 | 126 | resource "aws_volume_attachment" "consul_server03_ebs_attachment" { 127 | device_name = "/dev/xvdb" 128 | volume_id = "${aws_ebs_volume.consul_server03_ebs.id}" 129 | instance_id = "${aws_instance.consul_server03.id}" 130 | } 131 | 132 | resource "aws_ebs_volume" "consul_server03_ebs" { 133 | availability_zone = "us-east-1c" 134 | size = 1 135 | } 136 | 137 | 138 | # Use a fixed ip address for various reasons: 139 | # * Avoid consul raft logic problems with changing ips when instance is 140 | # recreated (https://github.com/hashicorp/consul/issues/457). Another solution 141 | # will be to let consul server leave the raft cluster but this will make a 142 | # temporary 2 node cluster and, if another node fails, it will lose the quorum 143 | # requiring a manual operation to restore it in a working state. 144 | resource "aws_instance" "consul_server01" { 145 | ami = "${var.consul_ami}" 146 | instance_type = "t2.micro" 147 | iam_instance_profile = "default_instance_profile" 148 | key_name = "${var.ssh_keypair}" 149 | subnet_id = "${aws_subnet.front.0.id}" 150 | vpc_security_group_ids = ["${aws_security_group.allow_all.id}"] 151 | private_ip = "${cidrhost(aws_subnet.front.0.cidr_block, 10)}" 152 | tags = { 153 | "consul-type" = "server" 154 | } 155 | } 156 | 157 | resource "aws_instance" "consul_server02" { 158 | ami = "${var.consul_ami}" 159 | instance_type = "t2.micro" 160 | iam_instance_profile = "default_instance_profile" 161 | key_name = "${var.ssh_keypair}" 162 | subnet_id = "${aws_subnet.front.1.id}" 163 | vpc_security_group_ids = ["${aws_security_group.allow_all.id}"] 164 | private_ip = "${cidrhost(aws_subnet.front.1.cidr_block, 10)}" 165 | tags = { 166 | "consul-type" = "server" 167 | } 168 | } 169 | 170 | resource "aws_instance" "consul_server03" { 171 | ami = "${var.consul_ami}" 172 | instance_type = "t2.micro" 173 | iam_instance_profile = "default_instance_profile" 174 | key_name = "${var.ssh_keypair}" 175 | subnet_id = "${aws_subnet.front.2.id}" 176 | vpc_security_group_ids = ["${aws_security_group.allow_all.id}"] 177 | private_ip = "${cidrhost(aws_subnet.front.2.cidr_block, 10)}" 178 | tags = { 179 | "consul-type" = "server" 180 | } 181 | } 182 | 183 | ### Outputs ### 184 | 185 | output region { 186 | value = "${var.region}" 187 | } 188 | 189 | output vpc_id { 190 | value = "${aws_vpc.vpc.id}" 191 | } 192 | 193 | output security_group_allow_all_id { 194 | value = "${aws_security_group.allow_all.id}" 195 | } 196 | 197 | output subnets { 198 | value = "${join(",", aws_subnet.front.*.id)}" 199 | } 200 | -------------------------------------------------------------------------------- /infra/v2/run-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker run -it -v /var/run/docker.sock:/var/run/docker.sock -v "$SSH_AUTH_SOCK:/tmp/ssh_auth_sock" -e "SSH_AUTH_SOCK=/tmp/ssh_auth_sock" -e "ORIG_SSH_AUTH_SOCK=$SSH_AUTH_SOCK" -e ENV=${ENV} -e S3_BUCKET=${S3_BUCKET} -e SSH_KEYPAIR=${SSH_KEYPAIR} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION infra:${VERSION} ${1} 4 | -------------------------------------------------------------------------------- /infra/v2/scripts/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | [ -z ${ENV} ] && error "missing ENV environment variable" 15 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 16 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 17 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 18 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 19 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 20 | 21 | export TF_VAR_access_key=${AWS_ACCESS_KEY_ID} 22 | export TF_VAR_secret_key=${AWS_SECRET_ACCESS_KEY} 23 | export TF_VAR_region=${AWS_DEFAULT_REGION} 24 | export TF_VAR_ssh_keypair=${SSH_KEYPAIR} 25 | 26 | TFSTATE_KEY="terraform/$ENV/base" 27 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 28 | 29 | [ -z ${CONSUL_AMI_ID} ] && error "undefined CONSUL_AMI_ID" 30 | export TF_VAR_consul_ami=${CONSUL_AMI_ID} 31 | 32 | 33 | pushd ${__root} 34 | 35 | # Remove local cached terraform.tfstate file. This is to avoid having a cached state file referencing another environment due to manual tests or wrong operations. 36 | rm -f ${TFSTATE_FILE} 37 | terraform remote config -backend=s3 --backend-config="bucket=${S3_BUCKET}" --backend-config="key=$TFSTATE_KEY" --backend-config="region=${AWS_DEFAULT_REGION}" 38 | 39 | terraform plan -input=false -var "env=$ENV" || error "terraform plan failed" 40 | 41 | terraform apply -input=false -var "env=$ENV" || error "terraform apply failed" 42 | 43 | ALL_INSTANCE_IDS=$(tf_get_all_instance_ids ${TFSTATE_FILE}) 44 | aws ec2 wait instance-running --instance-ids ${ALL_INSTANCE_IDS} || error "some instances not active" 45 | 46 | # Wait all instances are reachable via ssh 47 | ansible-playbook -i ${__root}/scripts/terraform_to_ansible_inventory.sh ${__ansible}/wait_instance_up.yml 48 | 49 | # Wait for all the consul server being active. Check this using the first consul server. 50 | consul01ip=$(tf_get_instance_public_ip ${TFSTATE_FILE} "consul_server01") 51 | ansible-playbook -i ${consul01ip}, ${__ansible}/test_consul_servers_active.yml 52 | -------------------------------------------------------------------------------- /infra/v2/scripts/destroy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | [ -z ${ENV} ] && error "missing ENV environment variable" 15 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 16 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 17 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 18 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 19 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 20 | 21 | export TF_VAR_access_key=${AWS_ACCESS_KEY_ID} 22 | export TF_VAR_secret_key=${AWS_SECRET_ACCESS_KEY} 23 | export TF_VAR_region=${AWS_DEFAULT_REGION} 24 | export TF_VAR_ssh_keypair=${SSH_KEYPAIR} 25 | 26 | TFSTATE_KEY="terraform/$ENV/base" 27 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 28 | 29 | export TF_VAR_consul_ami="IDONTCARE" 30 | 31 | # Remove local cached terraform.tfstate file. This is to avoid having a cached state file referencing another environment due to manual tests or wrong operations. 32 | rm -f ${TFSTATE_FILE} 33 | terraform remote config -backend=s3 --backend-config="bucket=${S3_BUCKET}" --backend-config="key=$TFSTATE_KEY" --backend-config="region=${AWS_DEFAULT_REGION}" 34 | 35 | # shutdown instance before doing terraform apply or it will fail to remove the aws_volume_attachment since it's mounted. See also https://github.com/hashicorp/terraform/issues/2957 36 | INSTANCE_IDS=$(tf_get_all_instance_ids ${TFSTATE_FILE}) 37 | if [ ${INSTANCE_IDS} != "[]" ]; then 38 | aws ec2 stop-instances --instance-ids ${INSTANCE_IDS} 39 | aws ec2 wait instance-stopped --instance-ids ${INSTANCE_IDS} || error "some instance are not stopped" 40 | fi 41 | 42 | terraform destroy -input=false -force -var "env=$ENV" || error "terraform plan failed" 43 | -------------------------------------------------------------------------------- /infra/v2/scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__dir}/utils.sh 11 | 12 | case ${1} in 13 | "create") 14 | ${__root}/scripts/create.sh 15 | ;; 16 | "upgrade") 17 | ${__root}/scripts/upgrade.sh 18 | ;; 19 | "destroy") 20 | ${__root}/scripts/destroy.sh 21 | ;; 22 | "test-create") 23 | ${__root}/scripts/test_create.sh 24 | ;; 25 | "test-upgrade") 26 | ${__root}/scripts/test_upgrade.sh 27 | ;; 28 | *) 29 | error "Usage: ${0} {create|upgrade|destroy|test-create|test-upgrade}" 30 | esac 31 | 32 | -------------------------------------------------------------------------------- /infra/v2/scripts/terraform_to_ansible_inventory.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # A simple script that generates an ansible inventory from a terraform state file 4 | set -x 5 | set -e 6 | 7 | __dir="$(readlink -f $(dirname ${0}))" 8 | __root="$(readlink -f ${__dir}/../)" 9 | 10 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 11 | 12 | cat ${TFSTATE_FILE} | jq -c -e -r -M '{ all: .modules[0].resources | to_entries | map(select(.key | test("aws_instance\\..*"))) | map(.value.primary.attributes.public_ip) }' 13 | -------------------------------------------------------------------------------- /infra/v2/scripts/test_create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__dir}/utils.sh 11 | 12 | function cleanup() { 13 | ${__dir}/destroy.sh 14 | delete_s3_object ${S3_BUCKET} ${TFSTATE_KEY} 15 | } 16 | 17 | env 18 | 19 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 20 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 21 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 22 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 23 | 24 | # Generate a randon environment name 25 | export ENV="test-$(uuidgen | cut -c1-8)" 26 | 27 | TFSTATE_KEY="terraform/$ENV/base" 28 | 29 | trap cleanup EXIT 30 | 31 | pushd ${__root} 32 | 33 | # Remove local cached terraform.tfstate file. Shouldn't exist since test 34 | # environment is new but useful for local test runs. 35 | rm -f .terraform/terraform.tfstate 36 | 37 | ${__dir}/create.sh || error "create failed" 38 | -------------------------------------------------------------------------------- /infra/v2/scripts/test_upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | function cleanup() { 15 | ${__dir}/destroy.sh 16 | delete_s3_object ${S3_BUCKET} ${TFSTATE_KEY} 17 | } 18 | 19 | env 20 | 21 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 22 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 23 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 24 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 25 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 26 | 27 | [ -z ${PREV_VERSION} ] && error "PREV_VERSION undefined, cannot test upgrade" 28 | 29 | # Generate a randon environment name 30 | export ENV="test-$(uuidgen | cut -c1-8)" 31 | 32 | TFSTATE_KEY="terraform/$ENV/base" 33 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 34 | 35 | trap cleanup EXIT 36 | 37 | pushd ${__root} 38 | 39 | # Remove old remote tfstate 40 | delete_s3_object ${S3_BUCKET} ${TFSTATE_KEY} 41 | 42 | # Remove local cached terraform.tfstate file. Shouldn't exist since test environment is new. 43 | rm -f ${TFSTATE_FILE} 44 | 45 | # Create ourself from previous version 46 | docker run -it -v "$ORIG_SSH_AUTH_SOCK:/tmp/ssh_auth_sock" -e "SSH_AUTH_SOCK=/tmp/ssh_auth_sock" -e ENV=${ENV} -e S3_BUCKET=${S3_BUCKET} -e SSH_KEYPAIR=${SSH_KEYPAIR} -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION infra:${PREV_VERSION} create 47 | 48 | # Run upgrade on current version 49 | 50 | ${__dir}/upgrade.sh || die "upgrade failed" 51 | -------------------------------------------------------------------------------- /infra/v2/scripts/upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | __dir="$(readlink -f $(dirname ${0}))" 7 | __root="$(readlink -f ${__dir}/../)" 8 | __ansible="${__root}/ansible" 9 | 10 | source ${__root}/config 11 | 12 | source ${__dir}/utils.sh 13 | 14 | [ -z ${ENV} ] && error "missing ENV environment variable" 15 | [ -z ${AWS_ACCESS_KEY_ID} ] && error "missing AWS_ACCESS_KEY_ID environment variable" 16 | [ -z ${AWS_SECRET_ACCESS_KEY} ] && error "missing SECRET_ACCESS_KEY environment variable" 17 | [ -z ${AWS_DEFAULT_REGION} ] && error "missing AWS_DEFAULT_REGION environment variable" 18 | [ -z ${S3_BUCKET} ] && error "missing S3_BUCKET environment variable" 19 | [ -z ${SSH_KEYPAIR} ] && error "missing SSH_KEYPAIR environment variable" 20 | 21 | export TF_VAR_access_key=${AWS_ACCESS_KEY_ID} 22 | export TF_VAR_secret_key=${AWS_SECRET_ACCESS_KEY} 23 | export TF_VAR_region=${AWS_DEFAULT_REGION} 24 | export TF_VAR_ssh_keypair=${SSH_KEYPAIR} 25 | 26 | TFSTATE_KEY="terraform/$ENV/base" 27 | TFSTATE_FILE="${__root}/.terraform/terraform.tfstate" 28 | 29 | [ -z ${CONSUL_AMI_ID} ] && error "undefined CONSUL_AMI_ID" 30 | export TF_VAR_consul_ami=${CONSUL_AMI_ID} 31 | 32 | 33 | pushd ${__root} 34 | 35 | # Remove local cached terraform.tfstate file. This is to avoid having a cached state file referencing another environment due to manual tests or wrong operations. 36 | rm -f ${TFSTATE_FILE} 37 | terraform remote config -backend=s3 --backend-config="bucket=${S3_BUCKET}" --backend-config="key=$TFSTATE_KEY" --backend-config="region=${AWS_DEFAULT_REGION}" 38 | 39 | # Consul server upgrade 40 | # Test all consul servers are active. Check this using the first consul server. 41 | consul01ip=$(tf_get_instance_public_ip ${TFSTATE_FILE} "consul_server01") 42 | ansible-playbook -i ${consul01ip}, ${__ansible}/test_consul_servers_active.yml 43 | 44 | ## Rolling upgrade of consul server 45 | for id in 01 02 03; do 46 | INSTANCE_ID=$(tf_get_instance_id ${TFSTATE_FILE} "consul_server${id}") 47 | if [ -z ${INSTANCE_ID} ]; then 48 | error "empty instance id" 49 | fi 50 | 51 | # check for changes 52 | set +e 53 | terraform plan -detailed-exitcode -input=false -var "env=$ENV" -target aws_instance.consul_server${id} -target aws_volume_attachment.consul_server${id}_ebs_attachment 54 | if [ $? -eq 0 ]; then 55 | echo "no changes for instance ${instance}" 56 | continue 57 | fi 58 | set -e 59 | 60 | # shutdown instance before doing terraform apply or it will fail to remove the aws_volume_attachment since it's mounted. See also https://github.com/hashicorp/terraform/issues/2957 61 | aws ec2 stop-instances --instance-ids ${INSTANCE_ID} 62 | aws ec2 wait instance-stopped --instance-ids ${INSTANCE_ID} || error "instance ${INSTANCE_ID} is not stopped" 63 | 64 | # recreate instance 65 | terraform apply -input=false -var "env=$ENV" -target aws_instance.consul_server${id} -target aws_volume_attachment.consul_server${id}_ebs_attachment 66 | 67 | # Get the new instance id 68 | INSTANCE_ID=$(tf_get_instance_id ${TFSTATE_FILE} consul_server${id}) 69 | if [ -z ${INSTANCE_ID} ]; then 70 | error "empty instance id" 71 | fi 72 | aws ec2 wait instance-running --instance-ids ${INSTANCE_ID} || error "instance ${INSTANCE_ID} not running" 73 | 74 | INSTANCE_PUBLIC_IP=$(tf_get_instance_public_ip ${TFSTATE_FILE} consul_server${id}) 75 | # Wait for the consul server instance being reachable via ssh 76 | ansible-playbook -i ${INSTANCE_PUBLIC_IP}, ${__ansible}/wait_instance_up.yml 77 | 78 | # Wait for all the consul server being active 79 | ansible-playbook -i ${INSTANCE_PUBLIC_IP}, ${__ansible}/test_consul_servers_active.yml 80 | 81 | done 82 | 83 | 84 | # Finally check that all the changes were applied 85 | # If there're some changes left behind, then we forgot to do something. 86 | echo "Checking that no changes were left behind" 87 | set +e 88 | terraform plan -detailed-exitcode -input=false -var "env=$ENV" 89 | ret=$? 90 | if [ ${ret} -eq 1 ]; then 91 | error "terraform plan error!" 92 | fi 93 | if [ ${ret} -eq 2 ]; then 94 | error "some changes were left behind!" 95 | fi 96 | set -e 97 | -------------------------------------------------------------------------------- /infra/v2/scripts/utils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | function error() { 7 | echo $1 8 | exit 1 9 | } 10 | 11 | function tf_get_instance_id() { 12 | local tfstatefile=${1} 13 | local instance=${2} 14 | local id 15 | id=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".primary.id') 16 | if [ $? -ne 0 ]; then 17 | # if someone has tainted the resource try with tainted instead of primary 18 | id=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".tainted[0].id') 19 | if [ $? -ne 0 ]; then 20 | echo "" 21 | return 22 | fi 23 | fi 24 | echo $id 25 | } 26 | 27 | function tf_get_instance_public_ip() { 28 | local tfstatefile=${1} 29 | local instance=${2} 30 | local ip 31 | ip=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".primary.attributes.public_ip') 32 | if [ $? -ne 0 ]; then 33 | # if someone has tainted the resource try with tainted instead of primary 34 | ip=$(cat ${tfstatefile} | jq -e -r -M '.modules[0].resources."aws_instance.'"${instance}"'".tainted[0].attributes.public_ip') 35 | if [ $? -ne 0 ]; then 36 | echo "" 37 | return 38 | fi 39 | fi 40 | echo $ip 41 | } 42 | 43 | function tf_get_all_instance_ids() { 44 | local tfstatefile=${1} 45 | local ids 46 | ids=$(cat ${tfstatefile} | jq -c -e -r -M '.modules[0].resources | to_entries | map(select(.key | test("aws_instance\\..*"))) | map(.value.primary.id)') 47 | if [ $? -ne 0 ]; then 48 | echo "" 49 | return 50 | fi 51 | echo $ids 52 | } 53 | 54 | function tf_get_all_instance_public_ips() { 55 | local tfstatefile=${1} 56 | local ids 57 | ids=$(cat ${tfstatefile} | jq -c -e -r -M '.modules[0].resources | to_entries | map(select(.key | test("aws_instance\\..*"))) | map(.value.primary.attributes.public_ip)') 58 | if [ $? -ne 0 ]; then 59 | echo "" 60 | return 61 | fi 62 | echo $ids 63 | } 64 | 65 | # Get the latest (by creation date) ami id with specified version tag value 66 | function get_ami_id_by_version() { 67 | local ami_id=$(aws ec2 describe-images --filters "Name=tag:version,Values=${1}" --query 'Images[].[ImageId,CreationDate]' --output text | sort -n -k2 | head -1 | awk '{ print $1 }') 68 | echo $ami_id 69 | } 70 | 71 | function delete_s3_object() { 72 | # Ignore errors if file doesn't exists 73 | set +e 74 | aws s3 rm "s3://${1}/${2}" 75 | set -e 76 | } 77 | -------------------------------------------------------------------------------- /setup/iam_roles.tf: -------------------------------------------------------------------------------- 1 | variable "access_key" {} 2 | variable "secret_key" {} 3 | 4 | variable "region" { 5 | default = "us-east-1" 6 | } 7 | 8 | provider "aws" { 9 | access_key = "${var.access_key}" 10 | secret_key = "${var.secret_key}" 11 | region = "${var.region}" 12 | } 13 | 14 | # Define an iam instance profile needed for executing the aws cli inside the 15 | # instances without expliciting providing and access and a secret key 16 | resource "aws_iam_role" "instance_aws_access_role" { 17 | name = "instance_aws_access" 18 | path = "/" 19 | assume_role_policy = <