├── .gitignore
├── terraform
    ├── provider.tf
    ├── variables.tf
    ├── loadbalancing.tf
    ├── iam.tf
    ├── securitygroups.tf
    └── autoscaling.tf
├── packer
    ├── docker.conf
    ├── docker.json
    └── init.py
└── README.md


/.gitignore:
--------------------------------------------------------------------------------
1 | *tfstate*
2 | .env
3 | node_modules
4 | package.json
5 | 


--------------------------------------------------------------------------------
/terraform/provider.tf:
--------------------------------------------------------------------------------
1 | provider "aws" {
2 |   region = "${var.aws_region}"
3 | }
4 | 


--------------------------------------------------------------------------------
/packer/docker.conf:
--------------------------------------------------------------------------------
1 | [Service]
2 | ExecStart=
3 | ExecStart=/usr/bin/dockerd -H fd:// -H tcp://0.0.0.0:2376
4 | 


--------------------------------------------------------------------------------
/terraform/variables.tf:
--------------------------------------------------------------------------------
 1 | variable "aws_region" {
 2 |   type    = "string"
 3 |   default = "us-east-1"
 4 | }
 5 | 
 6 | variable "aws_az" {
 7 |   type    = "list"
 8 |   default = ["us-east-1a","us-east-1b","us-east-1c"]
 9 | }
10 | 
11 | variable "ec2_ami" {}
12 | 
13 | variable "manager_count" {}
14 | 
15 | variable "worker_count" {}
16 | 
17 | variable "manager_size" {
18 |   default = "t2.micro"
19 | }
20 | 
21 | variable "worker_size" {
22 |   default = "t2.micro"
23 | }
24 | 
25 | variable "key" {}
26 | 


--------------------------------------------------------------------------------
/terraform/loadbalancing.tf:
--------------------------------------------------------------------------------
 1 | resource "aws_elb" "swarm_lb" {
 2 |   name                = "swarm-lb"
 3 |   availability_zones  = "${var.aws_az}"
 4 |   security_groups     = ["${aws_security_group.swarm_lb_sg.id}"]
 5 | 
 6 |   listener {
 7 |     instance_port     = 80
 8 |     instance_protocol = "http"
 9 |     lb_port           = 80
10 |     lb_protocol       = "http"
11 |   }
12 | 
13 |   health_check {
14 |     healthy_threshold   = 2
15 |     unhealthy_threshold = 2
16 |     timeout             = 3
17 |     target              = "TCP:2376"
18 |     interval            = 30
19 |   }
20 | 
21 |   cross_zone_load_balancing   = true
22 |   idle_timeout                = 400
23 |   connection_draining         = true
24 |   connection_draining_timeout = 400
25 | 
26 |   tags {
27 |     Name = "swarm-elb"
28 |   }
29 | }
30 | 


--------------------------------------------------------------------------------
/terraform/iam.tf:
--------------------------------------------------------------------------------
 1 | resource "aws_iam_instance_profile" "swarm_cluster_profile" {
 2 |   name    = "swarm-cluster-profile"
 3 |   roles   = ["${aws_iam_role.swarm_cluster_role.name}"]
 4 | }
 5 | 
 6 | resource "aws_iam_role_policy" "swarm_cluster_policy" {
 7 |   name = "swarm-cluster-policy"
 8 |   role = "${aws_iam_role.swarm_cluster_role.id}"
 9 | 
10 |   policy = <<EOF
11 | {
12 |   "Version": "2012-10-17",
13 |   "Statement": [
14 |     {
15 |       "Action": [
16 |         "ec2:Describe*",
17 |         "ec2:CreateTags"
18 |       ],
19 |       "Effect": "Allow",
20 |       "Resource": "*"
21 |     }
22 |   ]
23 | }
24 | EOF
25 | }
26 | 
27 | resource "aws_iam_role" "swarm_cluster_role" {
28 |   name = "swarm-cluster-role"
29 |   path = "/"
30 | 
31 |   assume_role_policy = <<EOF
32 | {
33 |     "Version": "2012-10-17",
34 |     "Statement": [
35 |         {
36 |             "Action": "sts:AssumeRole",
37 |             "Principal": {
38 |                "Service": "ec2.amazonaws.com"
39 |             },
40 |             "Effect": "Allow",
41 |             "Sid": ""
42 |         }
43 |     ]
44 | }
45 | EOF
46 | }
47 | 


--------------------------------------------------------------------------------
/packer/docker.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "variables": {
 3 |         "region": null
 4 |     },
 5 |     "builders": [{
 6 |         "type": "amazon-ebs",
 7 |         "ami_name": "docker-ami-{{timestamp}}",
 8 |         "instance_type": "t2.micro",
 9 |         "region": "{{user `region`}}",
10 |         "source_ami": "ami-4dd2575b",
11 |         "ssh_username": "ubuntu"
12 |     }],
13 |     "provisioners": [{
14 |             "type": "file",
15 |             "source": "./docker.conf",
16 |             "destination": "/home/ubuntu/docker.conf"
17 |         },{
18 |             "type": "file",
19 |             "source": "./init.py",
20 |             "destination": "/home/ubuntu/init.py"
21 |         },
22 |         {
23 |             "type": "shell",
24 |             "inline": [
25 |                 "sudo apt-get update",
26 |                 "sudo apt-get install -y apt-transport-https ca-certificates nfs-common",
27 |                 "sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D",
28 |                 "echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' | sudo tee /etc/apt/sources.list.d/docker.list",
29 |                 "sudo apt-get update",
30 |                 "sudo apt-get install -y docker-engine",
31 |                 "sudo usermod -aG docker ubuntu",
32 |                 "sudo mkdir -p /etc/systemd/system/docker.service.d",
33 |                 "sudo mv /home/ubuntu/docker.conf /etc/systemd/system/docker.service.d/docker.conf",
34 |                 "curl -sSL https://dl.bintray.com/emccode/rexray/install | sh -s -- stable",
35 |                 "sudo apt install -y python3-pip",
36 |                 "sudo pip3 install boto3"
37 |             ]
38 |         }
39 |     ]
40 | }
41 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Docker Swarm on AWS
 2 | This repo provisions a Docker Swarm Cluster on AWS using Packer and Terraform.
 3 | 
 4 | First export yout credentials and create the ami using packer. Provide the region variable when executing packer. The packer script uploads two files *docker.conf* and *init.py*. The first is for customize the docker service and the latter is a python script for initialize the cluster and add nodes to it.
 5 | 
 6 | ```
 7 | export AWS_ACCESS_KEY_ID=your-access-key-here
 8 | export AWS_SECRET_ACCESS_KEY=your-secret-access-key-here
 9 | export AWS_DEFAULT_REGION=your-region
10 | 
11 | cd packer/
12 | packer build -var 'region=us-east-1' docker.json
13 | ```
14 | 
15 | Write down the ami id generated.
16 | 
17 | Set the terraform variables like this:
18 | 
19 | ```
20 | export TF_VAR_manager_count=how-many-managers-do-you-want Ej: 1, 3, 5 (odd numbers recommended)
21 | export TF_VAR_worker_count=how-many-workers-do-you-want Ej: 1,5,10,1000
22 | export TF_VAR_key=your-key
23 | export TF_VAR_ec2_ami=the-ami-key-generated-with-packer
24 | ```
25 | 
26 | Also check all the variables on the `variables.tf` file and modify as you wish. Now you can provision the infrastructure with Terraform
27 | 
28 | ```
29 | cd terraform/
30 | terraform apply
31 | ```
32 | 
33 | A brief description of scripts is provided next:
34 | 
35 | * `provider.tf` defines the aws region.
36 | * `variables.tf` contains all variables used on the scripts.
37 | * `securitygroups.tf` defines the security groups for elastic load balancer and ec2 instances.
38 | * `iam.tf` contains the iam role assigned to cluster nodes.
39 | * `loadbalancing.tf` defines the elastic load balancer
40 | * `autoscaling.tf` contains the launch configurations and autoscaling groups for worker and manager nodes.
41 | 
42 | Currently there are no outputs defined so you should login to AWS console to get nodes information.
43 | 
44 | # TO-DO
45 | * ~~Think about how to initialize cluster and how worker nodes will automagically join to it~~
46 | * ~~Improve cluster initializer/join-nodes script~~
47 | * ~~Remove workers from cluster when replaced~~
48 | * Deploy some CI/CD tools on cluster (Jenkins?)
49 | * Add a Docker registry
50 | * Add a proxy for accessing containers (Docker Flow Proxy?)
51 | * Add a monitoring tool
52 | 


--------------------------------------------------------------------------------
/terraform/securitygroups.tf:
--------------------------------------------------------------------------------
 1 | resource "aws_security_group" "swarm_sg" {
 2 |   name        = "swarm-sg"
 3 |   description = "A security group for Swarm cluster instances"
 4 | 
 5 |   //for docker communication
 6 |   ingress {
 7 |     from_port   = 2377
 8 |     to_port     = 2377
 9 |     protocol    = "tcp"
10 |     self        = true
11 |   }
12 |   ingress {
13 |     from_port       = 2376
14 |     to_port         = 2376
15 |     protocol        = "tcp"
16 |     security_groups = ["${aws_security_group.swarm_lb_sg.id}"]
17 |     self            = true
18 |   }
19 |   ingress {
20 |     from_port   = 7946
21 |     to_port     = 7946
22 |     protocol    = "tcp"
23 |     self        = true
24 |   }
25 |   ingress {
26 |     from_port   = 7946
27 |     to_port     = 7946
28 |     protocol    = "udp"
29 |     self        = true
30 |   }
31 |   ingress {
32 |     from_port   = 4789
33 |     to_port     = 4789
34 |     protocol    = "udp"
35 |     self        = true
36 |   }
37 | 
38 |   //for accessing the inner services
39 |   //jenkins agents (internal only)
40 |   ingress {
41 |     from_port   = 50000
42 |     to_port     = 50000
43 |     protocol    = "tcp"
44 |     self        = true
45 |   }
46 |   //http
47 |   ingress {
48 |     from_port       = 80
49 |     to_port         = 80
50 |     protocol        = "tcp"
51 |     security_groups = ["${aws_security_group.swarm_lb_sg.id}"]
52 |   }
53 |   //https
54 |   ingress {
55 |     from_port       = 443
56 |     to_port         = 443
57 |     protocol        = "tcp"
58 |     security_groups = ["${aws_security_group.swarm_lb_sg.id}"]
59 |   }
60 |   //ssh
61 |   ingress {
62 |     from_port   = 22
63 |     to_port     = 22
64 |     protocol    = "tcp"
65 |     cidr_blocks = ["0.0.0.0/0"]
66 |   }
67 | 
68 |   //egress
69 |   egress {
70 |     from_port   = 0
71 |     to_port     = 0
72 |     protocol    = "-1"
73 |     cidr_blocks = ["0.0.0.0/0"]
74 |   }
75 | }
76 | 
77 | resource "aws_security_group" "swarm_lb_sg" {
78 |   name        = "swarm-lb-sg"
79 |   description = "A security group for using swarm load balancer"
80 | 
81 |   egress {
82 |     from_port   = 0
83 |     to_port     = 0
84 |     protocol    = "-1"
85 |     cidr_blocks = ["0.0.0.0/0"]
86 |   }
87 | 
88 |   ingress {
89 |     from_port   = 80
90 |     to_port     = 80
91 |     protocol    = "tcp"
92 |     cidr_blocks = ["0.0.0.0/0"]
93 |   }
94 | }
95 | 


--------------------------------------------------------------------------------
/terraform/autoscaling.tf:
--------------------------------------------------------------------------------
 1 | resource "aws_launch_configuration" "swarm_manager_as_conf" {
 2 |   name_prefix     = "swarm-manager-as-conf-"
 3 |   image_id        = "${var.ec2_ami}"
 4 |   instance_type   = "${var.manager_size}"
 5 |   key_name        = "${var.key}"
 6 |   security_groups = ["${aws_security_group.swarm_sg.id}"]
 7 |   iam_instance_profile  = "${aws_iam_instance_profile.swarm_cluster_profile.id}"
 8 | 
 9 |   user_data       = <<EOF
10 | #!/bin/bash
11 | export ROLE=manager
12 | export INSTANCE=$(curl http://169.254.169.254/latest/meta-data/instance-id)
13 | export AWS_DEFAULT_REGION=${var.aws_region}
14 | python3 /home/ubuntu/init.py
15 | EOF
16 | 
17 |   lifecycle {
18 |     create_before_destroy = true
19 |   }
20 | }
21 | 
22 | resource "aws_autoscaling_group" "swarm_manager_asg" {
23 |   name                  = "swarm-manager-asg"
24 |   max_size              = 5
25 |   min_size              = 1
26 |   desired_capacity      = "${var.manager_count}"
27 |   availability_zones    = "${var.aws_az}"
28 |   launch_configuration  = "${aws_launch_configuration.swarm_manager_as_conf.name}"
29 |   health_check_type     = "ELB"
30 |   load_balancers        = ["${aws_elb.swarm_lb.name}"]
31 | 
32 |   tag {
33 |     key                 = "Name"
34 |     value               = "SwarmManager"
35 |     propagate_at_launch = true
36 |   }
37 | 
38 |   tag {
39 |     key                 = "Init"
40 |     value               = "false"
41 |     propagate_at_launch = true
42 |   }
43 | 
44 |   lifecycle {
45 |     create_before_destroy = true
46 |   }
47 | }
48 | 
49 | resource "aws_launch_configuration" "swarm_worker_as_conf" {
50 |   name_prefix     = "swarm-worker-as-conf-"
51 |   image_id        = "${var.ec2_ami}"
52 |   instance_type   = "${var.worker_size}"
53 |   key_name        = "${var.key}"
54 |   security_groups = ["${aws_security_group.swarm_sg.id}"]
55 |   iam_instance_profile  = "${aws_iam_instance_profile.swarm_cluster_profile.id}"
56 | 
57 |   user_data       = <<EOF
58 | #!/bin/bash
59 | export ROLE=worker
60 | export INSTANCE=$(curl http://169.254.169.254/latest/meta-data/instance-id)
61 | export AWS_DEFAULT_REGION=${var.aws_region}
62 | python3 /home/ubuntu/init.py
63 | EOF
64 | 
65 |   lifecycle {
66 |     create_before_destroy = true
67 |   }
68 | }
69 | 
70 | resource "aws_autoscaling_group" "swarm_worker_asg" {
71 |   name                  = "swarm-worker-asg"
72 |   max_size              = 1000
73 |   min_size              = 1
74 |   desired_capacity      = "${var.worker_count}"
75 |   availability_zones    = "${var.aws_az}"
76 |   launch_configuration  = "${aws_launch_configuration.swarm_worker_as_conf.name}"
77 |   health_check_type     = "ELB"
78 |   load_balancers        = ["${aws_elb.swarm_lb.name}"]
79 |   depends_on            = ["aws_autoscaling_group.swarm_manager_asg"]
80 | 
81 |   tag {
82 |     key                 = "Name"
83 |     value               = "SwarmWorker"
84 |     propagate_at_launch = true
85 |   }
86 | 
87 |   lifecycle {
88 |     create_before_destroy = true
89 |   }
90 | }
91 | 


--------------------------------------------------------------------------------
/packer/init.py:
--------------------------------------------------------------------------------
 1 | ####    IMPORTS
 2 | import os
 3 | from subprocess import run
 4 | from subprocess import call
 5 | from subprocess import PIPE
 6 | import boto3
 7 | 
 8 | ####    INITIALIZE VARIABLES
 9 | ec2 = boto3.client('ec2')
10 | role = os.environ['ROLE']
11 | current_instance = os.environ['INSTANCE']
12 | 
13 | ####    SOME UTILITIES
14 | def add_tag(tags):
15 |     response_tags = ec2.create_tags(Resources=[current_instance], Tags=tags)
16 |     print(response_tags)
17 | 
18 | def describe_nodes(filters):
19 |     response = ec2.describe_instances(Filters=filters)
20 |     return response
21 | 
22 | def count_nodes(nodes):
23 |     count = 0
24 |     instances_list = []
25 |     for ins in nodes:
26 |         count += len(ins['Instances'])
27 |         for obj in ins['Instances']:
28 |             instances_list.append(obj)
29 |     print(count)
30 |     return count, instances_list
31 | 
32 | ####    QUERY MANAGERS && WORKERS
33 | managers_running = describe_nodes([{'Name': 'tag:Name', 'Values': ['SwarmManager']}, {'Name': 'instance-state-name', 'Values': ['running']}, {'Name': 'tag:Init', 'Values': ['true']}])
34 | managers_replaced = describe_nodes([{'Name': 'tag:Name', 'Values': ['SwarmManager']}, {'Name': 'instance-state-name', 'Values': ['shutting-down', 'stopped', 'terminated', 'stopping']}, {'Name': 'tag:Init', 'Values': ['true']}])
35 | workers_replaced = describe_nodes([{'Name': 'tag:Name', 'Values': ['SwarmWorker']}, {'Name': 'instance-state-name', 'Values': ['shutting-down', 'stopped', 'terminated', 'stopping']}])
36 | 
37 | instances_count, instances = count_nodes(managers_running['Reservations'])
38 | replaced_instances_count, replaced_instances = count_nodes(managers_replaced['Reservations'])
39 | replaced_workers_count, replaced_workers = count_nodes(workers_replaced['Reservations'])
40 | 
41 | ####    INITIALIZE/JOIN CLUSTER
42 | if instances_count > 0 or role == 'worker':
43 |     ####    JOIN CLUSTER
44 |     manager = instances[0]['PrivateIpAddress'] # any running manager
45 |     call(["echo", "'Here I join to the cluster {}'".format(manager)]) # a friendly message for debugging
46 |     token = run(["docker", "-H {}:2376".format(manager), "swarm", "join-token", "-q", role], stdout=PIPE) # get the token
47 |     call(['docker', 'swarm', 'join', '{}:2377'.format(manager), '--token', token.stdout.decode('utf-8').replace('\n', '')]) # join to cluster
48 | 
49 |     ####    ADD TAGS
50 |     hostname = run(['hostname'], stdout=PIPE)
51 |     if role == 'manager': # add init flag and hostname
52 |         add_tag([{'Key': 'Init', 'Value': 'true'}, {'Key': 'Hostname', 'Value': hostname.stdout.decode('utf-8').replace('\n', '')}])
53 |     else: # adds hostname only
54 |         add_tag([{'Key': 'Hostname', 'Value': hostname.stdout.decode('utf-8').replace('\n', '')}])
55 | 
56 |     ####    CLEAN NODES
57 |     if replaced_instances_count > 0 and role == 'manager':
58 |         ## docker demote node && docker rm node
59 |         for instance in replaced_instances:
60 |             for tag in instance['Tags']:
61 |                 if tag['Key'] == 'Hostname':
62 |                     call(["docker", "node", "demote", tag['Value']])
63 |                     call(["docker", "node", "rm", tag['Value']])
64 | 
65 |     if replaced_workers_count > 0 and role == 'manager':
66 |         ## docker rm node
67 |         for instance in replaced_workers:
68 |             for tag in instance['Tags']:
69 |                 if tag['Key'] == 'Hostname':
70 |                     call(["docker", "node", "rm", tag['Value']])
71 | 
72 | 
73 | else:
74 |     ####    INIT CLUSTER
75 |     call(["echo", "'Here I init the cluster'"])
76 |     call(["docker", "swarm", "init"])
77 | 
78 |     ####    ADD TAGS
79 |     hostname = run(['hostname'], stdout=PIPE)
80 |     add_tag([{'Key': 'Init', 'Value': 'true'}, {'Key': 'Hostname', 'Value': hostname.stdout.decode('utf-8').replace('\n', '')}])
81 | 


--------------------------------------------------------------------------------