├── docs └── img │ ├── hub-cfn.png │ └── nodes-cfn.png ├── .gitignore ├── docker ├── ecs-node-chrome │ ├── Makefile │ └── Dockerfile ├── ecs-node-firefox │ ├── Makefile │ └── Dockerfile └── common │ ├── ecs_entry_point.sh │ └── ecs-get-port-mapping.py ├── .env.sample ├── LICENSE ├── Makefile ├── README.md └── cloudformation └── ecs-selenium.cfn.yml /docs/img/hub-cfn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RetailMeNotSandbox/ecs-selenium/HEAD/docs/img/hub-cfn.png -------------------------------------------------------------------------------- /docs/img/nodes-cfn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RetailMeNotSandbox/ecs-selenium/HEAD/docs/img/nodes-cfn.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | *.jar 3 | /docker/common/selenium 4 | docker/ecs-node-chrome/common/ 5 | docker/ecs-node-firefox/common/ 6 | -------------------------------------------------------------------------------- /docker/ecs-node-chrome/Makefile: -------------------------------------------------------------------------------- 1 | include ../../.env 2 | export $(shell sed 's/=.*//' ../../.env) 3 | VERSION=$(ECS_SELENIUM_CHROME_REPOSITORY_VERSION) 4 | 5 | REGISTRY=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com 6 | 7 | clean: 8 | rm -rf common 9 | 10 | build: 11 | mkdir -p common/ 12 | rsync -av ../common/* common/ 13 | docker build -t $(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE) --build-arg VERSION=$(VERSION) . 14 | rm -rf common/ 15 | 16 | tag: build 17 | docker tag $(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE) $(REGISTRY)/$(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE):$(VERSION) 18 | docker tag $(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE) $(REGISTRY)/$(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE):latest 19 | 20 | push: tag 21 | docker push $(REGISTRY)/$(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE):$(VERSION) 22 | docker push $(REGISTRY)/$(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE):latest 23 | -------------------------------------------------------------------------------- /docker/ecs-node-firefox/Makefile: -------------------------------------------------------------------------------- 1 | include ../../.env 2 | export $(shell sed 's/=.*//' ../../.env) 3 | VERSION=$(ECS_SELENIUM_FIREFOX_REPOSITORY_VERSION) 4 | 5 | REGISTRY=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com 6 | 7 | clean: 8 | rm -rf common 9 | 10 | build: 11 | mkdir -p common/ 12 | rsync -av ../common/* common/ 13 | docker build -t $(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE) --build-arg VERSION=$(VERSION) . 14 | rm -rf common/ 15 | 16 | tag: build 17 | docker tag $(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE) $(REGISTRY)/$(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE):$(VERSION) 18 | docker tag $(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE) $(REGISTRY)/$(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE):latest 19 | 20 | push: tag 21 | docker push $(REGISTRY)/$(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE):$(VERSION) 22 | docker push $(REGISTRY)/$(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE):latest 23 | -------------------------------------------------------------------------------- /docker/common/ecs_entry_point.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2016 Chris Smith 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -o errexit 18 | set -o xtrace 19 | 20 | result="$(python /opt/bin/ecs-get-port-mapping.py)" 21 | eval "$result" 22 | 23 | export SE_OPTS="-remoteHost http://$EC2_HOST:$PORT_TCP_5555 -id $NODE_ID" 24 | 25 | # execute default extry_point.sh file 26 | /opt/bin/entry_point.sh 27 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | AWS_ACCOUNT_ID=111122223333 2 | AWS_REGION=eu-central-1 3 | 4 | ECS_SELENIUM_STACK_NAME=ecs-selenium 5 | ECS_SELENIUM_VPC_ID= 6 | ECS_SELENIUM_KEY_PAIR_NAME=<########> 7 | ECS_SELENIUM_SUBNET_IDS= 8 | ECS_SELENIUM_HUB_INSTANCE_TYPE=c5.xlarge 9 | ECS_SELENIUM_NODE_INSTANCE_TYPE=c5.xlarge 10 | ECS_SELENIUM_ADMIN_CIDR= 11 | ECS_SELENIUM_DESIRED_FLEET_CAP=<#> 12 | ECS_SELENIUM_DESIRED_CHROME_NODES=<#> 13 | ECS_SELENIUM_DESIRED_FIREFOX_NODES=<#> 14 | ECS_SELENIUM_DOMAIN_NAME= 15 | ECS_SELENIUM_CHROME_IMAGE= 16 | ECS_SELENIUM_FIREFOX_IMAGE= 17 | ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE=ecs-node-firefox 18 | ECS_SELENIUM_FIREFOX_REPOSITORY_VERSION= 19 | ECS_SELENIUM_CHROME_REPOSITORY_IMAGE=ecs-node-chrome 20 | ECS_SELENIUM_CHROME_REPOSITORY_VERSION= 21 | ECS_HUB_IMAGE=selenium/hub: 22 | CREATE_PRIVATE_HOSTED_ZONE=true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 RetailMeNot 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docker/ecs-node-firefox/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG VERSION=latest 2 | FROM selenium/node-firefox:${VERSION} 3 | LABEL authors=RetailMeNot 4 | 5 | USER root 6 | 7 | #================================================ 8 | # Customize sources for apt-get 9 | #================================================ 10 | RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe\n" > /etc/apt/sources.list \ 11 | && echo "deb http://archive.ubuntu.com/ubuntu xenial-updates main universe\n" >> /etc/apt/sources.list \ 12 | && echo "deb http://security.ubuntu.com/ubuntu xenial-security main universe\n" >> /etc/apt/sources.list 13 | 14 | 15 | #======================== 16 | # Install python and pip 17 | #======================== 18 | RUN apt-get update -qqy \ 19 | && apt-get -qqy --no-install-recommends install \ 20 | python \ 21 | python-pip \ 22 | coreutils \ 23 | && rm -rf /var/lib/apt/lists/* 24 | 25 | 26 | #======================== 27 | # Install boto3 28 | #======================== 29 | 30 | RUN pip install boto3 requests boto 31 | COPY common/ecs-get-port-mapping.py /opt/bin/ecs-get-port-mapping.py 32 | COPY common/ecs_entry_point.sh /opt/bin/ecs_entry_point.sh 33 | 34 | RUN chown -R seluser:seluser /opt/bin/ \ 35 | && chmod +x /opt/bin/* 36 | 37 | USER seluser 38 | 39 | CMD ["/opt/bin/ecs_entry_point.sh"] 40 | -------------------------------------------------------------------------------- /docker/ecs-node-chrome/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG VERSION=latest 2 | FROM selenium/node-chrome:${VERSION} 3 | LABEL authors=RetailMeNot 4 | 5 | 6 | USER root 7 | 8 | #================================================ 9 | # Customize sources for apt-get 10 | #================================================ 11 | RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe\n" > /etc/apt/sources.list \ 12 | && echo "deb http://archive.ubuntu.com/ubuntu xenial-updates main universe\n" >> /etc/apt/sources.list \ 13 | && echo "deb http://security.ubuntu.com/ubuntu xenial-security main universe\n" >> /etc/apt/sources.list 14 | 15 | 16 | #======================== 17 | # Install python and pip 18 | #======================== 19 | RUN apt-get update -qqy \ 20 | && apt-get -qqy --no-install-recommends install \ 21 | python \ 22 | python-pip \ 23 | coreutils \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | 27 | #======================== 28 | # Install boto3 29 | #======================== 30 | 31 | RUN pip install boto3 requests boto 32 | COPY common/ecs-get-port-mapping.py /opt/bin/ecs-get-port-mapping.py 33 | COPY common/ecs_entry_point.sh /opt/bin/ecs_entry_point.sh 34 | 35 | RUN chown -R seluser:seluser /opt/bin/ \ 36 | && chmod +x /opt/bin/* 37 | 38 | USER seluser 39 | 40 | CMD ["/opt/bin/ecs_entry_point.sh"] 41 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include ./.env 2 | export $(shell sed 's/=.*//' ./.env) 3 | 4 | NODE_FIREFOX_IMAGE=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE):$(ECS_SELENIUM_FIREFOX_REPOSITORY_VERSION) 5 | 6 | NODE_CHROME_IMAGE=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com/$(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE):$(ECS_SELENIUM_CHROME_REPOSITORY_VERSION) 7 | 8 | STACK_PARAMETERS=--parameters ParameterKey=VpcId,ParameterValue="$(ECS_SELENIUM_VPC_ID)" \ 9 | ParameterKey=KeyName,ParameterValue="$(ECS_SELENIUM_KEY_PAIR_NAME)" \ 10 | ParameterKey=SubnetIds,ParameterValue=\"$(ECS_SELENIUM_SUBNET_IDS)\" \ 11 | ParameterKey=HubInstanceType,ParameterValue="$(ECS_SELENIUM_HUB_INSTANCE_TYPE)" \ 12 | ParameterKey=NodeInstanceType,ParameterValue="$(ECS_SELENIUM_NODE_INSTANCE_TYPE)" \ 13 | ParameterKey=AdminCIDR,ParameterValue="$(ECS_SELENIUM_ADMIN_CIDR)" \ 14 | ParameterKey=DesiredFleetCapacity,ParameterValue="$(ECS_SELENIUM_DESIRED_FLEET_CAP)" \ 15 | ParameterKey=DesiredChromeNodes,ParameterValue="$(ECS_SELENIUM_DESIRED_CHROME_NODES)" \ 16 | ParameterKey=DesiredFirefoxNodes,ParameterValue="$(ECS_SELENIUM_DESIRED_FIREFOX_NODES)" \ 17 | ParameterKey=DomainName,ParameterValue="$(ECS_SELENIUM_DOMAIN_NAME)" \ 18 | ParameterKey=NodeFirefoxImage,ParameterValue="$(NODE_FIREFOX_IMAGE)" \ 19 | ParameterKey=NodeChromeImage,ParameterValue="$(NODE_CHROME_IMAGE)" \ 20 | ParameterKey=HubImage,ParameterValue="$(ECS_HUB_IMAGE)" \ 21 | ParameterKey=CreatePrivateHostedZone,ParameterValue="$(CREATE_PRIVATE_HOSTED_ZONE)" 22 | 23 | 24 | create-stack: 25 | aws cloudformation create-stack \ 26 | --region $(AWS_REGION) \ 27 | --stack-name $(ECS_SELENIUM_STACK_NAME) --capabilities CAPABILITY_NAMED_IAM \ 28 | --template-body file://./cloudformation/ecs-selenium.cfn.yml \ 29 | $(STACK_PARAMETERS) 30 | 31 | update-stack: 32 | aws cloudformation update-stack \ 33 | --region $(AWS_REGION) \ 34 | --stack-name $(ECS_SELENIUM_STACK_NAME) --capabilities CAPABILITY_NAMED_IAM \ 35 | --template-body file://./cloudformation/ecs-selenium.cfn.yml \ 36 | $(STACK_PARAMETERS) 37 | 38 | create-changeset: 39 | aws cloudformation create-change-set \ 40 | --region $(AWS_REGION) \ 41 | --stack-name $(ECS_SELENIUM_STACK_NAME) --capabilities CAPABILITY_NAMED_IAM \ 42 | --template-body file://./cloudformation/ecs-selenium.cfn.yml \ 43 | --change-set-name change-set-1 \ 44 | $(STACK_PARAMETERS) 45 | 46 | view-changeset: 47 | aws cloudformation describe-change-set \ 48 | --region $(AWS_REGION) \ 49 | --stack-name $(ECS_SELENIUM_STACK_NAME) \ 50 | --change-set-name change-set-1 51 | 52 | execute-changeset: 53 | aws cloudformation execute-change-set \ 54 | --region $(AWS_REGION) \ 55 | --stack-name $(ECS_SELENIUM_STACK_NAME) \ 56 | --change-set-name change-set-1 57 | 58 | delete-stack: 59 | aws cloudformation delete-stack \ 60 | --region $(AWS_REGION) \ 61 | --stack-name $(ECS_SELENIUM_STACK_NAME) 62 | 63 | ecr-login: 64 | aws ecr --region $(AWS_REGION) get-login --no-include-email 65 | 66 | ecr-create-firefox-node: 67 | aws --region $(AWS_REGION) ecr create-repository --repository-name $(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE) 68 | 69 | ecr-create-chrome-node: 70 | aws --region $(AWS_REGION) ecr create-repository --repository-name $(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE) 71 | 72 | update-chrome-desired: # make count=<#> update-chrome-desired 73 | aws ecs update-service --cluster ecs-selenium-nodes \ 74 | --region $(AWS_REGION) \ 75 | --service $(ECS_SELENIUM_CHROME_REPOSITORY_IMAGE) --desired-count 15 76 | 77 | update-firefox-desired: # make count=<#> update-firefox-desired 78 | aws ecs update-service --cluster ecs-selenium-nodes \ 79 | --region $(AWS_REGION) \ 80 | --service $(ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE) --desired-count $(count) 81 | 82 | -------------------------------------------------------------------------------- /docker/common/ecs-get-port-mapping.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Chris Smith 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import boto3 17 | import requests 18 | 19 | 20 | def get_contents(filename): 21 | with open(filename) as f: 22 | return f.read() 23 | 24 | 25 | def get_ecs_introspection_url(resource): 26 | # 172.17.0.1 is the docker network bridge ip 27 | return 'http://172.17.0.1:51678/v1/' + resource 28 | 29 | 30 | def contains_key(d, key): 31 | return key in d and d[key] is not None 32 | 33 | 34 | def get_local_container_info(): 35 | # get the docker container id 36 | # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-agent-introspection.html 37 | docker_id = os.path.basename(get_contents("/proc/1/cpuset")).strip() 38 | 39 | if docker_id is None: 40 | raise Exception("Unable to find docker id") 41 | 42 | ecs_local_task = requests.get(get_ecs_introspection_url('tasks') + '?dockerid=' + docker_id).json() 43 | 44 | task_arn = ecs_local_task['Arn'] 45 | 46 | if task_arn is None: 47 | raise Exception("Unable to find task arn for container %s in ecs introspection api" % docker_id) 48 | 49 | ecs_local_container = None 50 | 51 | if contains_key(ecs_local_task, 'Containers'): 52 | for c in ecs_local_task['Containers']: 53 | container_docker_id = c.get('DockerId') or c.get('DockerID') 54 | if container_docker_id == docker_id: 55 | ecs_local_container = c 56 | 57 | if ecs_local_container is None: 58 | raise Exception("Unable to find container %s in ecs introspection api" % docker_id) 59 | 60 | return ecs_local_container['Name'], task_arn 61 | 62 | 63 | def main(): 64 | 65 | region = os.environ["AWS_REGION"] 66 | 67 | ecs_metadata = requests.get(get_ecs_introspection_url('metadata')).json() 68 | cluster = ecs_metadata['Cluster'] 69 | 70 | container_name, task_arn = get_local_container_info() 71 | 72 | # Get the container info from ECS. This will give us the port mappings 73 | ecs = boto3.client('ecs', region_name=region) 74 | response = ecs.describe_tasks( 75 | cluster=cluster, 76 | tasks=[ 77 | task_arn, 78 | ] 79 | ) 80 | 81 | task = None 82 | if contains_key(response, 'tasks'): 83 | for t in response['tasks']: 84 | if t['taskArn'] == task_arn: 85 | task = t 86 | 87 | if task is None: 88 | raise Exception("Unable to locate task %s" % task_arn) 89 | 90 | containerInstanceId = task["containerInstanceArn"].split("/")[-1] 91 | print("export NODE_ID='%s';" % task_arn.split("/")[-1]) 92 | response = ecs.describe_container_instances( 93 | cluster=os.environ["CLUSTER"], 94 | containerInstances=[ 95 | containerInstanceId, 96 | ] 97 | ) 98 | instance_id = response["containerInstances"][0]["ec2InstanceId"] 99 | 100 | ec2 = boto3.resource('ec2', region_name=region) 101 | instance = ec2.Instance(instance_id) 102 | print("export EC2_HOST=%s;" % instance.private_ip_address) 103 | 104 | container = None 105 | if contains_key(task, 'containers'): 106 | for c in task['containers']: 107 | if c['name'] == container_name: 108 | container = c 109 | 110 | if container is None: 111 | raise Exception("Unable to find ecs container %s" % container_name) 112 | 113 | if contains_key(container, 'networkBindings'): 114 | for b in container['networkBindings']: 115 | print("export PORT_%s_%d=%d;" % (b['protocol'].upper(), b['containerPort'], b['hostPort'])) 116 | 117 | 118 | if __name__ == '__main__': 119 | main() 120 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ecs-selenium 2 | ========= 3 | 4 | ## Introduction 5 | 6 | `ecs-selenium` includes CloudFormation scripts for a reference implementation of a [Selenium Grid](https://github.com/SeleniumHQ/selenium/wiki/Grid2) running on [Amazon's EC2 Container Service](https://aws.amazon.com/ecs/). 7 | 8 | ## Design 9 | 10 | The reference implementation consists of two ECS Clusters. A cluster for the hub, and another for the nodes. 11 | 12 | The reasoning behind this is that we wouldn't want the hub competing with the nodes for resources, nor would we want a SpotFleet pricing mismatch to take down the Hub. 13 | 14 | ### The Hub 15 | 16 | ![The Hub](docs/img/hub-cfn.png) 17 | 18 | The Hub consists of a single instance and an ECS Cluster with a selenium server running in _hub_ mode (`-role hub`) for each node type (Firefox, Chrome). An [Application Load Balancer](http://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html) sits in front of it. The ALB uses the hostname prefix to route requests to the correct hub. 19 | 20 | ### The Nodes 21 | 22 | ![The Nodes](docs/img/nodes-cfn.png) 23 | 24 | The nodes component of the stack consists of an ECS Cluster with a couple of services, task definitions (one of each for Firefox, and Chrome), and a [Spot Fleet](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html) to provide the underlying hardware. 25 | 26 | > *Note:* An AutoScaling Group is also provided as fall-back; in case Spot Fleet doesn't fit your needs. All you need to do is scale it up. 27 | 28 | ## Selenium 29 | 30 | ### Configuration 31 | 32 | Setup all environments, before you start 33 | 34 | ```bash 35 | # setup environment 36 | cp .env.sample .env 37 | ``` 38 | 39 | Replacing the parameters with appropriate values: 40 | 41 | ``` 42 | AWS_ACCOUNT_ID= 43 | AWS_REGION= 44 | 45 | ECS_SELENIUM_STACK_NAME=ecs-selenium 46 | ECS_SELENIUM_VPC_ID= 47 | ECS_SELENIUM_KEY_PAIR_NAME=<########> 48 | ECS_SELENIUM_SUBNET_IDS= 49 | ECS_SELENIUM_HUB_INSTANCE_TYPE=t3.medium 50 | ECS_SELENIUM_NODE_INSTANCE_TYPE=c5.xlarge 51 | ECS_SELENIUM_ADMIN_CIDR= 52 | ECS_SELENIUM_DESIRED_FLEET_CAP=<#> 53 | ECS_SELENIUM_DESIRED_CHROME_NODES=<#> 54 | ECS_SELENIUM_DESIRED_FIREFOX_NODES=<#> 55 | ECS_SELENIUM_DOMAIN_NAME= 56 | ECS_SELENIUM_CHROME_IMAGE= 57 | ECS_SELENIUM_FIREFOX_IMAGE= 58 | ECS_SELENIUM_FIREFOX_REPOSITORY_IMAGE=ecs-node-firefox 59 | ECS_SELENIUM_FIREFOX_REPOSITORY_VERSION= 60 | ECS_SELENIUM_CHROME_REPOSITORY_IMAGE=ecs-node-chrome 61 | ECS_SELENIUM_CHROME_REPOSITORY_VERSION= 62 | ``` 63 | 64 | ### Create the ECR Repositories 65 | 66 | Create your repositories before building the images. These commands only need to be run once. 67 | 68 | ``` 69 | # create the Firefox ECR repository 70 | make ecr-create-firefox-node 71 | 72 | # create the Chrome ECR repository 73 | make ecr-create-chrome-node 74 | ``` 75 | 76 | ### Build the Images 77 | 78 | You now need to build and push the node images to your docker registry. ECR is assumed by default. 79 | You can rebuild your images periodically to get newer browsers in your cluster. 80 | You can use latest in the ECS_SELENIUM_CHROME_REPOSITORY_VERSION and ECS_SELENIUM_FIREFOX_REPOSITORY_VERSION but it will make ecr messy, images are automatically tagged as latest everytime you push a new image so we reccomend using the version number on dockerhub for the image version. 81 | 82 | ```bash 83 | 84 | # login to ecr 85 | make ecr-login 86 | 87 | # build and push firefox 88 | cd docker/ecs-node-firefox 89 | make push 90 | 91 | # build and push chrome 92 | cd docker/ecs-node-chrome 93 | make push 94 | ``` 95 | 96 | ## Setup 97 | 98 | To get a Selenium Grid up and running with ECS, run the following command: 99 | 100 | ```bash 101 | make create-stack 102 | ``` 103 | 104 | To update Selenium Grid, run: 105 | 106 | ```bash 107 | make update-stack 108 | ``` 109 | 110 | ## Scaling 111 | 112 | ### Up 113 | 114 | The hub handles the scaling of the node services dynamically by comparing the hub's slot availability and the number of tasks that are queued in the corresponding hub. Scaling up is done every two minutes. 115 | 116 | Keep in mind that _only_ the number of desired tasks for the service are scaled, and it depends on the EC2 instance infrastructure being readily available. 117 | 118 | ### Down 119 | 120 | The template includes a parameter for setting up a time to scale down the number of nodes back to the default desired capabilities. By default, it is setup to scale down at 12AM UTC. You can update the `ScaleDownCron` parameter with values that work for your timezone. 121 | 122 | ### Manual Scaling 123 | 124 | Additionally you can manually scale the services via API calls: 125 | 126 | ```bash 127 | # chrome 128 | make count=<#> update-chrome-desired 129 | 130 | # firefox 131 | make count=<#> update-firefox-desired 132 | ``` 133 | 134 | ## Making small changes 135 | 136 | If your grid is already up and running you may want to consider using changesets to upgrade to make the small change instead of using update stack. 137 | 138 | By using a changeset you can make a small change to the cloudformation template and check what resources will be changed. 139 | 140 | This should be useful for updating images. 141 | 142 | ## Using the Grid 143 | 144 | The template creates a Route53 hosted zone that can be used inside your VPC. By default, it uses the `example.com` domain name. You can change that by updating the `DomainName` parameter. 145 | 146 | Inside your VPC, hosts can connect to the hubs using the following endpoints: 147 | 148 | * `firefox.example.com:4444` 149 | * `chrome.example.com:4444` 150 | 151 | The ALB uses the domain name prefix to redirect to the proper hub. Domains starting with `firefox.*` will hit the Firefox hub; domains starting with `chrome.*` will hit the Chrome one. 152 | 153 | ## Troubleshooting 154 | ### Errors that occur when creating the stack 155 | `Service arn:aws:ecs:us-west-1:1234567890:service/ecs-node-firefox did not stabilize.` 156 | 157 | We've seen this in situations where the cluster is unable to provision the desired amount of nodes. There will be a mismatch between the Desired tasks value and the Running tasks value in the ECS console (Elastic Container Service -> ecs-selenium-nodes -> Services (tab). 158 | 159 | It's usually due to hardware capacity. Here's what I would check: 160 | 161 | * Did you actually get any spot instances? (EC2 -> Spot Requests -> Capacity Column (in the table) 162 | If you didn't get any instances, your bid price may be too low for the region. You can update the MaxSpotBidPrice parameter to something that works for you. (The default is set pretty low, so depending on the region, you may be outbid by others.) 163 | 164 | * You can look at the SpotFleet's History tab, and it will tell you if it was unable to provision instances due to your bid being too low. 165 | 166 | * Do you have enough EC2 capacity for the number of nodes you desire? 167 | MemoryReservation is set to 1024 per node. So you'd need 1024 MB * Number of Desired Nodes for it to provision successfully. 168 | If you don't have enough instances for your nodes, you can increase DesiredFleetCapacity to match it, or you can decrease DesiredChromeNodes and DesiredFirefoxNodes to a number that fits within your provisioned instances. 169 | 170 | * The fact that you don't have any entries in the ecs-selenium-nodes stream seems to indicate that it never provisioned a single node. So I'd start by looking at the SpotFleet. 171 | 172 | ### Check Cloudwatch for errors reported to the ecs-selenium logstream 173 | -------------------------------------------------------------------------------- /cloudformation/ecs-selenium.cfn.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: "2010-09-09" 2 | Mappings: 3 | AMIRegionMap: 4 | # amzn-ami-2017.09.l-amazon-ecs-optimized, http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html 5 | ap-northeast-1: 6 | "64": "ami-a99d8ad5" 7 | ap-northeast-2: 8 | "64": "ami-9d56f9f3" 9 | ap-southeast-1: 10 | "64": "ami-846144f8" 11 | ap-southeast-2: 12 | "64": "ami-efda148d" 13 | ca-central-1: 14 | "64": "ami-897ff9ed" 15 | eu-central-1: 16 | "64": "ami-9fc39c74" 17 | eu-west-1: 18 | "64": "ami-2d386654" 19 | eu-west-2: 20 | "64": "ami-2218f945" 21 | eu-west-3: 22 | "64": "ami-250eb858" 23 | us-east-1: 24 | "64": "ami-aff65ad2" 25 | us-east-2: 26 | "64": "ami-64300001" 27 | us-west-1: 28 | "64": "ami-69677709" 29 | us-west-2: 30 | "64": "ami-40ddb938" 31 | Parameters: 32 | VpcId: 33 | Type: "AWS::EC2::VPC::Id" 34 | Description: VpcId of your existing Virtual Private Cloud (VPC) 35 | ConstraintDescription: must be the VPC Id of an existing Virtual Private Cloud. 36 | SubnetIds: 37 | Type: List 38 | Description: A comma-delimitted list of SubnetIds in your Virtual Private Cloud (VPC) 39 | HubInstanceType: 40 | Description: Hub instance type 41 | Type: String 42 | Default: c5.xlarge 43 | NodeInstanceType: 44 | Description: Node instance type 45 | Type: String 46 | Default: c5.xlarge 47 | KeyName: 48 | Description: The EC2 Key Pair to allow SSH access to the instances 49 | Type: "AWS::EC2::KeyPair::KeyName" 50 | ConstraintDescription: Must be the name of an existing EC2 KeyPair. 51 | AdminCIDR: 52 | Type: String 53 | Default: 10.0.0.0/8 54 | Description: CIDR with SSH access for management 55 | MaxSpotBidPrice: 56 | Type: Number 57 | Default: 0.17 58 | Description: Maximum price to bid for Spot Instances 59 | DesiredFleetCapacity: 60 | Type: Number 61 | Default: 1 62 | Description: Desired number of Spot Instances to request 63 | DesiredChromeNodes: 64 | Type: Number 65 | Default: 1 66 | Description: Desired number of Chrome nodes 67 | MaxChromeNodes: 68 | Type: Number 69 | Default: 100 70 | Description: Maximum number of Chrome nodes 71 | DesiredFirefoxNodes: 72 | Type: Number 73 | Default: 1 74 | Description: Desired number of Firefox nodes 75 | MaxFirefoxNodes: 76 | Type: Number 77 | Default: 100 78 | Description: Maximum number of Firefox nodes 79 | NodeChromeImage: 80 | Type: String 81 | Description: URI to the ecs-node-chrome image. 82 | NodeFirefoxImage: 83 | Type: String 84 | Description: URI to the ecs-node-firefox image. 85 | DomainName: 86 | Type: String 87 | Default: example.com 88 | Description: Domain for VPC Hosted Zone. 89 | ScaleDownCron: 90 | Type: String 91 | Default: 0 0 * * * 92 | Description: Cron expression for scaling down the nodes to the default capacity (DesiredChromeNodes, DesiredFirefoxNodes). 93 | DockerVolumeSize: 94 | Type: Number 95 | Default: 22 96 | Description: Size (in GiB) of the EBS volume that Docker users for image and metadata storage. # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-ami-storage-config.html 97 | DockerDeviceName: 98 | Type: String 99 | Default: "/dev/xvdcz" 100 | Description: Name of volume that Docker uses for image and metadata storage. 101 | HubImage: 102 | Type: String 103 | Default: "selenium/hub:3.141.58" 104 | Description: Allows the version number for the image of the selenium hub to be changed. 105 | CreatePrivateHostedZone: 106 | Type: String 107 | Default: "true" 108 | Description: Whether to host Selenium grid publicly or privately. True for private. 109 | AllowedValues: [true, false] 110 | 111 | Conditions: 112 | ShouldCreatePrivateHostedZone: 113 | !Equals [!Ref CreatePrivateHostedZone, 'true'] 114 | 115 | Resources: 116 | 117 | ################################### 118 | # DNS # 119 | ################################### 120 | SeleniumHostedZone: 121 | Type: "AWS::Route53::HostedZone" 122 | Properties: 123 | HostedZoneConfig: 124 | Comment: "selenium-ecs hosted zone" 125 | Name: !Ref DomainName 126 | VPCs: !If [ShouldCreatePrivateHostedZone, [{VPCId: !Ref VpcId, VPCRegion: !Ref "AWS::Region"}], !Ref 'AWS::NoValue'] 127 | 128 | ChromeRecord: 129 | Type: "AWS::Route53::RecordSet" 130 | Properties: 131 | HostedZoneId: !Ref SeleniumHostedZone 132 | Comment: Record for Chrome Hub. 133 | Name: !Sub chrome.${DomainName}. 134 | Type: A 135 | AliasTarget: 136 | DNSName: !GetAtt HubLoadBalancer.DNSName 137 | HostedZoneId: !GetAtt HubLoadBalancer.CanonicalHostedZoneID 138 | 139 | FirefoxRecord: 140 | Type: "AWS::Route53::RecordSet" 141 | Properties: 142 | HostedZoneId: !Ref SeleniumHostedZone 143 | Comment: Record for Firefox Hub. 144 | Name: !Sub firefox.${DomainName}. 145 | Type: A 146 | AliasTarget: 147 | DNSName: !GetAtt HubLoadBalancer.DNSName 148 | HostedZoneId: !GetAtt HubLoadBalancer.CanonicalHostedZoneID 149 | 150 | ################################### 151 | # HUB # 152 | ################################### 153 | 154 | HubRole: 155 | Type: "AWS::IAM::Role" 156 | Properties: 157 | AssumeRolePolicyDocument: 158 | Version: "2012-10-17" 159 | Statement: 160 | - Action: "sts:AssumeRole" 161 | Effect: Allow 162 | Principal: 163 | Service: ec2.amazonaws.com 164 | ManagedPolicyArns: 165 | - "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" 166 | Policies: 167 | - PolicyName: WriteCloudWatchMetrics 168 | PolicyDocument: 169 | Version: "2012-10-17" 170 | Statement: 171 | - Action: 172 | - "cloudwatch:PutMetricData" 173 | Effect: "Allow" 174 | Resource: "*" 175 | - PolicyName: ServiceScaler 176 | PolicyDocument: 177 | Version: "2012-10-17" 178 | Statement: 179 | - Action: 180 | - "ecs:DescribeServices" 181 | - "ecs:UpdateService" 182 | Effect: "Allow" 183 | Resource: "*" 184 | 185 | HubInstanceProfile: 186 | Type: "AWS::IAM::InstanceProfile" 187 | Properties: 188 | Roles: 189 | - !Ref HubRole 190 | 191 | HubSecurityGroup: 192 | Type: "AWS::EC2::SecurityGroup" 193 | Properties: 194 | GroupName: ecs-selenium-hub 195 | GroupDescription: ECS Selenium Hub Security Group 196 | VpcId: !Ref VpcId 197 | SecurityGroupEgress: 198 | - IpProtocol: -1 199 | CidrIp: 0.0.0.0/0 200 | SecurityGroupIngress: 201 | - IpProtocol: tcp 202 | FromPort: 22 203 | ToPort: 22 204 | CidrIp: !Ref AdminCIDR 205 | - IpProtocol: tcp 206 | FromPort: 4444 207 | ToPort: 4444 208 | SourceSecurityGroupId: !GetAtt HubLoadBalancerSecurityGroup.GroupId 209 | - IpProtocol: tcp 210 | FromPort: 4455 211 | ToPort: 4455 212 | SourceSecurityGroupId: !GetAtt HubLoadBalancerSecurityGroup.GroupId 213 | 214 | HubAutoScalingGroup: 215 | Type: "AWS::AutoScaling::AutoScalingGroup" 216 | Properties: 217 | DesiredCapacity: 1 218 | LaunchConfigurationName: !Ref HubLaunchConfig 219 | MaxSize: 1 220 | MinSize: 1 221 | VPCZoneIdentifier: !Ref SubnetIds 222 | TargetGroupARNs: 223 | - !Ref ChromeTargetGroup 224 | - !Ref FirefoxTargetGroup 225 | Tags: 226 | - Key: Name 227 | Value: "ecs-selenium-hub" 228 | PropagateAtLaunch: true 229 | 230 | HubLaunchConfig: 231 | Type: "AWS::AutoScaling::LaunchConfiguration" 232 | Metadata: 233 | "AWS::CloudFormation::Init": 234 | config: 235 | files: 236 | '/usr/local/bin/scaler': 237 | mode: '000744' 238 | owner: root 239 | group: root 240 | content: !Sub | 241 | #!/usr/bin/env python 242 | 243 | import os 244 | import logging 245 | import argparse 246 | import boto3 247 | import requests 248 | 249 | logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO) 250 | logging.getLogger("boto3").setLevel(logging.WARNING) 251 | logging.getLogger("botocore").setLevel(logging.WARNING) 252 | logging.getLogger("urllib3").setLevel(logging.WARNING) 253 | logging.getLogger("requests").setLevel(logging.WARNING) 254 | logger = logging.getLogger(__name__) 255 | 256 | DEFAULT_CAPACITY = { 257 | "ecs-node-chrome": ${DesiredChromeNodes}, 258 | "ecs-node-firefox": ${DesiredFirefoxNodes} 259 | } 260 | 261 | MAX_CAPACITY = { 262 | "ecs-node-chrome": ${MaxChromeNodes}, 263 | "ecs-node-firefox": ${MaxFirefoxNodes} 264 | } 265 | 266 | ecs = boto3.client("ecs", region_name="${AWS::Region}") 267 | 268 | 269 | def get_unsatisfied_demand(host, port): 270 | data = requests.get("http://%s:%s/grid/api/hub/" % (host, port), timeout=20.0).json() 271 | pending = data["newSessionRequestCount"] 272 | return pending 273 | 274 | 275 | def get_running_count(cluster, service): 276 | for svc in ecs.describe_services(cluster=cluster, services=[service])["services"]: 277 | if svc["serviceName"] == service: 278 | return svc["runningCount"] 279 | 280 | 281 | def get_pending_count(cluster, service): 282 | for svc in ecs.describe_services(cluster=cluster, services=[service])["services"]: 283 | if svc["serviceName"] == service: 284 | return svc["pendingCount"] 285 | 286 | 287 | def update_service(cluster, service, desired): 288 | logger.info("Updating desiredCount for service '%s' to %d...", service, desired) 289 | ecs.update_service( 290 | cluster=cluster, 291 | service=service, 292 | desiredCount=desired, 293 | ) 294 | 295 | 296 | def main(): 297 | parser = argparse.ArgumentParser(description="Check hub demand and update service.") 298 | parser.add_argument("--cluster", required=True, help="cluster name") 299 | parser.add_argument("--direction", default="up", choices=["up", "down"]) 300 | parser.add_argument("--service", choices=["ecs-node-chrome", "ecs-node-firefox"]) 301 | parser.add_argument("--host", default="localhost", help="hostname for the server to be polled") 302 | parser.add_argument("--port", help="port for the server to be polled") 303 | args = parser.parse_args() 304 | 305 | u_demand = get_unsatisfied_demand(args.host, args.port) 306 | running = get_running_count(args.cluster, args.service) 307 | pending = get_pending_count(args.cluster, args.service) 308 | 309 | if args.direction == "up" and u_demand > 0: 310 | desired = max(u_demand + running, MAX_CAPACITY[args.service]) 311 | if pending == 0: # only update when there are no pending tasks 312 | update_service(args.cluster, args.service, desired) 313 | else: 314 | logger.info("There are pending tasks for service '%s'. Skipping...", service) 315 | elif args.direction == "down" and u_demand == 0 and running > DEFAULT_CAPACITY[args.service]: 316 | update_service(args.cluster, args.service, DEFAULT_CAPACITY[args.service]) 317 | 318 | 319 | if __name__ == "__main__": 320 | main() 321 | 322 | '/etc/cron.d/firefox-scaler': 323 | mode: '000644' 324 | owner: root 325 | group: root 326 | content: !Sub | 327 | */2 * * * * root /usr/local/bin/scaler --cluster ecs-selenium-nodes --service ecs-node-firefox --port 4455 >> /var/log/ecs-selenium/firefox.log 2>&1 328 | ${ScaleDownCron} root /usr/local/bin/scaler --cluster ecs-selenium-nodes --service ecs-node-firefox --port 4455 --direction down >> /var/log/ecs-selenium/firefox.log 2>&1 329 | '/etc/cron.d/chrome-scaler': 330 | mode: '000644' 331 | owner: root 332 | group: root 333 | content: !Sub | 334 | */2 * * * * root /usr/local/bin/scaler --cluster ecs-selenium-nodes --service ecs-node-chrome --port 4444 >> /var/log/ecs-selenium/chrome.log 2>&1 335 | ${ScaleDownCron} root /usr/local/bin/scaler --cluster ecs-selenium-nodes --service ecs-node-chrome --port 4444 --direction down >> /var/log/ecs-selenium/chrome.log 2>&1 336 | Properties: 337 | AssociatePublicIpAddress: true 338 | IamInstanceProfile: !Ref HubInstanceProfile 339 | ImageId: !FindInMap [AMIRegionMap, !Ref "AWS::Region", "64"] 340 | InstanceMonitoring: true 341 | InstanceType: !Ref HubInstanceType 342 | KeyName: !Ref KeyName 343 | SecurityGroups: 344 | - !Ref HubSecurityGroup 345 | UserData: 346 | "Fn::Base64": 347 | "Fn::Sub": | 348 | #!/bin/bash -e 349 | yum update -y ecs-init 350 | echo "ECS_CLUSTER=${HubCluster}" > /etc/ecs/ecs.config 351 | yum install -y aws-cfn-bootstrap python27-pip 352 | pip install boto3 requests 353 | mkdir -p /var/log/ecs-selenium/ 354 | /opt/aws/bin/cfn-init --stack ${AWS::StackName} --region ${AWS::Region} --resource HubLaunchConfig --verbose 355 | 356 | HubLoadBalancerSecurityGroup: 357 | Type: "AWS::EC2::SecurityGroup" 358 | Properties: 359 | GroupName: ecs-selenium-hub-lb1 360 | GroupDescription: ECS Selenium Hub Load Balancer Security Group 361 | VpcId: !Ref VpcId 362 | SecurityGroupEgress: 363 | - IpProtocol: -1 364 | CidrIp: 0.0.0.0/0 365 | SecurityGroupIngress: 366 | - IpProtocol: tcp 367 | FromPort: 4444 368 | ToPort: 4444 369 | CidrIp: !Ref AdminCIDR 370 | - IpProtocol: tcp 371 | FromPort: 4444 372 | ToPort: 4444 373 | SourceSecurityGroupId: !GetAtt NodeSecurityGroup.GroupId 374 | 375 | HubLoadBalancer: 376 | Type: "AWS::ElasticLoadBalancingV2::LoadBalancer" 377 | Properties: 378 | Scheme: internal 379 | SecurityGroups: 380 | - !Ref HubLoadBalancerSecurityGroup 381 | Subnets: !Ref SubnetIds 382 | 383 | ChromeTargetGroup: 384 | Type: "AWS::ElasticLoadBalancingV2::TargetGroup" 385 | Properties: 386 | HealthCheckIntervalSeconds: 60 387 | HealthCheckPath: / 388 | HealthCheckPort: 4444 389 | HealthCheckProtocol: HTTP 390 | HealthCheckTimeoutSeconds: 30 391 | Name: ecs-selenium-hub-chrome 392 | Port: 4444 393 | Protocol: HTTP 394 | UnhealthyThresholdCount: 2 395 | VpcId: !Ref VpcId 396 | 397 | FirefoxTargetGroup: 398 | Type: "AWS::ElasticLoadBalancingV2::TargetGroup" 399 | Properties: 400 | HealthCheckIntervalSeconds: 60 401 | HealthCheckPath: / 402 | HealthCheckPort: 4455 403 | HealthCheckProtocol: HTTP 404 | HealthCheckTimeoutSeconds: 30 405 | Name: ecs-selenium-hub-firefox 406 | Port: 4455 407 | Protocol: HTTP 408 | UnhealthyThresholdCount: 2 409 | VpcId: !Ref VpcId 410 | 411 | HubListener: 412 | Type: "AWS::ElasticLoadBalancingV2::Listener" 413 | Properties: 414 | LoadBalancerArn: !Ref HubLoadBalancer 415 | DefaultActions: 416 | - TargetGroupArn: !Ref ChromeTargetGroup 417 | Type: forward 418 | Port: 4444 419 | Protocol: HTTP 420 | 421 | ChromeDomainRule: 422 | Type: "AWS::ElasticLoadBalancingV2::ListenerRule" 423 | Properties: 424 | Actions: 425 | - Type: forward 426 | TargetGroupArn: !Ref ChromeTargetGroup 427 | Conditions: 428 | - Field: host-header 429 | Values: 430 | - !Sub chrome.${DomainName} 431 | ListenerArn: !Ref HubListener 432 | Priority: 1 433 | 434 | FirefoxDomainRule: 435 | Type: "AWS::ElasticLoadBalancingV2::ListenerRule" 436 | Properties: 437 | Actions: 438 | - Type: forward 439 | TargetGroupArn: !Ref FirefoxTargetGroup 440 | Conditions: 441 | - Field: host-header 442 | Values: 443 | - !Sub firefox.${DomainName} 444 | ListenerArn: !Ref HubListener 445 | Priority: 2 446 | 447 | HubCluster: 448 | Type: "AWS::ECS::Cluster" 449 | Properties: 450 | ClusterName: ecs-selenium-hub 451 | 452 | ChromeHubService: 453 | Type: "AWS::ECS::Service" 454 | Properties: 455 | Cluster: !Ref HubCluster 456 | DeploymentConfiguration: 457 | MaximumPercent: 200 458 | MinimumHealthyPercent: 50 459 | DesiredCount: 1 460 | ServiceName: ecs-selenium-hub-chrome 461 | TaskDefinition: !Ref ChromeHubTask 462 | 463 | ChromeHubTask: 464 | Type: "AWS::ECS::TaskDefinition" 465 | Properties: 466 | Family: ecs-selenium-hub-chrome 467 | NetworkMode: bridge 468 | ContainerDefinitions: 469 | - Environment: 470 | - Name: SE_OPTS 471 | Value: -jettyThreads 1024 472 | Essential: True 473 | Image: !Ref HubImage 474 | LogConfiguration: 475 | LogDriver: awslogs 476 | Options: 477 | awslogs-group: !Ref HubLogGroup 478 | awslogs-region: !Ref "AWS::Region" 479 | awslogs-stream-prefix: ecs-hub-chrome 480 | MemoryReservation: 2048 481 | Name: ecs-selenium-hub-chrome 482 | PortMappings: 483 | - ContainerPort: 4444 484 | HostPort: 4444 485 | Protocol: tcp 486 | 487 | FirefoxHubService: 488 | Type: "AWS::ECS::Service" 489 | Properties: 490 | Cluster: !Ref HubCluster 491 | DeploymentConfiguration: 492 | MaximumPercent: 200 493 | MinimumHealthyPercent: 50 494 | DesiredCount: 1 495 | ServiceName: ecs-selenium-hub-firefox 496 | TaskDefinition: !Ref FirefoxHubTask 497 | 498 | FirefoxHubTask: 499 | Type: "AWS::ECS::TaskDefinition" 500 | Properties: 501 | Family: ecs-selenium-hub-firefox 502 | NetworkMode: bridge 503 | ContainerDefinitions: 504 | - Environment: 505 | - Name: SE_OPTS 506 | Value: -jettyThreads 1024 507 | Essential: True 508 | Image: !Ref HubImage 509 | LogConfiguration: 510 | LogDriver: awslogs 511 | Options: 512 | awslogs-group: !Ref HubLogGroup 513 | awslogs-region: !Ref "AWS::Region" 514 | awslogs-stream-prefix: ecs-hub-firefox 515 | MemoryReservation: 2048 516 | Name: ecs-selenium-hub-firefox 517 | PortMappings: 518 | - ContainerPort: 4444 519 | HostPort: 4455 520 | Protocol: tcp 521 | 522 | HubLogGroup: 523 | Type: "AWS::Logs::LogGroup" 524 | Properties: 525 | LogGroupName: ecs-selenium-hub 526 | RetentionInDays: 30 527 | 528 | 529 | ################################### 530 | # NODES # 531 | ################################### 532 | SpotFleetRole: 533 | Type: "AWS::IAM::Role" 534 | Properties: 535 | AssumeRolePolicyDocument: 536 | Version: "2012-10-17" 537 | Statement: 538 | - Action: sts:AssumeRole 539 | Effect: Allow 540 | Principal: 541 | Service: spotfleet.amazonaws.com 542 | ManagedPolicyArns: 543 | - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole 544 | 545 | 546 | NodeRole: 547 | Type: "AWS::IAM::Role" 548 | Properties: 549 | AssumeRolePolicyDocument: 550 | Version: "2012-10-17" 551 | Statement: 552 | - Action: "sts:AssumeRole" 553 | Effect: Allow 554 | Principal: 555 | Service: ec2.amazonaws.com 556 | ManagedPolicyArns: 557 | - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role 558 | Policies: 559 | - PolicyName: WriteCloudWatchMetrics 560 | PolicyDocument: 561 | Version: "2012-10-17" 562 | Statement: 563 | - Action: 564 | - "cloudwatch:PutMetricData" 565 | Effect: "Allow" 566 | Resource: "*" 567 | - PolicyName: EC2Permissions 568 | PolicyDocument: 569 | Version: "2012-10-17" 570 | Statement: 571 | - Action: 572 | - ec2:CreateTags 573 | - ec2:DescribeTags 574 | Effect: Allow 575 | Resource: "*" 576 | 577 | NodeInstanceProfile: 578 | Type: "AWS::IAM::InstanceProfile" 579 | Properties: 580 | Roles: 581 | - !Ref NodeRole 582 | 583 | NodeTaskRole: 584 | Type: "AWS::IAM::Role" 585 | Properties: 586 | AssumeRolePolicyDocument: 587 | Version: "2012-10-17" 588 | Statement: 589 | - Action: sts:AssumeRole 590 | Effect: Allow 591 | Principal: 592 | Service: ecs-tasks.amazonaws.com 593 | Policies: 594 | - PolicyName: Introspection 595 | PolicyDocument: 596 | Version: "2012-10-17" 597 | Statement: 598 | - Action: 599 | - ecs:Describe* 600 | - ec2:DescribeInstances 601 | Effect: Allow 602 | Resource: "*" 603 | 604 | NodeSecurityGroup: 605 | Type: "AWS::EC2::SecurityGroup" 606 | Properties: 607 | GroupName: ecs-selenium-node 608 | GroupDescription: ECS Selenium Node Security Group 609 | VpcId: !Ref VpcId 610 | SecurityGroupEgress: 611 | - IpProtocol: -1 612 | CidrIp: 0.0.0.0/0 613 | SecurityGroupIngress: 614 | - IpProtocol: tcp 615 | FromPort: 22 616 | ToPort: 22 617 | CidrIp: !Ref AdminCIDR 618 | 619 | FromHub: 620 | Type: "AWS::EC2::SecurityGroupIngress" 621 | Properties: 622 | GroupId: !GetAtt NodeSecurityGroup.GroupId 623 | IpProtocol: tcp 624 | SourceSecurityGroupId: !GetAtt HubSecurityGroup.GroupId 625 | FromPort: 32768 626 | ToPort: 65535 627 | 628 | SpotFleet: 629 | Type: "AWS::EC2::SpotFleet" 630 | Properties: 631 | SpotFleetRequestConfigData: 632 | SpotPrice: !Ref MaxSpotBidPrice 633 | TargetCapacity: !Ref DesiredFleetCapacity 634 | IamFleetRole: !GetAtt SpotFleetRole.Arn 635 | LaunchSpecifications: 636 | - IamInstanceProfile: 637 | Arn: !GetAtt NodeInstanceProfile.Arn 638 | ImageId: !FindInMap [AMIRegionMap, !Ref "AWS::Region", "64"] 639 | InstanceType: !Ref NodeInstanceType 640 | KeyName: !Ref KeyName 641 | BlockDeviceMappings: 642 | - DeviceName: !Ref DockerDeviceName 643 | Ebs: 644 | VolumeSize: !Ref DockerVolumeSize 645 | Monitoring: 646 | Enabled: 'true' 647 | SecurityGroups: 648 | - GroupId: !Ref NodeSecurityGroup 649 | SubnetId: !Join [ ", ", !Ref SubnetIds ] 650 | UserData: 651 | "Fn::Base64": 652 | "Fn::Sub": | 653 | #!/bin/bash -e 654 | yum update -y ecs-init 655 | yum install -y aws-cli 656 | echo "ECS_CLUSTER=${NodeCluster}" > /etc/ecs/ecs.config 657 | echo "ECS_ENABLE_TASK_IAM_ROLE=true" >> /etc/ecs/ecs.config 658 | # SpotFleet doesn't yet support tags via CloudFormation 659 | aws ec2 create-tags --region ${AWS::Region} --resources $(curl -s http://169.254.169.254/latest/meta-data/instance-id) --tags Key=Name,Value=ecs-selenium-node 660 | sysctl -w net.ipv4.conf.all.route_localnet=1 661 | iptables --insert FORWARD 1 --in-interface docker+ --destination 169.254.169.254/32 --jump DROP 662 | iptables -t nat -A PREROUTING -p tcp -d 169.254.170.2 --dport 80 -j DNAT --to-destination 127.0.0.1:51679 663 | iptables -t nat -A OUTPUT -d 169.254.170.2 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 51679 664 | service docker start 665 | 666 | NodeAutoScalingGroup: 667 | Type: "AWS::AutoScaling::AutoScalingGroup" 668 | Properties: 669 | DesiredCapacity: 0 670 | LaunchConfigurationName: !Ref NodeLaunchConfig 671 | MaxSize: 10 672 | MinSize: 0 673 | VPCZoneIdentifier: !Ref SubnetIds 674 | Tags: 675 | - Key: Name 676 | Value: "ecs-selenium-node" 677 | PropagateAtLaunch: true 678 | 679 | NodeLaunchConfig: 680 | Type: "AWS::AutoScaling::LaunchConfiguration" 681 | Properties: 682 | AssociatePublicIpAddress: false 683 | BlockDeviceMappings: 684 | - DeviceName: !Ref DockerDeviceName 685 | Ebs: 686 | VolumeSize: !Ref DockerVolumeSize 687 | IamInstanceProfile: !Ref NodeInstanceProfile 688 | ImageId: !FindInMap [AMIRegionMap, !Ref "AWS::Region", "64"] 689 | InstanceMonitoring: true 690 | InstanceType: !Ref NodeInstanceType 691 | KeyName: !Ref KeyName 692 | SecurityGroups: 693 | - !Ref NodeSecurityGroup 694 | UserData: 695 | "Fn::Base64": 696 | "Fn::Sub": | 697 | #!/bin/bash -e 698 | yum update -y ecs-init 699 | echo "ECS_CLUSTER=${NodeCluster}" > /etc/ecs/ecs.config 700 | echo "ECS_ENABLE_TASK_IAM_ROLE=true" >> /etc/ecs/ecs.config 701 | sysctl -w net.ipv4.conf.all.route_localnet=1 702 | iptables --insert FORWARD 1 --in-interface docker+ --destination 169.254.169.254/32 --jump DROP 703 | iptables -t nat -A PREROUTING -p tcp -d 169.254.170.2 --dport 80 -j DNAT --to-destination 127.0.0.1:51679 704 | iptables -t nat -A OUTPUT -d 169.254.170.2 -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 51679 705 | service docker start 706 | 707 | NodeCluster: 708 | Type: "AWS::ECS::Cluster" 709 | Properties: 710 | ClusterName: ecs-selenium-nodes 711 | 712 | NodeChromeTask: 713 | Type: "AWS::ECS::TaskDefinition" 714 | Properties: 715 | TaskRoleArn: !GetAtt NodeTaskRole.Arn 716 | Family: ecs-node-chrome 717 | NetworkMode: bridge 718 | Volumes: 719 | - Host: 720 | SourcePath: "/dev/shm" 721 | Name: "shm" 722 | ContainerDefinitions: 723 | - Environment: 724 | - Name: HUB_PORT_4444_TCP_ADDR 725 | Value: !Ref ChromeRecord 726 | - Name: HUB_PORT_4444_TCP_PORT 727 | Value: 4444 728 | - Name: AWS_REGION 729 | Value: !Ref "AWS::Region" 730 | - Name: CLUSTER 731 | Value: !Ref NodeCluster 732 | Essential: True 733 | Image: !Ref NodeChromeImage 734 | LogConfiguration: 735 | LogDriver: awslogs 736 | Options: 737 | awslogs-group: !Ref NodeLogGroup 738 | awslogs-region: !Ref "AWS::Region" 739 | awslogs-stream-prefix: ecs-node-chrome 740 | MemoryReservation: 1024 741 | Name: ecs-node-chrome 742 | PortMappings: 743 | - ContainerPort: 5555 744 | Protocol: tcp 745 | MountPoints: 746 | - ContainerPath: /dev/shm 747 | SourceVolume: shm 748 | ReadOnly: False 749 | 750 | NodeChromeService: 751 | Type: "AWS::ECS::Service" 752 | Properties: 753 | Cluster: !Ref NodeCluster 754 | DeploymentConfiguration: 755 | MaximumPercent: 200 756 | MinimumHealthyPercent: 50 757 | DesiredCount: !Ref DesiredChromeNodes 758 | ServiceName: ecs-node-chrome 759 | TaskDefinition: !Ref NodeChromeTask 760 | 761 | NodeFirefoxTask: 762 | Type: "AWS::ECS::TaskDefinition" 763 | Properties: 764 | TaskRoleArn: !GetAtt NodeTaskRole.Arn 765 | Family: ecs-node-firefox 766 | NetworkMode: bridge 767 | Volumes: 768 | - Host: 769 | SourcePath: "/dev/shm" 770 | Name: "shm" 771 | ContainerDefinitions: 772 | - Environment: 773 | - Name: HUB_PORT_4444_TCP_ADDR 774 | Value: !Ref FirefoxRecord 775 | - Name: HUB_PORT_4444_TCP_PORT 776 | Value: 4444 777 | - Name: AWS_REGION 778 | Value: !Ref "AWS::Region" 779 | - Name: CLUSTER 780 | Value: !Ref NodeCluster 781 | Essential: True 782 | Image: !Ref NodeFirefoxImage 783 | LogConfiguration: 784 | LogDriver: awslogs 785 | Options: 786 | awslogs-group: !Ref NodeLogGroup 787 | awslogs-region: !Ref "AWS::Region" 788 | awslogs-stream-prefix: ecs-node-firefox 789 | MemoryReservation: 1024 790 | Name: ecs-node-firefox 791 | PortMappings: 792 | - ContainerPort: 5555 793 | Protocol: tcp 794 | MountPoints: 795 | - ContainerPath: /dev/shm 796 | SourceVolume: shm 797 | ReadOnly: False 798 | 799 | NodeFirefoxService: 800 | Type: "AWS::ECS::Service" 801 | Properties: 802 | Cluster: !Ref NodeCluster 803 | DeploymentConfiguration: 804 | MaximumPercent: 200 805 | MinimumHealthyPercent: 50 806 | DesiredCount: !Ref DesiredFirefoxNodes 807 | ServiceName: ecs-node-firefox 808 | TaskDefinition: !Ref NodeFirefoxTask 809 | 810 | NodeLogGroup: 811 | Type: "AWS::Logs::LogGroup" 812 | Properties: 813 | LogGroupName: ecs-selenium-nodes 814 | RetentionInDays: 30 815 | 816 | Outputs: 817 | FirefoxHubURI: 818 | Description: URI for the Firefox Hub 819 | Value: !Sub "http://${FirefoxRecord}:4444" 820 | Export: 821 | Name: "FirefoxHubURI" 822 | 823 | ChromeHubURI: 824 | Description: URI for the Chrome Hub 825 | Value: !Sub "http://${ChromeRecord}:4444" 826 | Export: 827 | Name: "ChromeHubURI" 828 | --------------------------------------------------------------------------------