├── IaC ├── terraform │ ├── provider.tf │ ├── selenium-grid-k8s-resources │ │ ├── outputs.tf │ │ ├── selenium-grid-k8s-namespace.tf │ │ ├── selenium-grid-k8s-services.tf │ │ └── selenium-grid-k8s-deployment.tf │ ├── account.example.json │ ├── selenium-firewall-rules.tf │ ├── outputs.tf │ ├── variables.tf │ ├── selenium-grid-k8s-cluster.tf │ ├── selenoid-web.tf │ └── selenoid-android.tf └── ansible │ ├── selenoid-web.yml │ ├── selenoid-android.yml │ ├── docker-compose.yml │ └── docker.yml ├── .gitignore ├── dockerBasedTools ├── selenoid-android │ ├── browsers.json │ └── docker-compose.yml ├── selenoid-web │ ├── browsers.json │ └── docker-compose.yml └── selenium-grid │ └── docker-compose.yml ├── uiTestsJS ├── appium-mobile │ ├── package.json │ ├── android-create-session.test.js │ └── config.js └── selenium-web │ ├── package.json │ └── google_search_test.js ├── cloudProviders └── gcp │ ├── K8s │ └── selenium │ │ ├── selenium-hub-svc.yaml │ │ ├── selenium-hub-deployment.yaml │ │ └── selenium-node-chrome-deployment.yaml │ └── .gitlab-ci.example.gcp.selenoid.web.android.yml ├── ci └── gitlab-ci │ ├── docker-compose-gitlab-macos.yml │ └── .gitlab-ci.example.local.running.yml └── README.md /IaC/terraform/provider.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | version = "3.4.0" 3 | credentials = "${file("account.json")}" 4 | project = "${var.project_name}" 5 | region = "europe-west4-a" 6 | } -------------------------------------------------------------------------------- /IaC/terraform/selenium-grid-k8s-resources/outputs.tf: -------------------------------------------------------------------------------- 1 | output "Selenium-Grid" { 2 | value = "http://${kubernetes_service.selenium-hub-svc-external.load_balancer_ingress[0].ip}:4444/grid/console" 3 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .idea/ 3 | **/node_modules/ 4 | .DS_Store 5 | 6 | ci/gitlab-ci/local-run/ 7 | local-run/ 8 | 9 | *.tfstate 10 | *.tfstate.*.backup 11 | *.tfstate.backup 12 | *.tfvars 13 | .terraform/ 14 | **/account.json -------------------------------------------------------------------------------- /IaC/terraform/selenium-grid-k8s-resources/selenium-grid-k8s-namespace.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_namespace" "selenium-namespace" { 2 | metadata { 3 | labels = { 4 | name = "selenium" 5 | } 6 | 7 | name = "selenium" 8 | } 9 | } -------------------------------------------------------------------------------- /dockerBasedTools/selenoid-android/browsers.json: -------------------------------------------------------------------------------- 1 | { 2 | "android": { 3 | "default": "6.0", 4 | "versions": { 5 | "6.0": { 6 | "image": "selenoid/android:6.0", 7 | "port": "4444", 8 | "path": "/wd/hub" 9 | } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /uiTestsJS/appium-mobile/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "appium-mobile", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "demoTest": "mocha -t 600000 android-create-session.test" 7 | }, 8 | "dependencies": { 9 | "wd": "1.11.4", 10 | "mocha": "6.2.2", 11 | "chai": "4.2.0" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /uiTestsJS/selenium-web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "idea", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "demoTest": "mocha -t 30000 google_search_test.js" 7 | }, 8 | "dependencies": { 9 | "selenium-webdriver": "4.0.0-alpha.5", 10 | "mocha": "6.2.2", 11 | "chromedriver": "79.0.0" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /cloudProviders/gcp/K8s/selenium/selenium-hub-svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: selenium-hub 5 | labels: 6 | app: selenium-hub 7 | spec: 8 | ports: 9 | - port: 4444 10 | targetPort: 4444 11 | name: port0 12 | selector: 13 | app: selenium-hub 14 | type: NodePort 15 | sessionAffinity: None 16 | -------------------------------------------------------------------------------- /dockerBasedTools/selenoid-web/browsers.json: -------------------------------------------------------------------------------- 1 | { 2 | "chrome": { 3 | "default": "76.0", 4 | "versions": { 5 | "66.0": { 6 | "image": "selenoid/vnc:chrome_66.0", 7 | "port": "4444" 8 | }, 9 | "76.0": { 10 | "image": "selenoid/vnc:chrome_76.0", 11 | "port": "4444" 12 | } 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /IaC/terraform/account.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "", 3 | "project_id": "", 4 | "private_key_id": "", 5 | "private_key": "", 6 | "client_email": "", 7 | "client_id": "", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://oauth2.googleapis.com/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "" 12 | } 13 | -------------------------------------------------------------------------------- /IaC/ansible/selenoid-web.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | become: yes 4 | become_user: root 5 | 6 | tasks: 7 | - name: Ansible copy directory with docker-compose to the remote server 8 | copy: 9 | src: ../../dockerBasedTools/selenoid-web/ 10 | dest: /home/ 11 | 12 | - name: Pull Chrome image 13 | shell: docker pull selenoid/vnc:chrome_76.0 14 | 15 | - name: Docker compose up 16 | shell: cd ../ && docker-compose up -d -------------------------------------------------------------------------------- /IaC/terraform/selenium-firewall-rules.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_firewall" "selenium-docker-based" { 2 | name = "selenium-docker-based-terraform" 3 | network = "default" 4 | 5 | allow { 6 | protocol = "icmp" 7 | } 8 | 9 | allow { 10 | protocol = "tcp" 11 | ports = ["4444", "4445", "4446", "8080", "8081", "5900", "7070", "9090"] 12 | } 13 | 14 | source_ranges = ["0.0.0.0/0"] 15 | target_tags = ["selenium"] 16 | } -------------------------------------------------------------------------------- /IaC/ansible/selenoid-android.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | become: yes 4 | become_user: root 5 | 6 | tasks: 7 | - name: Ansible copy directory with docker-compose to the remote server 8 | copy: 9 | src: ../../dockerBasedTools/selenoid-android/ 10 | dest: /home/ 11 | 12 | - name: Pull Android image 13 | shell: docker pull selenoid/android:6.0 14 | 15 | - name: Docker compose up 16 | shell: cd ../ && docker-compose up -d -------------------------------------------------------------------------------- /ci/gitlab-ci/docker-compose-gitlab-macos.yml: -------------------------------------------------------------------------------- 1 | web: 2 | image: 'gitlab/gitlab-ce:latest' 3 | restart: always 4 | hostname: 'gitlab.example.com' 5 | environment: 6 | GITLAB_OMNIBUS_CONFIG: | 7 | external_url 'http://localhost' 8 | ports: 9 | - '80:80' 10 | - '443:443' 11 | - '2222:22' 12 | volumes: 13 | - '/opt/gitlab/config:/etc/gitlab' 14 | - '/opt/gitlab/logs:/var/log/gitlab' 15 | - '/opt/gitlab/data:/var/opt/gitlab' -------------------------------------------------------------------------------- /IaC/ansible/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | become: yes 4 | become_user: root 5 | 6 | tasks: 7 | - name: Install prerequisites 8 | apt: 9 | name: ['python3-pip', 'python3-setuptools', 'virtualenv'] 10 | update_cache: yes 11 | 12 | - name: Install docker-compose 13 | shell: curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose -------------------------------------------------------------------------------- /IaC/ansible/docker.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: all 3 | become: yes 4 | become_user: root 5 | 6 | tasks: 7 | - name: Install prerequisites 8 | apt: 9 | name: ['apt-transport-https', 'ca-certificates', 'curl', 'gnupg2' ,'software-properties-common'] 10 | update_cache: yes 11 | 12 | - name: Add Docker GPG key 13 | apt_key: url=https://download.docker.com/linux/ubuntu/gpg 14 | 15 | - name: Add Docker APT repository 16 | apt_repository: 17 | repo: deb [arch=amd64] https://download.docker.com/linux/{{ansible_distribution|lower}} {{ansible_distribution_release}} stable 18 | 19 | - name: Install Docker 20 | apt: 21 | name: docker-ce 22 | update_cache: yes -------------------------------------------------------------------------------- /uiTestsJS/selenium-web/google_search_test.js: -------------------------------------------------------------------------------- 1 | require('chromedriver'); 2 | const {Builder, By, Key, until} = require('selenium-webdriver'); 3 | 4 | describe('Google Search', function() { 5 | let driver; 6 | 7 | before(async function() { 8 | driver = new Builder() 9 | .forBrowser('chrome') 10 | .usingServer(process.env.REMOTE_HOST || '') 11 | .build(); 12 | }); 13 | 14 | it('demo', async function() { 15 | await driver.get('https://www.google.com/ncr'); 16 | await driver.findElement(By.name('q')).sendKeys('webdriver', Key.RETURN); 17 | await driver.wait(until.titleIs('webdriver - Google Search'), 1000); 18 | }); 19 | 20 | after(() => driver && driver.quit()); 21 | }); 22 | -------------------------------------------------------------------------------- /cloudProviders/gcp/.gitlab-ci.example.gcp.selenoid.web.android.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - uiTests 3 | 4 | runTestViaSelenoidWeb: 5 | stage: uiTests 6 | script: 7 | - cd $WORK_DIRECTORY/uiTestsJS/selenium-web 8 | - export REMOTE_HOST=http://:4445/wd/hub 9 | - rm -r node_modules/ || true 10 | - npm i 11 | - npm run demoTest 12 | allow_failure: false 13 | tags: 14 | - local-run 15 | 16 | runTestViaSelenoidAndroid: 17 | stage: uiTests 18 | script: 19 | - cd $WORK_DIRECTORY/uiTestsJS/appium-mobile 20 | - export USE_SELENOID=true 21 | - export APPIUM_HOST= 22 | - rm -r node_modules/ || true 23 | - npm i 24 | - npm run demoTest 25 | allow_failure: false 26 | tags: 27 | - local-run -------------------------------------------------------------------------------- /IaC/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "Selenoid-Web" { 2 | value = "http://${google_compute_instance.selenoid-web.network_interface.0.access_config.0.nat_ip}:8081/#/" 3 | } 4 | 5 | output "Selenoid-Web-ssh" { 6 | value = "ssh ${var.ssh_user}@${google_compute_instance.selenoid-web.network_interface.0.access_config.0.nat_ip} -i ${var.private_key_path}" 7 | } 8 | 9 | output "Selenoid-Android" { 10 | value = "http://${google_compute_instance.selenoid-android.network_interface.0.access_config.0.nat_ip}:8081/#/" 11 | } 12 | 13 | output "Selenoid-Android-ssh" { 14 | value = "ssh ${var.ssh_user}@${google_compute_instance.selenoid-web.network_interface.0.access_config.0.nat_ip} -i ${var.private_key_path}" 15 | } 16 | 17 | output "Cluster-Endpoint" { 18 | value = "${google_container_cluster.selenium-grid-k8s.endpoint}" 19 | } -------------------------------------------------------------------------------- /dockerBasedTools/selenoid-android/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | selenoid: 4 | image: "aerokube/selenoid" 5 | network_mode: bridge 6 | restart: always 7 | ports: 8 | - "4446:4446" 9 | volumes: 10 | - "$PWD:/etc/selenoid/" # assumed current dir contains browsers.json 11 | - "/var/run/docker.sock:/var/run/docker.sock" 12 | - "${HOME}:/root" 13 | environment: 14 | - "OVERRIDE_HOME=${HOME}" 15 | command: -listen :4446 -conf /etc/selenoid/browsers.json -limit 2 -session-attempt-timeout 4m0s -service-startup-timeout 4m0s -timeout 4m0s -mem 1g -cpu 1.0 16 | selenoid-ui: 17 | image: "aerokube/selenoid-ui:latest" 18 | network_mode: bridge 19 | restart: always 20 | links: 21 | - selenoid 22 | ports: 23 | - "8081:8080" 24 | command: ["--selenoid-uri", "http://selenoid:4446"] -------------------------------------------------------------------------------- /dockerBasedTools/selenoid-web/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | selenoid: 4 | image: "aerokube/selenoid" 5 | network_mode: bridge 6 | ports: 7 | - "4445:4445" 8 | volumes: 9 | - "$PWD:/etc/selenoid/" # assumed current dir contains browsers.json 10 | - "/var/run/docker.sock:/var/run/docker.sock" 11 | - "${HOME}:/root" 12 | - "$PWD/video/:/opt/selenoid/video/" 13 | environment: 14 | - "OVERRIDE_HOME=${HOME}" 15 | - "OVERRIDE_VIDEO_OUTPUT_DIR=$PWD/video/" 16 | command: -listen :4445 -conf /etc/selenoid/browsers.json -limit 2 -disable-queue -session-attempt-timeout 120s -session-delete-timeout 120s -timeout 120s -video-output-dir /opt/selenoid/video/ 17 | selenoid-ui: 18 | image: "aerokube/selenoid-ui:latest" 19 | network_mode: bridge 20 | links: 21 | - selenoid 22 | ports: 23 | - "8081:8080" 24 | command: ["--selenoid-uri", "http://selenoid:4445"] 25 | -------------------------------------------------------------------------------- /IaC/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "public_key_path" { 2 | description = "Path to the public SSH key you want to bake into the instance." 3 | default = "~/.ssh/devopsTools.pub" 4 | } 5 | 6 | variable "private_key_path" { 7 | description = "Path to the private SSH key, used to access the instance." 8 | default = "~/.ssh/devopsTools" 9 | } 10 | 11 | variable "project_name" { 12 | description = "Name of your GCP project. Example: devops-tools-263410" 13 | default = "devops-tools-263410" 14 | } 15 | 16 | variable "ssh_user" { 17 | description = "SSH user name to connect to your instance." 18 | default = "alexey" 19 | } 20 | 21 | variable "nested_vm_disk_name" { 22 | default = "kvm-disk-terraform" 23 | description = "disk for kvm immage support" 24 | } 25 | 26 | variable "nested_vm_image_name" { 27 | default = "kvm-image-terraform" 28 | description = "nested-vm-ubuntu-1804-image with installed selenoid-android" 29 | } 30 | -------------------------------------------------------------------------------- /cloudProviders/gcp/K8s/selenium/selenium-hub-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: selenium-hub 5 | labels: 6 | app: selenium-hub 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: selenium-hub 12 | template: 13 | metadata: 14 | labels: 15 | app: selenium-hub 16 | spec: 17 | containers: 18 | - name: selenium-hub 19 | image: selenium/hub:3.141 20 | ports: 21 | - containerPort: 4444 22 | resources: 23 | limits: 24 | memory: "1000Mi" 25 | cpu: ".5" 26 | livenessProbe: 27 | httpGet: 28 | path: /wd/hub/status 29 | port: 4444 30 | initialDelaySeconds: 30 31 | timeoutSeconds: 5 32 | readinessProbe: 33 | httpGet: 34 | path: /wd/hub/status 35 | port: 4444 36 | initialDelaySeconds: 30 37 | timeoutSeconds: 5 38 | -------------------------------------------------------------------------------- /cloudProviders/gcp/K8s/selenium/selenium-node-chrome-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: selenium-node-chrome 5 | labels: 6 | app: selenium-node-chrome 7 | spec: 8 | replicas: 2 9 | selector: 10 | matchLabels: 11 | app: selenium-node-chrome 12 | template: 13 | metadata: 14 | labels: 15 | app: selenium-node-chrome 16 | spec: 17 | volumes: 18 | - name: dshm 19 | emptyDir: 20 | medium: Memory 21 | containers: 22 | - name: selenium-node-chrome 23 | image: selenium/node-chrome-debug:3.141 24 | ports: 25 | - containerPort: 5555 26 | volumeMounts: 27 | - mountPath: /dev/shm 28 | name: dshm 29 | env: 30 | - name: HUB_HOST 31 | value: "selenium-hub" 32 | - name: HUB_PORT 33 | value: "4444" 34 | resources: 35 | limits: 36 | memory: "1000Mi" 37 | cpu: ".5" 38 | -------------------------------------------------------------------------------- /dockerBasedTools/selenium-grid/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | selenium-hub: 4 | image: selenium/hub:3.141.59-selenium 5 | container_name: selenium-hub 6 | ports: 7 | - "4444:4444" 8 | environment: 9 | - GRID_MAX_SESSION=10 10 | - GRID_TIMEOUT=7200 11 | chrome: 12 | image: selenium/node-chrome:3.141.59-selenium 13 | volumes: 14 | - /dev/shm:/dev/shm 15 | - /dev/urandom:/dev/random 16 | depends_on: 17 | - selenium-hub 18 | environment: 19 | - HUB_HOST=selenium-hub 20 | - HUB_PORT=4444 21 | - START_XVFB=true 22 | - NODE_MAX_SESSION=5 23 | - NODE_MAX_INSTANCES=5 24 | firefox: 25 | image: selenium/node-firefox:3.141.59-selenium 26 | volumes: 27 | - /dev/shm:/dev/shm 28 | - /dev/urandom:/dev/random 29 | depends_on: 30 | - selenium-hub 31 | environment: 32 | - HUB_HOST=selenium-hub 33 | - HUB_PORT=4444 34 | - START_XVFB=true 35 | - NODE_MAX_SESSION=5 36 | - NODE_MAX_INSTANCES=5 -------------------------------------------------------------------------------- /ci/gitlab-ci/.gitlab-ci.example.local.running.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - uiTests 3 | 4 | runTestViaSeleniumGrid: 5 | stage: uiTests 6 | script: 7 | - cd $WORK_DIRECTORY/dockerBasedTools/selenium-grid 8 | - docker-compose up -d 9 | - cd $WORK_DIRECTORY/uiTestsJS/selenium-web 10 | - export REMOTE_HOST=http://localhost:4444/wd/hub 11 | - rm -r node_modules/ || true 12 | - npm i 13 | - npm run demoTest 14 | after_script: 15 | - cd $WORK_DIRECTORY/dockerBasedTools/selenium-grid 16 | - docker-compose down 17 | allow_failure: false 18 | tags: 19 | - local-run 20 | 21 | runTestViaSelenoid: 22 | stage: uiTests 23 | script: 24 | - cd $WORK_DIRECTORY/dockerBasedTools/selenoid-web 25 | - docker-compose up -d 26 | - cd $WORK_DIRECTORY/uiTestsJS/selenium-web 27 | - export REMOTE_HOST=http://localhost:4445/wd/hub 28 | - rm -r node_modules/ || true 29 | - npm i 30 | - npm run demoTest 31 | after_script: 32 | - cd $WORK_DIRECTORY/dockerBasedTools/selenoid-web 33 | - docker-compose down 34 | allow_failure: false 35 | tags: 36 | - local-run -------------------------------------------------------------------------------- /IaC/terraform/selenium-grid-k8s-resources/selenium-grid-k8s-services.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_service" "selenium-hub-svc-internal" { 2 | metadata { 3 | name = "selenium-hub-internal" 4 | 5 | labels = { 6 | app = "selenium-hub" 7 | } 8 | } 9 | 10 | spec { 11 | selector = { 12 | app = "selenium-hub" 13 | } 14 | session_affinity = "None" 15 | port { 16 | port = 4444 17 | target_port = 4444 18 | name = "port0" 19 | } 20 | 21 | type = "NodePort" 22 | } 23 | } 24 | 25 | resource "kubernetes_service" "selenium-hub-svc-external" { 26 | metadata { 27 | name = "selenium-hub-external" 28 | 29 | labels = { 30 | app = "selenium-hub" 31 | external = true 32 | } 33 | } 34 | 35 | spec { 36 | selector = { 37 | app = "selenium-hub" 38 | } 39 | 40 | external_traffic_policy = "Cluster" 41 | load_balancer_ip = "" 42 | 43 | port { 44 | node_port = 30109 45 | port = 4444 46 | target_port = 4444 47 | protocol = "TCP" 48 | } 49 | 50 | type = "LoadBalancer" 51 | } 52 | } -------------------------------------------------------------------------------- /IaC/terraform/selenium-grid-k8s-cluster.tf: -------------------------------------------------------------------------------- 1 | resource "google_container_cluster" "selenium-grid-k8s" { 2 | name = "selenium-grid-k8s-terraform" 3 | location = "europe-west3-a" 4 | 5 | remove_default_node_pool = true 6 | initial_node_count = 1 7 | 8 | master_auth { 9 | username = "" 10 | password = "" 11 | 12 | client_certificate_config { 13 | issue_client_certificate = false 14 | } 15 | } 16 | } 17 | 18 | resource "google_container_node_pool" "selenium-grid-k8s-preemptible-nodes" { 19 | name = "selenium-node-pool" 20 | location = "europe-west3-a" 21 | cluster = google_container_cluster.selenium-grid-k8s.name 22 | node_count = 2 23 | 24 | node_config { 25 | preemptible = true 26 | machine_type = "n1-standard-2" 27 | 28 | metadata = { 29 | disable-legacy-endpoints = "true" 30 | } 31 | 32 | oauth_scopes = [ 33 | "https://www.googleapis.com/auth/logging.write", 34 | "https://www.googleapis.com/auth/monitoring", 35 | ] 36 | } 37 | 38 | depends_on = ["google_container_cluster.selenium-grid-k8s"] 39 | } 40 | 41 | resource "null_resource" "configure_kubectl" { 42 | provisioner "local-exec" { 43 | command = "gcloud container clusters get-credentials ${google_container_cluster.selenium-grid-k8s.name} --zone ${google_container_cluster.selenium-grid-k8s.location} --project ${var.project_name}" 44 | } 45 | 46 | depends_on = ["google_container_node_pool.selenium-grid-k8s-preemptible-nodes"] 47 | } -------------------------------------------------------------------------------- /uiTestsJS/appium-mobile/android-create-session.test.js: -------------------------------------------------------------------------------- 1 | const wd = require('wd'); 2 | const chai = require('chai'); 3 | const path = require('path'); 4 | const {androidCaps, serverConfig, androidApiDemos} = require(path.resolve(__dirname, 'config')); 5 | 6 | const {assert} = chai; 7 | const SEARCH_ACTIVITY = '.app.SearchInvoke'; 8 | 9 | describe('Basic Android interactions', function () { 10 | let driver; 11 | let allPassed = true; 12 | 13 | before(async function () { 14 | driver = await wd.promiseChainRemote(serverConfig) 15 | 16 | androidCaps.app = androidApiDemos 17 | androidCaps['appActivity'] = SEARCH_ACTIVITY, // Android-specific capability. Can open a specific activity. 18 | 19 | await driver.init(androidCaps); 20 | }); 21 | 22 | it('should send keys to search box and then check the value', async function () { 23 | // Enter text in a search box 24 | const searchBoxElement = await driver.elementById('txt_query_prefill'); 25 | await searchBoxElement.sendKeys('Hello world!'); 26 | 27 | // Press on 'onSearchRequestedButton' 28 | const onSearchRequestedButton = await driver.elementById('btn_start_search'); 29 | await onSearchRequestedButton.click(); 30 | 31 | // Check that the text matches the search term 32 | const searchText = await driver.waitForElementById('android:id/search_src_text'); 33 | const searchTextValue = await searchText.text(); 34 | assert.equal(searchTextValue, 'Hello world!'); 35 | }); 36 | 37 | afterEach(function () { 38 | // keep track of whether all the tests have passed, since mocha does not do this 39 | allPassed = allPassed && (this.currentTest.state === 'passed'); 40 | }); 41 | 42 | after(async function () { 43 | await driver.quit(); 44 | }); 45 | }); -------------------------------------------------------------------------------- /uiTestsJS/appium-mobile/config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | // Leave the Android platformVersion blank and set deviceName to a random string 4 | // (Android deviceName is ignored by Appium but is still required) 5 | // If we're using SauceLabs, set the Android deviceName and platformVersion to 6 | // the latest supported SauceLabs device and version 7 | const DEFAULT_ANDROID_DEVICE_NAME = process.env.USE_SELENOID 8 | ? 'android' 9 | : 'Android Emulator' 10 | const DEFAULT_ANDROID_PLATFORM_VERSION = null; 11 | 12 | const androidCaps = { 13 | platformName: 'Android', 14 | automationName: 'UiAutomator2', 15 | deviceName: process.env.ANDROID_DEVICE_NAME || DEFAULT_ANDROID_DEVICE_NAME, 16 | platformVersion: process.env.ANDROID_PLATFORM_VERSION || DEFAULT_ANDROID_PLATFORM_VERSION, 17 | androidInstallTimeout: 90000, 18 | app: undefined, // Will be added in tests 19 | }; 20 | 21 | const host = process.env.APPIUM_HOST || 'localhost'; 22 | const port = process.env.APPIUM_PORT || '4723'; 23 | 24 | // figure out where the Appium server should be pointing to 25 | const serverConfig = process.env.USE_SELENOID 26 | ? `http://${host}:4446/wd/hub` 27 | : { 28 | host, 29 | port 30 | }; 31 | 32 | 33 | // figure out the location of the apps under test 34 | const GITHUB_ASSET_BASE = 'http://appium.github.io/appium/assets'; 35 | const LOCAL_ASSET_BASE = path.resolve(__dirname, '..', '..', '..', 'apps'); 36 | 37 | let androidApiDemos; 38 | if (true) { 39 | // TODO: Change thes URLs to updated locations 40 | androidApiDemos = `${GITHUB_ASSET_BASE}/ApiDemos-debug.apk`; 41 | } else { 42 | androidApiDemos = path.resolve(LOCAL_ASSET_BASE, 'ApiDemos-debug.apk'); 43 | } 44 | 45 | module.exports = { 46 | androidApiDemos, 47 | androidCaps, 48 | serverConfig, 49 | }; -------------------------------------------------------------------------------- /IaC/terraform/selenoid-web.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_instance" "selenoid-web" { 2 | name = "selenoid-web-terraform" 3 | machine_type = "g1-small" 4 | zone = "europe-west4-a" 5 | 6 | tags = ["selenium"] 7 | 8 | boot_disk { 9 | initialize_params { 10 | size = 50 11 | image = "ubuntu-1604-lts" 12 | } 13 | } 14 | 15 | scheduling { 16 | preemptible = true 17 | automatic_restart = false 18 | } 19 | 20 | network_interface { 21 | network = "default" 22 | 23 | access_config { 24 | // Ephemeral IP 25 | } 26 | } 27 | 28 | metadata_startup_script = "docker-compose up -d" 29 | 30 | metadata = { 31 | ssh-keys = "${var.ssh_user}:${file("${var.public_key_path}")}" 32 | } 33 | 34 | ############################################################################# 35 | # This is the 'local exec' method. 36 | # Ansible runs from the same host you run Terraform from 37 | ############################################################################# 38 | 39 | provisioner "remote-exec" { 40 | inline = ["echo 'Hello World'"] 41 | 42 | connection { 43 | type = "ssh" 44 | host = "${google_compute_instance.selenoid-web.network_interface.0.access_config.0.nat_ip}" 45 | user = "${var.ssh_user}" 46 | private_key = "${file("${var.private_key_path}")}" 47 | } 48 | } 49 | 50 | provisioner "local-exec" { 51 | command = "ansible-playbook -i '${google_compute_instance.selenoid-web.network_interface.0.access_config.0.nat_ip},' --private-key ${var.private_key_path} ../ansible/docker.yml" 52 | } 53 | 54 | provisioner "local-exec" { 55 | command = "ansible-playbook -i '${google_compute_instance.selenoid-web.network_interface.0.access_config.0.nat_ip},' --private-key ${var.private_key_path} ../ansible/docker-compose.yml" 56 | } 57 | 58 | provisioner "local-exec" { 59 | command = "ansible-playbook -i '${google_compute_instance.selenoid-web.network_interface.0.access_config.0.nat_ip},' --private-key ${var.private_key_path} ../ansible/selenoid-web.yml" 60 | } 61 | } -------------------------------------------------------------------------------- /IaC/terraform/selenoid-android.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_disk" "kvm_disk" { 2 | name = "${var.nested_vm_disk_name}" 3 | zone = "europe-west4-a" 4 | image = "ubuntu-1804-lts" 5 | size = 50 6 | } 7 | 8 | resource "google_compute_image" "selenoid_android_image" { 9 | name = "${var.nested_vm_image_name}" 10 | source_disk = "https://www.googleapis.com/compute/v1/projects/${var.project_name}/zones/europe-west4-a/disks/${var.nested_vm_disk_name}" 11 | 12 | licenses = [ 13 | "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/licenses/ubuntu-1804-lts", 14 | "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" 15 | ] 16 | 17 | depends_on = ["google_compute_disk.kvm_disk"] 18 | } 19 | 20 | resource "google_compute_instance" "selenoid-android" { 21 | name = "selenoid-android-terraform" 22 | machine_type = "n1-standard-4" 23 | zone = "europe-west3-a" 24 | 25 | tags = ["selenium"] 26 | 27 | boot_disk { 28 | initialize_params { 29 | size = 50 30 | image = "${var.nested_vm_image_name}" 31 | } 32 | } 33 | 34 | scheduling { 35 | preemptible = true 36 | automatic_restart = false 37 | } 38 | 39 | network_interface { 40 | network = "default" 41 | 42 | access_config { 43 | // Ephemeral IP 44 | } 45 | } 46 | 47 | metadata_startup_script = "docker-compose up -d" 48 | 49 | metadata = { 50 | ssh-keys = "${var.ssh_user}:${file("${var.public_key_path}")}" 51 | } 52 | 53 | ############################################################################# 54 | # This is the 'local exec' method. 55 | # Ansible runs from the same host you run Terraform from 56 | ############################################################################# 57 | 58 | provisioner "remote-exec" { 59 | inline = ["echo 'Hello World'"] 60 | 61 | connection { 62 | type = "ssh" 63 | host = "${google_compute_instance.selenoid-android.network_interface.0.access_config.0.nat_ip}" 64 | user = "${var.ssh_user}" 65 | private_key = "${file("${var.private_key_path}")}" 66 | } 67 | } 68 | 69 | provisioner "local-exec" { 70 | command = "ansible-playbook -i '${google_compute_instance.selenoid-android.network_interface.0.access_config.0.nat_ip},' --private-key ${var.private_key_path} ../ansible/docker.yml" 71 | } 72 | 73 | provisioner "local-exec" { 74 | command = "ansible-playbook -i '${google_compute_instance.selenoid-android.network_interface.0.access_config.0.nat_ip},' --private-key ${var.private_key_path} ../ansible/docker-compose.yml" 75 | } 76 | 77 | provisioner "local-exec" { 78 | command = "ansible-playbook -i '${google_compute_instance.selenoid-android.network_interface.0.access_config.0.nat_ip},' --private-key ${var.private_key_path} ../ansible/selenoid-android.yml" 79 | } 80 | 81 | depends_on = ["google_compute_image.selenoid_android_image"] 82 | } -------------------------------------------------------------------------------- /IaC/terraform/selenium-grid-k8s-resources/selenium-grid-k8s-deployment.tf: -------------------------------------------------------------------------------- 1 | resource "kubernetes_deployment" "selenium-hub" { 2 | metadata { 3 | name = "selenium-hub" 4 | labels = { 5 | app = "selenium-hub" 6 | } 7 | } 8 | 9 | spec { 10 | replicas = 1 11 | 12 | selector { 13 | match_labels = { 14 | app = "selenium-hub" 15 | } 16 | } 17 | 18 | template { 19 | metadata { 20 | labels = { 21 | app = "selenium-hub" 22 | } 23 | } 24 | 25 | spec { 26 | container { 27 | image = "selenium/hub:3.141" 28 | name = "selenium-hub" 29 | 30 | port { 31 | container_port = 4444 32 | } 33 | 34 | resources { 35 | limits { 36 | cpu = ".5" 37 | memory = "1000Mi" 38 | } 39 | } 40 | 41 | liveness_probe { 42 | http_get { 43 | path = "/wd/hub/status" 44 | port = 4444 45 | } 46 | 47 | initial_delay_seconds = 30 48 | timeout_seconds = 5 49 | } 50 | 51 | readiness_probe { 52 | http_get { 53 | path = "/wd/hub/status" 54 | port = 4444 55 | } 56 | 57 | initial_delay_seconds = 30 58 | timeout_seconds = 5 59 | } 60 | } 61 | } 62 | } 63 | } 64 | } 65 | 66 | resource "kubernetes_deployment" "selenium-node-chrome" { 67 | metadata { 68 | name = "selenium-node-chrome" 69 | labels = { 70 | app = "selenium-node-chrome" 71 | } 72 | } 73 | 74 | spec { 75 | replicas = 2 76 | 77 | selector { 78 | match_labels = { 79 | app = "selenium-node-chrome" 80 | } 81 | } 82 | 83 | template { 84 | metadata { 85 | labels = { 86 | app = "selenium-node-chrome" 87 | } 88 | } 89 | 90 | spec { 91 | volume { 92 | name = "dshm" 93 | 94 | empty_dir { 95 | medium = "Memory" 96 | } 97 | } 98 | 99 | container { 100 | image = "selenium/node-chrome-debug:3.141" 101 | name = "selenium-node-chrome" 102 | 103 | port { 104 | container_port = 5555 105 | } 106 | 107 | volume_mount { 108 | mount_path = "/dev/shm" 109 | name = "dshm" 110 | } 111 | 112 | env { 113 | name = "HUB_HOST" 114 | value = "selenium-hub-internal" 115 | } 116 | 117 | env { 118 | name = "HUB_PORT" 119 | value = "4444" 120 | } 121 | 122 | resources { 123 | limits { 124 | cpu = ".5" 125 | memory = "1000Mi" 126 | } 127 | } 128 | } 129 | } 130 | } 131 | } 132 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Devops tools for qa automation infrastructure 2 | ======== 3 | The repo contains step by step guide how test automation infrastructure could 4 | be improved starting from running test locally to Infrastructure as Code (IaC) practices with using Cloud providers 5 | 6 | **Plan**: 7 | 8 | 1. Prepare web / mobile demo tests and run it locally ```Node.js, Selenium, Appium``` 9 | 10 | 2. Docker based Tools ```Selenium grid, Selenoid (Web, Android)``` 11 | 12 | 3. CI/CD system ```Gitlab CI``` 13 | 14 | 4. Cloud platforms ```Google Cloud Platform``` 15 | 16 | 5. Orchestration tool ```Kubernetes``` 17 | 18 | 6. Infrastructure as a code tools ```Terraform, Ansible``` 19 | 20 | ## 1. Prepare web / mobile demo tests and run it locally 21 | 22 | The first step is just to prepare demo tests for web/android and run it locally. 23 | 24 | **Preconditions:** 25 | * Installed [Node.js](https://nodejs.org/en/) 26 | * Installed [Chrome](https://www.google.com/intl/ru/chrome/) 27 | * Installed [Appium](http://appium.io/docs/en/about-appium/getting-started/) 28 | * Installed [Android emulator](https://developer.android.com/studio/run/emulator) 29 | 30 | **Steps to execute tests:** 31 | * Web 32 | ``` 33 | $ cd uiTestsJS/selenium-web 34 | $ npm i 35 | $ npm run demoTest 36 | ``` 37 | * Android 38 | ``` 39 | - start appium or Appium Desktop 40 | - run Android simulator 41 | 42 | $ cd uiTestsJS/appium-mobile 43 | $ npm i 44 | $ npm run demoTest 45 | ``` 46 | 47 | 48 | Be sure that tests are started and passed. We will not run it via local 49 | browsers/emulators anymore! 50 | 51 | **Links:** 52 | * [Selenium tests](https://github.com/SeleniumHQ/selenium/tree/master/javascript/node/selenium-webdriver/example) 53 | * [Appium tests](https://github.com/appium/appium/tree/master/sample-code/javascript-wd) 54 | 55 | **What could be used instead:** 56 | * Any programming languages that you like for selenium/appium tests 57 | * Any tests that you like 58 | * Any test runner 59 | 60 | ## 2. Docker based Tools 61 | 62 | The second step is to run tests via popular docker based tools 63 | 64 | **Preconditions:** 65 | * Installed / run [Docker](https://www.docker.com) 66 | * Installed [Docker Compose](https://docs.docker.com/compose/install/) 67 | 68 | **Steps to execute tests:** 69 | * Selenium grid 70 | ``` 71 | $ cd dockerBasedTools/selenium-grid 72 | $ docker-compose up -d // run selenium hub with nodes 73 | 74 | - http://localhost:4444/grid/console // open in browser 75 | 76 | $ export REMOTE_HOST=http://localhost:4444/wd/hub 77 | 78 | - run web tests from part 1 79 | 80 | $ docker-compose down // stop/delete all containers 81 | ``` 82 | * Selenoid web 83 | ``` 84 | $ cd dockerBasedTools/selenoid-web 85 | $ cat browsers.json // set browsers that you need, ex: selenoid/vnc:chrome_76.0 86 | $ docker pull selenoid/vnc:chrome_76.0 // pull image 87 | $ docker-compose up -d // run selenoid server + ui 88 | 89 | - http://localhost:8080/#/ // open in browser 90 | 91 | $ export REMOTE_HOST=http://localhost:4445/wd/hub 92 | 93 | - run web tests from part 1 94 | 95 | $ docker-compose down // stop/delete all containers 96 | ``` 97 | 98 | * Selenoid Android 99 | 100 | ["Android emulator can only be launched on a hardware 101 | server or a particular type of virtual machines supporting nested 102 | virtualization."](https://medium.com/@aandryashin/selenium-more-android-sweets-3839148d1bac) 103 | 104 | ["Docker for Mac does not forward KVM"](https://github.com/aerokube/selenoid/issues/687) 105 | 106 | Instruction how to setup VM with KVM via GCP will be in part 4 of this guide. 107 | ``` 108 | $ cd dockerBasedTools/selenoid-android 109 | $ cat browsers.json // set simulator that you need, ex: selenoid/android:6.0 110 | $ docker pull selenoid/android:6.0 // pull image 111 | $ docker-compose up -d // run selenoid server + ui 112 | 113 | - http://localhost:8081/#/ // open in browser 114 | 115 | $ export USE_SELENOID=true 116 | 117 | - run android tests from part 1 118 | 119 | $ docker-compose down // stop/delete all containers 120 | ``` 121 | 122 | **Links:** 123 | * [docker-selenium](https://github.com/SeleniumHQ/docker-selenium) 124 | * [selenoid](https://github.com/aerokube/selenoid) 125 | 126 | **What could be used instead:** 127 | 128 | Docker is the most popular container runtime environment. 129 | [Though Docker still made up 83 percent of containers in 2018, that number is 130 | down from 99 percent in 2017.](https://containerjournal.com/topics/container-ecosystems/5-container-alternatives-to-docker/) 131 | However in this guide we can use docker as these tools for test running (Selenium grid, Selenoid) are docker based. 132 | 133 | 134 | ## 3. CI/CD system 135 | 136 | The third step is to run tests via CI system (Gitlab CI) that will be setup 137 | locally . There will be 2 jobs to run test via selenium-grid and selenoid-web 138 | . Android test will be executed in next step as we are going to setup GCP VM 139 | with KVM support. 140 | 141 | **Preconditions:** 142 | * Installed / run [Docker](https://www.docker.com) 143 | * Installed [Docker Compose](https://docs.docker.com/compose/install/) 144 | 145 | **Steps to execute tests:** 146 | 147 | * Run Gitlab server 148 | ``` 149 | $ cd ci/gitlab-ci 150 | $ cat docker-compose-gitlab-macos.yml // check volumes ex: /opt/gitlab/* , for MacOS the directory should be created manually and path should be added to docker File Sharing. For other systems the will be link at the and of the step. 151 | $ docker-compose -f docker-compose-gitlab-macos.yml up -d // can take some minutes until server is up 152 | ``` 153 | 154 | * Create project via UI 155 | ``` 156 | - open http://localhost/ 157 | - set password ex:12345678 158 | - username: root 159 | - open http://localhost/admin 160 | - create a new project with name: local-run 161 | - open http://localhost/root/local-run/-/settings/ci_cd 162 | - open Variables 163 | - set Type: Variable, Key: WORK_DIRECTORY, Value: path to the current project ex: /Users/alexey/*/devops-tools-for-qa-automation-infrastructure/ 164 | - click Save variables 165 | ``` 166 | 167 | * Install/run gitlab-runner 168 | ``` 169 | - open http://localhost/root/local-run/-/settings/ci_cd 170 | - open Runners 171 | - check instruction: Set up a specific Runner manually 172 | ``` 173 | 174 | input ex: 175 | ``` 176 | Please enter the gitlab-ci coordinator URL (e.g. https://gitlab.com/): 177 | http://localhost/ 178 | Please enter the gitlab-ci token for this runner: 179 | 180 | Please enter the gitlab-ci description for this runner: 181 | [Mac-Mini-ALEXEY]: test 182 | Please enter the gitlab-ci tags for this runner (comma separated): 183 | local-run 184 | Registering runner... succeeded runner=6tMT7ztw 185 | Please enter the executor: custom, parallels, shell, ssh, docker+machine, docker-ssh+machine, docker, docker-ssh, virtualbox, kubernetes: 186 | shell 187 | Runner registered successfully. Feel free to start it, but if it's running already the config should be automatically reloaded! 188 | ``` 189 | 190 | ``` 191 | $ cat ~/.gitlab-runner/config.toml // be sure that added above changes are there 192 | 193 | - be sure that runner tag is local-run as it is used in .gitlab-ci.yml 194 | 195 | $ gitlab-runner start 196 | 197 | - open http://localhost/root/local-run/-/settings/ci_cd 198 | - open Runners 199 | - check that runner is added and active 200 | ``` 201 | 202 | * Push pipeline configuration (.gitlab-ci.yml) 203 | ``` 204 | $ git clone http://localhost/root/local-run.git 205 | $ cp .gitlab-ci.example.local.running.yml local-run/.gitlab-ci.yml 206 | $ cd local-run/ 207 | $ git add .gitlab-ci.yml 208 | $ git commit -m "add pipeline config" 209 | $ git push 210 | 211 | - open http://localhost/root/local-run/pipelines 212 | - check that all jobs are passed (green) 213 | - click on each job, check what was executed and compare with .gitlab-ci.yml 214 | 215 | $ gitlab-runner stop // stop runner 216 | $ cd ../ 217 | $ docker-compose -f docker-compose-gitlab-macos.yml down // stop gitlab 218 | $ docker ps // check there are not running containers 219 | ``` 220 | 221 | **Links:** 222 | * [gitlab docs](https://docs.gitlab.com/) 223 | * [install gitlab using docker compose](https://docs.gitlab.com/omnibus/docker/README.html#install-gitlab-using-docker-compose) 224 | * [gitlab runner docs](https://docs.gitlab.com/runner/) 225 | * [gitlab ci yml](https://docs.gitlab.com/ee/ci/yaml/) 226 | 227 | **What could be used instead:** 228 | 229 | Whatever you like. CI/CD system is **key part** of test automation infrastructure but it is completely up to you to use that one that you like 230 | more or used by your company. 231 | 232 | The most popular systems: 233 | 234 | * [Jenkins](https://jenkins.io/) 235 | * [TeamCity](https://jetbrains.ru/products/teamcity/) 236 | * [Bamboo](https://www.atlassian.com/software/bamboo) 237 | * [Travis](https://travis-ci.com/plans) 238 | 239 | 240 | ## 4. Cloud platforms (GCP) 241 | 242 | The fourth step is to use Google Cloud Platform to run each docker based tool on separate VM: 243 | * VM with Selenoid (web) 244 | * VM with Selenoid (android) with KVM support 245 | * K8s cluster with Selenium grid **(will be setup in the next step)** 246 | 247 | Home work (in this step we will run it locally from step 3 setup): 248 | * VM with Gitlab server 249 | * VM with gitlab runner 250 | 251 | **Preconditions:** 252 | * Created [GCP account](https://console.cloud.google.com), use 300$ trial 253 | * Created GCP project, ex: **devops-tools** 254 | * Added [SSH keys](https://console.cloud.google.com/compute/metadata/sshKeys) that can be used to connect to the VM instances of a project 255 | * Installed [gcloud](https://cloud.google.com/sdk/docs/) 256 | 257 | **Steps to execute tests:** 258 | 259 | * Create VM with Selenoid web 260 | ``` 261 | $ gcloud auth list // get account list 262 | $ gcloud config set account `ACCOUNT` // setup gcloud to use your account 263 | 264 | // create instance: 265 | $ gcloud compute instances create selenoid-web \ 266 | --boot-disk-size=50GB \ 267 | --image-family ubuntu-1604-lts \ 268 | --image-project=ubuntu-os-cloud \ 269 | --machine-type=g1-small \ 270 | --tags selenium \ 271 | --preemptible \ 272 | --restart-on-failure 273 | ``` 274 | 275 | * Access selenoid-web VM via SSH 276 | ``` 277 | $ ssh user@ -i ~/.ssh/publicKey 278 | $ sudo su -- 279 | ``` 280 | 281 | * Install docker, docker-compose, pull selenoid components 282 | ``` 283 | $ apt update 284 | $ apt install apt-transport-https ca-certificates curl software-properties-common 285 | $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 286 | $ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" 287 | $ apt update 288 | $ apt-cache policy docker-ce 289 | $ apt install docker-ce 290 | $ systemctl status docker 291 | 292 | $ docker pull selenoid/android:6.0 293 | $ docker pull selenoid/vnc:chrome_76.0 294 | 295 | $ curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 296 | $ chmod +x /usr/local/bin/docker-compose 297 | ``` 298 | 299 | * Clone the project with selenoid docker-compose 300 | ``` 301 | $ git clone https://github.com/AlexeyAltunin/devops-tools-for-qa-automation-infrastructure.git 302 | 303 | - exit VM 304 | ``` 305 | 306 | * Create image for Selenoid Android with KVM support based on disk for selenoid-web 307 | ``` 308 | $ gcloud compute instances stop selenoid-web 309 | $ gcloud compute images create docker--selenoid-kvm-vm-ubuntu-1604-image --source-disk selenoid-web --source-disk-zone europe-west4-a --licenses "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" 310 | ``` 311 | 312 | * Create VM with Selenoid Android based on previously created KVM image 313 | ``` 314 | $ gcloud compute instances create selenoid-android \ 315 | --boot-disk-size=50GB \ 316 | --image docker-selenoid-kvm-vm-ubuntu-1604-image \ 317 | --image-project=devops-tools-263410 \ 318 | --machine-type=n1-standard-4 \ 319 | --tags selenium \ 320 | --preemptible \ 321 | --restart-on-failure 322 | ``` 323 | 324 | * Access selenoid-android VM via SSH and [turn on KVM](https://www.server-world.info/en/note?os=Ubuntu_18.04&p=kvm&f=8) 325 | ``` 326 | $ ssh user@ -i ~/.ssh/publicKey 327 | $ sudo su -- 328 | $ cat /sys/module/kvm_intel/parameters/nested 329 | $ echo 'options kvm_intel nested=1' >> /etc/modprobe.d/qemu-system-x86.conf 330 | 331 | - exit VM 332 | 333 | $ gcloud compute instances stop selenoid-android 334 | ``` 335 | 336 | * Create firewall rules to open ports 337 | ``` 338 | $ gcloud compute firewall-rules create selenium-docker-based \ 339 | --target-tags selenium \ 340 | --source-ranges 0.0.0.0/0 \ 341 | --allow tcp:4444,tcp:4445,tcp:4446,tcp:8080,tcp:5900,tcp:7070,tcp:9090,tcp:8081 342 | ``` 343 | 344 | * Start VMs 345 | ``` 346 | $ gcloud compute instances list 347 | $ gcloud compute instances start selenoid-web 348 | $ gcloud compute instances start selenoid-android 349 | ``` 350 | 351 | * Run Selenoid web 352 | ``` 353 | $ ssh user@ -i ~/.ssh/publicKey 354 | $ sudo su -- 355 | $ cd devops-tools-for-qa-automation-infrastructure/dockerBasedTools/selenoid-web/ 356 | $ docker-compose up -d 357 | 358 | $ docker-compose ps 359 | ``` 360 | 361 | output: 362 | ``` 363 | selenoid-web_selenoid-ui_1_dd6873036cac /selenoid-ui --selenoid-ur ... Up (healthy) 0.0.0.0:8080->8080/tcp 364 | selenoid-web_selenoid_1_d8308aba93e6 /usr/bin/selenoid -listen ... Up 4444/tcp, 0.0.0.0:4445->4445/tcp 365 | ``` 366 | 367 | ``` 368 | - exit from VM 369 | - open in browser http://:8080 370 | ``` 371 | 372 | * Run Selenoid Android 373 | ``` 374 | $ ssh user@ -i ~/.ssh/publicKey 375 | $ sudo su -- 376 | 377 | - check KVM: 378 | 379 | $ cat /sys/module/kvm_intel/parameters/nested 380 | - output: Y 381 | 382 | $ cd devops-tools-for-qa-automation-infrastructure/dockerBasedTools/selenoid-android/ 383 | $ docker-compose up -d 384 | 385 | $ docker-compose ps 386 | ``` 387 | 388 | output: 389 | ``` 390 | selenoid-android_selenoid-ui_1_888c3c8c22ed /selenoid-ui --selenoid-ur ... Up (healthy) 0.0.0.0:8081->8080/tcp 391 | selenoid-android_selenoid_1_cc45d1fd456d /usr/bin/selenoid -listen ... Up 4444/tcp, 0.0.0.0:4446->4446/tcp 392 | ``` 393 | 394 | ``` 395 | - exit from VM 396 | - open in browser http://:8080 397 | ``` 398 | 399 | * Run gitlab server and runner from part 3 400 | 401 | * Push pipeline configuration (.gitlab-ci.yml) 402 | ``` 403 | $ git clone http://localhost/root/local-run.git 404 | 405 | - open cloudProviders/gcp/.gitlab-ci.example.gcp.selenoid.web.android.yml and set for each job 406 | 407 | $ cp cloudProviders/gcp/.gitlab-ci.example.gcp.selenoid.web.android.yml local-run/.gitlab-ci.yml 408 | $ cd local-run/ 409 | $ git add .gitlab-ci.yml 410 | $ git commit -m "add pipeline config with selenoid web/android in GCP" 411 | $ git push 412 | 413 | - open http://localhost/root/local-run/pipelines 414 | - check that all jobs are passed (green) 415 | - click on each job, check what was executed and compare with .gitlab-ci.yml 416 | 417 | $ gitlab-runner stop // stop runner 418 | $ cd ../ 419 | $ docker-compose -f docker-compose-gitlab-macos.yml down // stop gitlab 420 | $ docker ps // check there are not running containers 421 | ``` 422 | 423 | * Stop VMs 424 | ``` 425 | $ gcloud compute instances stop selenoid-web 426 | $ gcloud compute instances stop selenoid-android 427 | ``` 428 | 429 | **Links:** 430 | * [GCP](https://cloud.google.com/) 431 | * [Cloud SDK](https://cloud.google.com/sdk/docs/) 432 | 433 | **What could be used instead:** 434 | 435 | Whatever you like. Public cloud is very valuable and flexible part of 436 | automation infrastructure but it is completely up to you to use that provider that you like more or used by your company. 437 | 438 | The most popular cloud providers: 439 | 440 | * [Amazon AWS](https://aws.amazon.com/) 441 | * [Microsoft Azure](https://azure.microsoft.com/en-us/) 442 | * [Openstack](https://www.openstack.org) 443 | 444 | ## 5.Orchestration tool (K8s) 445 | 446 | The fifth step is to use Google Cloud Platform to run Selenium grid in Kubernetes cluster. 447 | 448 | **Preconditions:** 449 | * Enabled [Kubernetes Engine API](https://console.cloud.google.com/apis/library/) 450 | * Installed [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 451 | 452 | **Steps to execute tests:** 453 | 454 | * create gcp cluster: 455 | ``` 456 | $ gcloud container clusters create \ 457 | --machine-type n1-standard-2 \ 458 | --num-nodes 2 \ 459 | --zone europe-west4-a \ 460 | --cluster-version latest \ 461 | selenium-grid 462 | ``` 463 | 464 | * check kubectl context 465 | ``` 466 | $ gcloud container clusters get-credentials selenium-grid --zone europe-west4-a --project devops-tools-263410 467 | $ kubectl config current-context 468 | ``` 469 | 470 | * create Selenium grid components 471 | ``` 472 | $ cd cloudProviders/gcp/K8s/selenium/ 473 | 474 | $ kubectl create -f selenium-hub-deployment.yaml 475 | $ kubectl create -f selenium-hub-svc.yaml 476 | $ kubectl create -f selenium-node-chrome-deployment.yaml 477 | ``` 478 | 479 | * expose your hub via the internet 480 | ``` 481 | $ kubectl expose deployment selenium-hub --name=selenium-hub-external --labels="app=selenium-hub,external=true" --type=LoadBalancer 482 | $ kubectl get svc selenium-hub-external // wait EXTERNAL-IP 483 | ``` 484 | 485 | * check self healing feature (delete pod and check that it is recreated) 486 | ``` 487 | $ kubectl get pods 488 | ``` 489 | 490 | output: 491 | ``` 492 | NAME READY STATUS RESTARTS AGE 493 | selenium-hub-7ff45ff687-f4xr6 1/1 Running 0 43s 494 | selenium-node-chrome-69fbcd9bf5-bn9nh 1/1 Running 0 37s 495 | selenium-node-chrome-69fbcd9bf5-ttk4w 1/1 Running 0 37s 496 | ``` 497 | 498 | ``` 499 | $ kubectl delete pod selenium-node-chrome-69fbcd9bf5-bn9nh 500 | ``` 501 | 502 | output: 503 | ``` 504 | $ pod "selenium-node-chrome-69fbcd9bf5-bn9nh" deleted 505 | ``` 506 | 507 | ``` 508 | $ kubectl get pods 509 | ``` 510 | 511 | output: 512 | ``` 513 | NAME READY STATUS RESTARTS AGE 514 | selenium-hub-7ff45ff687-f4xr6 1/1 Running 1 6m3s 515 | selenium-node-chrome-69fbcd9bf5-qfz5c 1/1 Running 0 4m58s 516 | selenium-node-chrome-69fbcd9bf5-ttk4w 1/1 Running 0 5m57s 517 | ``` 518 | 519 | * open browser and check scaling feature 520 | ``` 521 | - open http://:4444/grid/console 522 | - check there are 2 pods with chrome 523 | 524 | $ kubectl scale deployment selenium-node-chrome --replicas=3 525 | 526 | - open http://:4444/grid/console 527 | - check there are 3 pods with chromes 528 | 529 | $ kubectl get pods 530 | $ kubectl scale deployment selenium-node-chrome --replicas=2 // return back 531 | ``` 532 | 533 | * run tests from step 1 534 | ``` 535 | $ export REMOTE_HOST=http://:4444/wd/hub 536 | 537 | $ cd uiTestsJS/selenium-web 538 | $ npm i 539 | $ npm run demoTest 540 | ``` 541 | 542 | * open browser and check auto scaling feature (HorizontalPodAutoscaler) 543 | ``` 544 | $ kubectl autoscale deployment selenium-node-chrome --max 4 --min 1 --cpu-percent 1 545 | 546 | $ run tests 547 | 548 | - open http://:4444/grid/console 549 | - check that number of pods with chromes automatically increased 550 | 551 | $ kubectl get pods 552 | 553 | $ kubectl get hpa selenium-node-chrome // get created pod autoscaling 554 | $ kubectl delete hpa selenium-node-chrome // delete autoscaling 555 | ``` 556 | 557 | * delete cluster (in the next step it will be recreated via Terraform) 558 | ``` 559 | $ gcloud container clusters delete selenium-grid 560 | ``` 561 | 562 | **Links:** 563 | * [Kubernetes engine](https://cloud.google.com/kubernetes-engine/docs/) 564 | * [Selenium grid K8s example](https://github.com/kubernetes/examples/tree/master/staging/selenium) 565 | * [Medium Getting Started guide](https://medium.com/@subbarao.pilla/k8s-selenium-grid-selenium-grid-with-docker-on-kubernetes-42af8b9a2cba) 566 | * [Scaling an application](https://cloud.google.com/kubernetes-engine/docs/how-to/scaling-apps) 567 | 568 | **What could be used instead:** 569 | 570 | You can use [any another way](https://kubernetes.io/docs/setup/) to get K8s cluster instead of GCP. 571 | 572 | The most popular Container Orchestration Tools: 573 | 574 | * [Docker Swarm](https://www.docker.com/products/docker-swarm) 575 | * [Marathon](https://mesosphere.github.io/marathon/) 576 | 577 | Instead of Selenium grid in K8s you can use [Moon:](https://aerokube.com/moon/latest/) 578 | 579 | "Moon is a browser automation solution compatible with Selenium Webdriver protocol and using Kubernetes to launch browsers." 580 | But [it takes money.](https://aerokube.com/moon/latest/#_pricing) 581 | 582 | ## 6. IaC 583 | 584 | The sixth step is to use Infrastructure as code (IaC) practices and tools to create all that we did in previous 585 | steps by typing one magic command. We will use **Terrafrom** to create infrastructure in GCP (VMs, cluster, firewall 586 | etc) + 587 | Terraform will use **Ansible** as configuration management tool to install all required software. There we will setup, 588 | configure and run of Selenium-grid, Selenoid(web/android). 589 | To do the same to run Gitlab or any CI system is still your 590 | homework :) 591 | 592 | **Preconditions:** 593 | * Installed [Terrafrom](https://learn.hashicorp.com/terraform/getting-started/install.html) 594 | * Installed [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) 595 | * Get [GCP project credential](https://cloud.google.com/community/tutorials/getting-started-on-gcp-with-terraform) 596 | (generate file `IaC/terraform/account.json` analogically with `IaC/terraform/account.example.json`) 597 | 598 | **Steps to execute tests:** 599 | 600 | * set correct variables: 601 | ``` 602 | - open IaC/terraform/variables.tf 603 | - set correct public_key_path, private_key_path, project_name, ssh_user 604 | ``` 605 | 606 | * setup Ansible config to turn off host checking to avoid script getting stuck 607 | ``` 608 | - open /etc/ansible/ansible.cfg or ~/.ansible.cfg file: 609 | 610 | [defaults] 611 | host_key_checking = False 612 | ``` 613 | 614 | * apply terraform 615 | ``` 616 | $ cd IaC/terraform/ 617 | $ terraform init 618 | $ terraform apply --auto-approve 619 | ``` 620 | 621 | 622 | output: 623 | ``` 624 | Cluster-Endpoint = 35.198.157.214 625 | Selenoid-Android = http://35.234.100.248:8081/#/ 626 | Selenoid-Android-ssh = ssh alexey@35.204.80.129 -i ~/.ssh/devopsTools 627 | Selenoid-Web = http://35.204.80.129:8081/#/ 628 | Selenoid-Web-ssh = ssh alexey@35.204.80.129 -i ~/.ssh/devopsTools 629 | ``` 630 | 631 | * apply terraform to setup selenium grid cluster 632 | ``` 633 | $ cd IaC/terraform/selenium-grid-k8s-resources 634 | $ terraform init 635 | $ terraform apply --auto-approve 636 | ``` 637 | 638 | output: 639 | ``` 640 | Selenium-Grid = http://35.198.143.162:4444/grid/console 641 | ``` 642 | 643 | That's all! It works like magic, save a lot of time and observable. You can start to run tests. Take your time to 644 | investigate *.tf files. 645 | 646 | * destroy all resources after work to avoid wasting of money 647 | ``` 648 | $ cd IaC/terraform/selenium-grid-k8s-resources 649 | $ terraform destroy --auto-approve 650 | 651 | $ cd IaC/terraform/ 652 | $ terraform destroy --auto-approve 653 | ``` 654 | 655 | **Links:** 656 | * [Terrafrom](https://www.terraform.io) 657 | * [Ansible](https://docs.ansible.com) 658 | * [Ansible and HashiCorp: Better Together](https://www.hashicorp.com/resources/ansible-terraform-better-together) 659 | * [How to use Ansible with Terraform](https://alex.dzyoba.com/blog/terraform-ansible/) 660 | * [Ansible Selenoid](https://github.com/SergeyPirogov/selenoid-ansible) instead of docker-compose that is used in this guide 661 | 662 | **What could be used instead:** 663 | 664 | Whatever you like again. Actually Ansible could be used instead of Terraform in a lot of cases and vice versa. But these 665 | tools have their specific destination. You can check comparison [here](https://blog.gruntwork.io/why-we-use-terraform-and-not-chef-puppet-ansible-saltstack-or-cloudformation-7989dad2865c). 666 | 667 | The most popular IaC tools: 668 | 669 | * [Chef](https://www.chef.io) 670 | * [Puppet](https://puppet.com) 671 | * [SaltStack](https://saltstack.com) 672 | * [CloudFormation](https://aws.amazon.com/cloudformation/) 673 | * [Vagrant](https://www.vagrantup.com) 674 | * [Spacelift](https://spacelift.io/) 675 | --------------------------------------------------------------------------------