├── .github ├── issue_template.md └── workflows │ └── node.js.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── buildBbcA11y.js ├── cleanBbcA11y.js ├── cli.js ├── config ├── .DS_Store ├── iplayer-web │ ├── README.md │ ├── all.js │ ├── app-atoz-test.js │ ├── app-bundles-test-signed-in.js │ ├── app-bundles-test.js │ ├── app-features-test.js │ ├── app-guide-test.js │ ├── app-highlights-test.js │ ├── app-lists-test-themed.js │ ├── app-lists-test.js │ ├── app-myprogrammes-test.js │ ├── app-playback-test.js │ └── common.js ├── sounds-web │ ├── README.md │ ├── all.js │ ├── category_pages.js │ ├── common.js │ ├── listen_page.js │ ├── my_sounds.js │ ├── playspace.js │ └── url.js └── test │ ├── just-paths.json │ ├── no-paths.json │ ├── paths-and-baseurl-and-visit-and-options.js │ ├── paths-and-baseurl.json │ ├── paths-with-baseurl-and-options.json │ ├── paths-with-signed-in-and-baseurl-and-options.json │ └── paths-with-signed-in-and-baseurl.json ├── lib ├── bbcA11YJUnitReporter.js ├── bbcA11y.js ├── colourfulLog.js ├── common.js ├── external.js ├── lighthouse.js ├── request.js └── xunitViewer.js ├── package-lock.json ├── package.json ├── runLighthouse.js └── test ├── fixtures ├── inspectableTargets.json └── lighthouseReport.json └── lib ├── bbcA11YJUnitReporter.js ├── bbcA11y.js ├── colourfulLog.js └── lighthouse.js /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | ## Summary 6 | 7 | 8 | 9 | ## Expected Behaviour 10 | 11 | 12 | 13 | 14 | 15 | ## Current Behaviour 16 | 17 | 18 | 19 | 20 | 21 | ~~~ 22 | ~~~ 23 | 24 | ## Possible Solution 25 | 26 | 27 | 28 | 29 | ## Your Environment 30 | 31 | 32 | * Version used: 33 | * Operating System and version: 34 | * Link to your project: 35 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | name: Node.js CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | strategy: 11 | matrix: 12 | node-version: [10.x, 12.x, 14.x] 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Use Node.js ${{ matrix.node-version }} 17 | uses: actions/setup-node@v1 18 | with: 19 | node-version: ${{ matrix.node-version }} 20 | - run: npm ci 21 | - run: npm test 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | node_modules/ 3 | a11y.js 4 | bbc-a11y-report.xml 5 | lighthouse-report.xml 6 | .nyc_output/ 7 | coverage/ 8 | 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This repo is **public**, avoid including any internal company URLs in issues and PRs. 4 | 5 | ## Submitting changes 6 | 7 | If you'd like to add your own config for your team, or contribute to this repo in some other way... 8 | * Fork the repo 9 | * Create a branch 10 | * Make your changes 11 | * Ensure the following jobs runs successfully (and fix any issues if not): 12 | * tests - `npm run test` 13 | * linting - `npm run lint` 14 | * Ensure the test coverage level is still good, by running `npm run coverage` 15 | * Update the README if necessary 16 | * Submit a PR 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-present British Broadcasting Corporation 2 | 3 | All rights reserved 4 | 5 | (http://www.bbc.co.uk) and contributors of dependencies 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Please contact us for an alternative licence 14 | 15 | Unless required by applicable law or agreed to in writing, software 16 | distributed under the License is distributed on an "AS IS" BASIS, 17 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | See the License for the specific language governing permissions and 19 | limitations under the License. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # a11y-tests-web 2 | 3 | ![Build Status](https://github.com/bbc/a11y-tests-web/workflows/Node.js%20CI/badge.svg) 4 | 5 | Uses [bbc-a11y](https://github.com/bbc/bbc-a11y) and [Google Lighthouse](https://developers.google.com/web/tools/lighthouse/) to run a suite of automated tests to test accessibility across a set of webpages, defined in a config file. 6 | 7 | ## Requirements 8 | - Node v10 or above 9 | - libgconf-2-4 10 | - Docker (if using the `ci` option) - NB The docker image is not always necessary to use bbc-a11y for continuous integration. For example, in TravisCI one option available is to prepend the run script with xvfb-run - [relevant TravisCI documentation](https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI). Furthermore, an alternative to using the docker image in Jenkins might be to use the [xvfb plugin](https://wiki.jenkins.io/display/JENKINS/Xvfb+Plugin) - though this is untested. 11 | 12 | ## Running anywhere from the command-line 13 | This package can be run anywhere from the command-line, making it easy to integrate with your existing projects. Here's how: 14 | 15 | 1. Install the package globally: `npm install @bbc/a11y-tests-web --global` 16 | 2. Run the command anywhere from your command-line, e.g. `A11Y_CONFIG=iplayer-web/all a11y-tests-web lighthouse -m junit-headless` 17 | 18 | You could also make it part of your application's scripts. Here's how: 19 | 1. Add the package to your application's dev dependencies: `npm install @bbc/a11y-tests-web --save-dev` 20 | 2. Add a line to your application's scripts e.g. `"test:a11y": "A11Y_CONFIG=iplayer-web/app-playback-test a11y-tests-web bbc-a11y -m interactive"` 21 | 22 | You can find out more about using the CLI option by running `a11y-tests-web --help`. 23 | 24 | ## Usage as an independent package 25 | 26 | ### Installation of dependencies 27 | 28 | ``` 29 | npm install 30 | ``` 31 | 32 | ### Run bbc-a11y using a config, e.g. iplayer-web/all 33 | 34 | To run bbc-a11y in interactive mode: 35 | 36 | ``` 37 | A11Y_CONFIG=iplayer-web/all npm run start:bbc-a11y 38 | ``` 39 | 40 | This will generate the commands for bbc-a11y and then run the tests against the pages listed in the iplayer-web/all config file in the config directory. 41 | 42 | ### Run bbc-a11y in headless mode 43 | 44 | To run bbc-a11y in headless mode: 45 | 46 | ``` 47 | A11Y_CONFIG=iplayer-web/all npm run start:bbc-a11y:headless 48 | ``` 49 | 50 | ### Run bbc-a11y and generate a JUnit report 51 | 52 | To generate a JUnit report, you can tell bbc-a11y to use the JUnit reporter: 53 | 54 | ``` 55 | A11Y_CONFIG=iplayer-web/all npm run start:bbc-a11y:junit 56 | ``` 57 | 58 | ### Run bbc-a11y and generate a JUnit report in headless mode 59 | 60 | To generate a JUnit report in headless mode: 61 | 62 | ``` 63 | A11Y_CONFIG=iplayer-web/all npm run start:bbc-a11y:junit-headless 64 | ``` 65 | 66 | ### Run bbc-a11y and generate a JUnit report using Docker 67 | 68 | If you don't have all the necessary libraries on your system required to run Electron, for example if you want to run this on a CI server, or if you want the process to always exit successfully, you can run this command to run them inside a Docker container and exit with success: 69 | 70 | ``` 71 | A11Y_CONFIG=iplayer-web/all npm run start:bbc-a11y:ci 72 | ``` 73 | 74 | Note that Docker obviously needs to be running and you can ignore any messages about XLib and libudev. 75 | 76 | ### Run Google Lighthouse and generate a JUnit report using a config, e.g. iplayer-web/all 77 | 78 | To run Google Lighthouse and generate a JUnit report: 79 | 80 | ``` 81 | A11Y_CONFIG=iplayer-web/all npm run start:lighthouse:junit 82 | ``` 83 | 84 | This will run the Google Lighthouse accessibility audit against the URLs defined in the iplayer-web/all config file, and generate a JUnit report called lighthouse-report.xml. 85 | 86 | If you'd like a more human readable report, you can simply use Google Chrome to run the audit, by opening dev tools and going to Audits. 87 | 88 | ### Run Google Lighthouse in headless mode and generate a JUnit report 89 | 90 | To run Google Lighthouse in headless mode and generate a JUnit report: 91 | 92 | ``` 93 | A11Y_CONFIG=iplayer-web/all npm run start:lighthouse:junit-headless 94 | ``` 95 | 96 | ### Run Google Lighthouse in headless mode and print results to console 97 | 98 | To run Google Lighthouse in headless mode and generate human-readable output to console: 99 | 100 | ``` 101 | A11Y_CONFIG=iplayer-web/all npm run start:lighthouse:headless 102 | ``` 103 | 104 | ### Run Google Lighthouse with verbose logging enabled 105 | 106 | To run Google Lighthouse in headless mode and enable verbose logging to console for Lighthouse: 107 | 108 | ``` 109 | A11Y_CONFIG=iplayer-web/all A11Y_LOGGING_LEVEL=verbose npm run start:lighthouse:headless 110 | ``` 111 | 112 | ## Running on Jenkins 113 | 114 | If you'd like to run this on your Jenkins server, ensure your Jenkins meets the requirements above and has a [JUnit plugin](https://plugins.jenkins.io/junit) installed and then: 115 | - Create a Jenkins job 116 | - Add this repo to the Jenkins job 117 | - Get the job to run `npm i --production` 118 | - Get the job to run the `start:bbc-a11y:ci` command for bbc-a11y or `start:lighthouse:junit-headless` for Lighthouse, with your `A11Y_CONFIG` 119 | - Add a post-build action to "Publish JUnit test results report" (or such like). The bbc-a11y XML file is called "bbc-a11y-report.xml" and the Lighthouse XML file is called "lighthouse-report.xml". 120 | 121 | ## Creating a config 122 | 123 | If your product/team does not already have a folder, create one in `config`. 124 | You then need to create a new file in this folder which should either be a JSON file or a JS file that exports an object. 125 | 126 | The data should include: 127 | - `options` - Object - Options as defined by bbc-a11y, e.g. hide, skip and visit. Note that skip and visit are currently **ignored** by Google Lighthouse, but hide is used. 128 | - `baseUrl` - String - The domain to run the tests against, e.g. "https://www.bbc.co.uk" 129 | - `paths` - Array - The paths on that domain to run the tests against 130 | - `signedInPaths` - Array - An optional list of paths to run the tests against, after signing in to BBC ID. 131 | 132 | Note that if you have a list of `signedInPaths`, the username and password to use when logging in to BBC ID should be specified using the environment variables A11Y_USERNAME and A11Y_PASSWORD. 133 | 134 | ## Contributing 135 | See [CONTRIBUTING.md](https://github.com/bbc/a11y-tests-web/blob/master/CONTRIBUTING.md) 136 | -------------------------------------------------------------------------------- /buildBbcA11y.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { build } = require('./lib/bbcA11y'); 4 | build(); 5 | -------------------------------------------------------------------------------- /cleanBbcA11y.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { clean } = require('./lib/bbcA11y'); 4 | clean(); 5 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict'; 3 | 4 | const { spawn } = require('child_process'); 5 | const path = require('path'); 6 | const yargs = require('yargs'); 7 | const { clean: cleanBbcA11y, build: buildBbcA11y } = require('./lib/bbcA11y'); 8 | const { run: runLighthouse } = require('./lib/lighthouse'); 9 | const logger = require('./lib/colourfulLog'); 10 | 11 | yargs 12 | .usage('Usage: A11Y_CONFIG= $0 [options] -- [tool options]') 13 | .command('bbc-a11y', 'Runs bbc-a11y', runBbcA11yCli) 14 | .command('lighthouse', 'Runs Google Lighthouse', runLighthouseCli) 15 | .demandCommand() 16 | .alias('h', 'help') 17 | .alias('m', 'mode') 18 | .describe('m', 'The mode to run the specified tool in') 19 | .choices('m', ['interactive', 'headless', 'junit', 'junit-headless', 'ci']) 20 | .demandOption('m', 'You must specify a mode to run the tool in, e.g. junit-headless') 21 | .argv; 22 | 23 | async function runBbcA11yCli({ argv }) { 24 | buildBbcA11y(); 25 | 26 | const { mode } = argv; 27 | const runBbcA11y = mode === 'ci' ? runBbcA11yInCiMode : runBbcA11yOnHost; 28 | 29 | const bbcA11yExitCode = await runBbcA11y(argv); 30 | 31 | cleanBbcA11y(); 32 | process.exit(bbcA11yExitCode); 33 | } 34 | 35 | function runLighthouseCli({ argv }) { 36 | const { mode } = argv; 37 | const validModes = ['junit', 'junit-headless', 'headless']; 38 | 39 | if (validModes.includes(mode)) { 40 | if (mode === 'junit-headless' || mode === 'headless') { 41 | process.env.A11Y_HEADLESS = true; 42 | } 43 | if (mode === 'headless') { 44 | process.env.A11Y_PRETTY = 'true'; 45 | } 46 | return runLighthouse(); 47 | } 48 | 49 | logger.error(`Unknown mode. Lighthouse is only supported in the following modes: ${validModes.join(', ')}`); 50 | } 51 | 52 | function getBbcA11yArgs(mode) { 53 | switch (mode) { 54 | case 'headless': 55 | return []; 56 | case 'junit': 57 | return ['--interactive', '--reporter', './lib/bbcA11YJUnitReporter.js']; 58 | case 'junit-headless': 59 | return ['--reporter', './lib/bbcA11YJUnitReporter.js']; 60 | default: 61 | return ['--interactive']; 62 | } 63 | } 64 | 65 | function runBbcA11yOnHost(argv) { 66 | return new Promise((resolve) => { 67 | const { mode } = argv; 68 | const bbcA11yArgs = getBbcA11yArgs(mode); 69 | const configFile = path.resolve(`${__dirname}/a11y.js`); 70 | const bbcA11y = spawn('bbc-a11y', ['--config', configFile, ...bbcA11yArgs, ...argv._.slice(1)]); 71 | bbcA11y.stdout.pipe(process.stdout); 72 | bbcA11y.stderr.pipe(process.stderr); 73 | 74 | bbcA11y.on('close', (exitCode) => { 75 | resolve(exitCode); 76 | }); 77 | }); 78 | } 79 | 80 | function runBbcA11yInCiMode(argv) { 81 | const dockerArgs = ['run', '--rm', '--tty', '--volume', `${__dirname}:/ws`, 'bbca11y/bbc-a11y-docker', '--config', '/ws/a11y.js', '--reporter', '/ws/lib/bbcA11YJUnitReporter.js']; 82 | return new Promise((resolve) => { 83 | const bbcA11yDocker = spawn('docker', [...dockerArgs, ...argv._.slice(1)]); 84 | bbcA11yDocker.stdout.pipe(process.stdout); 85 | bbcA11yDocker.stderr.pipe(process.stderr); 86 | 87 | bbcA11yDocker.on('close', () => { 88 | resolve(0); 89 | }); 90 | }); 91 | } 92 | -------------------------------------------------------------------------------- /config/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bbc/a11y-tests-web/8cc223ba8d75a3afa1e30b38ac90cba559fbda10/config/.DS_Store -------------------------------------------------------------------------------- /config/iplayer-web/README.md: -------------------------------------------------------------------------------- 1 | # iPlayer Web Config 2 | 3 | ## Usage 4 | 5 | As well as all the normal usage options, the iPlayer Web tests can be pointed at different `baseUrl`s using the `A11Y_IPLAYER_WEB_BASE_URL` environment variable. For example: 6 | 7 | 8 | ``` 9 | A11Y_CONFIG=iplayer-web/all A11Y_IPLAYER_WEB_BASE_URL=https://some.url npm run start:bbc-a11y 10 | ``` 11 | 12 | We can also provide the username and password to run against the signed in pages 13 | 14 | For bbc-a11y 15 | ``` 16 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_IPLAYER_WEB_BASE_URL=https://some.url A11Y_CONFIG=iplayer-web/all npm run start:bbc-a11y:headless 17 | ``` 18 | 19 | For lighthouse 20 | ``` 21 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_IPLAYER_WEB_BASE_URL=https://some.url A11Y_CONFIG=iplayer-web/all npm run start:lighthouse:junit-headless 22 | ``` 23 | 24 | We have also added a way to execute against particular page, the below example will run against iPlayer homepage: 25 | 26 | For devs to run bbc-a11y on sandbox : 27 | ``` 28 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_IPLAYER_WEB_BASE_URL=https://sandbox.bbc.co.uk A11Y_CONFIG=iplayer-web/app-bundles-test npm run start:bbc-a11y:headless 29 | ``` 30 | 31 | For devs to run lighthouse on sandbox : 32 | ``` 33 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_IPLAYER_WEB_BASE_URL=https://sandbox.bbc.co.uk A11Y_CONFIG=iplayer-web/app-bundles-test npm run start:lighthouse:junit-headless 34 | ``` 35 | -------------------------------------------------------------------------------- /config/iplayer-web/all.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const features = require('./app-features-test'); 4 | const highlights = require('./app-highlights-test'); 5 | const bundles = require('./app-bundles-test'); 6 | const lists = require('./app-lists-test'); 7 | const myprogrammes = require('./app-myprogrammes-test'); 8 | const playback = require('./app-playback-test'); 9 | const guide = require('./app-guide-test'); 10 | const atoz = require('./app-atoz-test'); 11 | const { options } = require('./common'); 12 | 13 | const baseUrl = process.env.A11Y_IPLAYER_WEB_BASE_URL || 'https://www.bbc.co.uk'; 14 | 15 | module.exports = { 16 | options, 17 | baseUrl, 18 | paths: [ 19 | ...features.paths, 20 | ...highlights.paths, 21 | ...bundles.paths, 22 | ...lists.paths, 23 | ...myprogrammes.paths, 24 | ...playback.paths, 25 | ...guide.paths, 26 | ...atoz.paths, 27 | '/bbcfour/collections' 28 | ], 29 | signedInPaths: [ 30 | ...bundles.signedInPaths, 31 | ...myprogrammes.signedInPaths 32 | ] 33 | }; 34 | -------------------------------------------------------------------------------- /config/iplayer-web/app-atoz-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options, baseUrl } = require('./common'); 4 | 5 | module.exports = { 6 | options, 7 | baseUrl, 8 | paths: [ 9 | '/iplayer/a-z/a' 10 | ] 11 | }; 12 | -------------------------------------------------------------------------------- /config/iplayer-web/app-bundles-test-signed-in.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options: commonOptions, baseUrl } = require('./common'); 4 | 5 | const commonSkips = commonOptions.skip; 6 | 7 | // Temporarily skip test for the dropdown component on the category pages 8 | const listSpecificSkips = ['Forms: Managing focus: Forms must have submit buttons']; 9 | 10 | const options = Object.assign({}, commonOptions, 11 | { 12 | skip: [...commonSkips, ...listSpecificSkips] 13 | } 14 | ); 15 | 16 | module.exports = { 17 | options, 18 | baseUrl, 19 | signedInPaths: [ 20 | '/iplayer', 21 | '/iplayer/categories/arts/featured' 22 | ] 23 | }; 24 | -------------------------------------------------------------------------------- /config/iplayer-web/app-bundles-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const bundlesSignedIn = require('./app-bundles-test-signed-in'); 4 | 5 | module.exports = { 6 | ...bundlesSignedIn, 7 | paths: [ 8 | '/iplayer', 9 | '/iplayer/categories/arts/featured' 10 | ] 11 | }; 12 | -------------------------------------------------------------------------------- /config/iplayer-web/app-features-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options, baseUrl } = require('./common'); 4 | 5 | module.exports = { 6 | options, 7 | baseUrl, 8 | paths: [ 9 | '/iplayer/guidance' 10 | ] 11 | }; 12 | -------------------------------------------------------------------------------- /config/iplayer-web/app-guide-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options, baseUrl } = require('./common'); 4 | 5 | module.exports = { 6 | options, 7 | baseUrl, 8 | paths: [ 9 | '/iplayer/guide' 10 | ] 11 | }; 12 | -------------------------------------------------------------------------------- /config/iplayer-web/app-highlights-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options, baseUrl } = require('./common'); 4 | 5 | module.exports = { 6 | options, 7 | baseUrl, 8 | paths: [ 9 | '/bbcone', 10 | '/bbctwo', 11 | '/tv/bbcthree', 12 | '/bbcfour', 13 | '/tv/radio1', 14 | '/tv/cbbc', 15 | '/tv/cbeebies', 16 | '/tv/bbcnews', 17 | '/tv/bbcparliament', 18 | '/tv/bbcalba', 19 | '/tv/s4c', 20 | '/tv/bbcscotland' 21 | ] 22 | }; 23 | -------------------------------------------------------------------------------- /config/iplayer-web/app-lists-test-themed.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options, baseUrl } = require('./common'); 4 | 5 | module.exports = { 6 | options, 7 | baseUrl, 8 | signedInPaths: [ 9 | '/iplayer/group/u13-entertainment', 10 | '/iplayer/episodes/b006m86d' 11 | ] 12 | }; 13 | -------------------------------------------------------------------------------- /config/iplayer-web/app-lists-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options: commonOptions, baseUrl } = require('./common'); 4 | 5 | const commonSkips = commonOptions.skip; 6 | 7 | // Temporarily skip test for the dropdown component on the category pages 8 | const listSpecificSkips = ['Forms: Managing focus: Forms must have submit buttons']; 9 | 10 | const options = Object.assign({}, commonOptions, 11 | { 12 | skip: [...commonSkips, ...listSpecificSkips] 13 | } 14 | ); 15 | 16 | module.exports = { 17 | options, 18 | baseUrl, 19 | paths: [ 20 | '/bbcone/a-z', 21 | '/iplayer/most-popular', 22 | '/iplayer/categories/arts/a-z', 23 | '/iplayer/categories/arts/most-recent', 24 | '/iplayer/search?q=east', 25 | '/iplayer/episodes/b006m86d' 26 | ] 27 | }; 28 | -------------------------------------------------------------------------------- /config/iplayer-web/app-myprogrammes-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options, baseUrl } = require('./common'); 4 | 5 | module.exports = { 6 | options, 7 | baseUrl, 8 | paths: [ 9 | '/iplayer/recommendations', 10 | '/iplayer/watchlist', 11 | '/iplayer/continue-watching' 12 | ], 13 | signedInPaths: [ 14 | '/iplayer/recommendations', 15 | '/iplayer/watchlist', 16 | '/iplayer/continue-watching' 17 | ] 18 | }; 19 | -------------------------------------------------------------------------------- /config/iplayer-web/app-playback-test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options: commonOptions, baseUrl } = require('./common'); 4 | 5 | const commonSkips = commonOptions.skip; 6 | 7 | // Following is awaiting this being fixed: https://github.com/bbc/bbc-a11y/issues/238 8 | const playbackSpecificSkips = ['Structure: Headings: Content must follow headings']; 9 | 10 | const options = Object.assign({}, commonOptions, 11 | { 12 | skip: [...commonSkips, ...playbackSpecificSkips] 13 | } 14 | ); 15 | 16 | module.exports = { 17 | options, 18 | baseUrl, 19 | paths: [ 20 | '/iplayer/episode/p04qj936/face-to-face-adam-faith' 21 | ] 22 | }; 23 | -------------------------------------------------------------------------------- /config/iplayer-web/common.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const baseUrl = process.env.A11Y_IPLAYER_WEB_BASE_URL || 'https://sandbox.bbc.co.uk'; 4 | 5 | // Skipped tests are those for which we have tickets prioritised in the backlog to fix 6 | 7 | module.exports = { 8 | options: { 9 | hide: [ 10 | 'orb', 11 | 'bbccookies-prompt', 12 | 'smphtml5iframebbcMediaPlayer', 13 | 'edr_l_first', 14 | 'edr_lwrap_first' 15 | ], 16 | skip: [ 17 | 'Text equivalents: Visual formatting: Use tables for data', 18 | 'Design: Content resizing: Text must be styled with units that are resizable in all browsers' 19 | ] 20 | }, 21 | baseUrl 22 | }; 23 | -------------------------------------------------------------------------------- /config/sounds-web/README.md: -------------------------------------------------------------------------------- 1 | # BBC Sounds Web Config 2 | 3 | ## Usage 4 | 5 | As well as all the normal usage options, the BBC Sounds Web tests can be pointed at different `baseUrl`s using the `A11Y_SOUNDS_WEB_BASE_URL` environment variable. For example: 6 | 7 | 8 | ``` 9 | A11Y_SOUNDS_WEB_BASE_URL=https://some.url A11Y_CONFIG=sounds-web/all npm run start:bbc-a11y:headless 10 | ``` 11 | 12 | We can also provide the username and password to run against the signed in pages 13 | 14 | For bbc-a11y 15 | ``` 16 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_SOUNDS_WEB_BASE_URL=https://some.url A11Y_CONFIG=sounds-web/all npm run start:bbc-a11y:headless 17 | ``` 18 | 19 | For lighthouse 20 | ``` 21 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_SOUNDS_WEB_BASE_URL=https://some.url A11Y_CONFIG=sounds-web/all npm run start:lighthouse:junit-headless 22 | ``` 23 | 24 | We have also added a way to execute against particular page, the below example will run against Listen Page: 25 | 26 | For devs to run bbc-a11y on localhost : 27 | ``` 28 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_SOUNDS_WEB_BASE_URL=https://localhost.bbc.co.uk A11Y_CONFIG=sounds-web/listen_page npm run start:bbc-a11y:headless 29 | ``` 30 | 31 | For devs to run lighthouse on localhost : 32 | ``` 33 | A11Y_USERNAME=user A11Y_PASSWORD=password A11Y_SOUNDS_WEB_BASE_URL=https://localhost.bbc.co.uk A11Y_CONFIG=sounds-web/listen_page npm run start:lighthouse:junit-headless 34 | ``` 35 | ### Single Web page 36 | 37 | To test a single webpage (while also including the Sounds Web specific settings), run using the 'url' config, replacing the url with your own: 38 | ``` 39 | A11Y_URL='http://www.google.com' A11Y_CONFIG=sounds-web/url a11y-tests-web bbc-a11y -m headless 40 | ``` 41 | 42 | -------------------------------------------------------------------------------- /config/sounds-web/all.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const baseUrl = process.env.A11Y_SOUNDS_WEB_BASE_URL || 'https://www.bbc.co.uk'; 4 | 5 | const { options } = require('./common'); 6 | 7 | const listenPage = require('./listen_page'); 8 | const playspace = require('./playspace'); 9 | const mySounds = require('./my_sounds'); 10 | const category = require('./category_pages'); 11 | 12 | module.exports = { 13 | options, 14 | baseUrl, 15 | paths: [ 16 | ...listenPage.paths, 17 | ...playspace.paths, 18 | ...mySounds.paths, 19 | ...category.paths 20 | ], 21 | signedInPaths: [ 22 | ...listenPage.signedInPaths, 23 | ...playspace.signedInPaths, 24 | ...mySounds.signedInPaths, 25 | ...category.signedInPaths 26 | ] 27 | }; 28 | -------------------------------------------------------------------------------- /config/sounds-web/category_pages.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const baseUrl = process.env.A11Y_SOUNDS_WEB_BASE_URL || 'https://www.bbc.co.uk'; 4 | 5 | const { options } = require('./common'); 6 | 7 | module.exports = { 8 | options, 9 | baseUrl, 10 | paths: [ 11 | '/sounds/brand/b006tnxf', 12 | '/sounds/categories', 13 | '/sounds/category/music-rockandindie', 14 | '/sounds/category/comedy' 15 | ], 16 | signedInPaths: [ 17 | '/sounds/brand/b006tnxf', 18 | '/sounds/categories', 19 | '/sounds/category/music-rockandindie', 20 | '/sounds/category/comedy' 21 | ] 22 | }; 23 | -------------------------------------------------------------------------------- /config/sounds-web/common.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const baseUrl = process.env.A11Y_SOUNDS_WEB_BASE_URL || 'https://www.bbc.co.uk'; 4 | 5 | module.exports = { 6 | options: { 7 | hide: [ 8 | 'orb', 9 | 'bbccookies-prompt', 10 | 'smphtml5iframebbcMediaPlayer', 11 | 'smphtml5iframesmp-wrapper', 12 | 'edr_l_first', 13 | 'bbcprivacy-prompt', 14 | 'id4-cta-', 15 | 'msi-modal', 16 | 'p_audioui_', 17 | 'Non-focusable element with tabindex=0: //div' // no way around having divs with role button for layout reasons 18 | ], 19 | skip: [] 20 | }, 21 | baseUrl 22 | }; 23 | -------------------------------------------------------------------------------- /config/sounds-web/listen_page.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const baseUrl = process.env.A11Y_SOUNDS_WEB_BASE_URL || 'https://www.bbc.co.uk'; 4 | 5 | const { options } = require('./common'); 6 | 7 | module.exports = { 8 | options, 9 | baseUrl, 10 | paths: [ 11 | '/sounds' 12 | ], 13 | signedInPaths: [ 14 | '/sounds' 15 | ] 16 | }; 17 | -------------------------------------------------------------------------------- /config/sounds-web/my_sounds.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const baseUrl = process.env.A11Y_SOUNDS_WEB_BASE_URL || 'https://www.bbc.co.uk'; 4 | 5 | const { options } = require('./common'); 6 | 7 | module.exports = { 8 | options, 9 | baseUrl, 10 | paths: [ 11 | '/sounds/my', 12 | '/sounds/my/bookmarks', 13 | '/sounds/my/subscribed' 14 | ], 15 | signedInPaths: [ 16 | '/sounds/my', 17 | '/sounds/my/bookmarks', 18 | '/sounds/my/subscribed' 19 | ] 20 | }; 21 | -------------------------------------------------------------------------------- /config/sounds-web/playspace.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const baseUrl = process.env.A11Y_SOUNDS_WEB_BASE_URL || 'https://www.bbc.co.uk'; 4 | 5 | const { options } = require('./common'); 6 | 7 | module.exports = { 8 | options, 9 | baseUrl, 10 | paths: [ 11 | '/sounds/play/series:b006qftk', 12 | '/sounds/play/live:bbc_radio_one', 13 | '/sounds/play/p08jhp3y' 14 | ], 15 | signedInPaths: [ 16 | '/sounds/play/series:b006qftk', 17 | '/sounds/play/live:bbc_radio_one', 18 | '/sounds/play/p08jhp3y' 19 | ] 20 | }; 21 | -------------------------------------------------------------------------------- /config/sounds-web/url.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { options } = require('./common'); 4 | 5 | module.exports = { 6 | options, 7 | baseUrl: process.env.A11Y_URL, 8 | paths: [''] 9 | }; 10 | 11 | // can be used to test single url passed in: 12 | // A11Y_URL='http://www.google.com' A11Y_CONFIG=sounds-web/url a11y-tests-web bbc-a11y -m headless 13 | -------------------------------------------------------------------------------- /config/test/just-paths.json: -------------------------------------------------------------------------------- 1 | { 2 | "paths": [ 3 | "/path/1", 4 | "/path/2" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /config/test/no-paths.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "http://base.url" 3 | } 4 | -------------------------------------------------------------------------------- /config/test/paths-and-baseurl-and-visit-and-options.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | baseUrl: 'http://base.url', 5 | paths: [ 6 | '/path/1', 7 | '/path/2' 8 | ], 9 | options: { 10 | some: 'option', 11 | visit: function () { 12 | /* Do something */ 13 | } 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /config/test/paths-and-baseurl.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "http://base.url", 3 | "paths": [ 4 | "/path/1", 5 | "/path/2" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /config/test/paths-with-baseurl-and-options.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "http://base.url", 3 | "paths": [ 4 | "/path/1", 5 | "/path/2" 6 | ], 7 | "options": { 8 | "some": "option", 9 | "hide": ["dodgy-svg"] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /config/test/paths-with-signed-in-and-baseurl-and-options.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "http://base.url", 3 | "paths": [ 4 | "/path/1", 5 | "/path/2" 6 | ], 7 | "signedInPaths": [ 8 | "/path/3", 9 | "/path/4" 10 | ], 11 | "options": { 12 | "some": "option" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /config/test/paths-with-signed-in-and-baseurl.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "http://base.url", 3 | "paths": [ 4 | "/path/1", 5 | "/path/2" 6 | ], 7 | "signedInPaths": [ 8 | "/path/3", 9 | "/path/4" 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /lib/bbcA11YJUnitReporter.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const builder = require('junit-report-builder'); 4 | const fs = require('fs'); 5 | 6 | function JUnitReport(devToolsConsole, commandLineConsole) { 7 | this.devToolsConsole = devToolsConsole; 8 | this.commandLineConsole = commandLineConsole; 9 | } 10 | 11 | JUnitReport.prototype.runStarted = function () {}; 12 | 13 | JUnitReport.prototype.pageChecked = function (page, validationResult) { 14 | const suiteName = page.url.replace(/.*?:\/\//g, '').replace('\/', './'); 15 | const suite = builder.testSuite(); 16 | suite.name(suiteName); 17 | 18 | validationResult.results.forEach((standardResult) => { 19 | const standard = standardResult.standard; 20 | const testName = standard.section.title + ': ' + standard.name; 21 | const docsUrl = standard.section.documentationUrl; 22 | const testCase = suite.testCase(); 23 | testCase.className(suiteName); 24 | testCase.name(testName); 25 | 26 | if (standardResult.errors && standardResult.errors.length) { 27 | const errors = standardResult.errors.map(prettyErrorOutput); 28 | testCase.failure('Error on ' + page.url + errors.join('') + '\nMore info at ' + docsUrl); 29 | } 30 | }); 31 | }; 32 | 33 | JUnitReport.prototype.pagePassed = function () {}; 34 | JUnitReport.prototype.pageFailed = function () {}; 35 | 36 | JUnitReport.prototype.runEnded = function () { 37 | const output = builder.build(); 38 | fs.writeFileSync(__dirname + '/../bbc-a11y-report.xml', output); 39 | this.log(output); 40 | }; 41 | 42 | JUnitReport.prototype.unexpectedError = function (error) { 43 | this.log('Unexpected error running tests: ' + error.stack); 44 | }; 45 | 46 | JUnitReport.prototype.configMissing = function () { 47 | this.log('Error running tests: Missing configuration file'); 48 | }; 49 | 50 | JUnitReport.prototype.configError = function (error) { 51 | this.log('Config error whilst running tests: ' + error.stack); 52 | }; 53 | 54 | JUnitReport.prototype.log = function (message) { 55 | this.commandLineConsole.log(message); 56 | }; 57 | 58 | function prettyErrorOutput(error) { 59 | const errorDetails = (error[1] && error[1].xpath) || ''; 60 | return '\n' + error[0] + ' ' + errorDetails; 61 | } 62 | 63 | module.exports = JUnitReport; 64 | -------------------------------------------------------------------------------- /lib/bbcA11y.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const fs = require('fs'); 4 | const path = require('path'); 5 | const logger = require('./colourfulLog'); 6 | const { 7 | getConfigName, 8 | getConfig, 9 | getSignedInPaths, 10 | getSignInCredentials 11 | } = require('./common'); 12 | 13 | const configFilePath = path.resolve(`${__dirname}/../a11y.js`); 14 | 15 | function build() { 16 | const configName = getConfigName(); 17 | const config = getConfig(configName); 18 | const baseUrl = config.baseUrl || 'http://www.bbc.co.uk'; 19 | const domain = baseUrl.replace(/(https?:\/\/)/, ''); 20 | const paths = config.paths || []; 21 | const signedInPaths = getSignedInPaths(config); 22 | 23 | if (!paths.length && !signedInPaths.length) { 24 | logger.error(`No paths listed in the config for ${configName}`); 25 | process.exit(1); 26 | } 27 | 28 | const signedOutOutput = pathsToOutput(baseUrl, paths, config.options); 29 | const signedInOutput = pathsToOutput(baseUrl, signedInPaths, config.options, true); 30 | const a11yOutput = ` 31 | ${signedOutOutput} 32 | ${signedInOutput} 33 | `; 34 | 35 | fs.writeFileSync(configFilePath, a11yOutput); 36 | 37 | logger.log(`Tests will run against: ${domain} ${paths.join(' ')}`); 38 | if (signedInPaths.length) { 39 | logger.log(`Tests will run signed in against: ${domain} ${signedInPaths.join(' ')}`); 40 | } 41 | } 42 | 43 | function clean() { 44 | try { 45 | fs.unlinkSync(configFilePath); 46 | } catch (e) { 47 | logger.log('Error cleaning but this should be OK', e); 48 | // Cannot delete file (probably because it does not exist). 49 | } 50 | } 51 | 52 | function pathToOutput(baseUrl, path, options = {}, signedIn) { 53 | const visitOptions = getVisitOption(baseUrl, path, signedIn, options); 54 | 55 | return ` 56 | page( 57 | "${baseUrl}${path}", 58 | { 59 | ${visitOptions} 60 | ${JSON.stringify(options).slice(1, -1)} 61 | } 62 | ) 63 | `; 64 | } 65 | 66 | function getVisitOption(baseUrl, path, signedIn, options) { 67 | if (signedIn) { 68 | const { username, password } = getSignInCredentials(); 69 | const url = baseUrl + path; 70 | const encodedUrl = encodeURIComponent(url); 71 | return `visit: function (frame) { 72 | frame.src = 'https://account.bbc.com/auth?ptrt=${encodedUrl}'; 73 | return new Promise(function (test) { 74 | frame.onload = function () { 75 | var loginPage = frame.contentDocument; 76 | loginPage.getElementById('user-identifier-input').value = '${username}'; 77 | loginPage.getElementById('submit-button').click(); 78 | loginPage.getElementById('password-input').value = '${password}'; 79 | loginPage.getElementById('submit-button').click(); 80 | frame.onload = test 81 | } 82 | }) 83 | },`; 84 | } 85 | 86 | if (options.visit) { 87 | return `visit: ${options.visit.toString()},`; 88 | } 89 | 90 | return ''; 91 | } 92 | 93 | function pathsToOutput(baseUrl, paths, options, signedIn = false) { 94 | return paths.reduce( 95 | (acc, path) => acc + pathToOutput(baseUrl, path, options, signedIn), 96 | '' 97 | ); 98 | } 99 | 100 | module.exports = { 101 | build, 102 | clean 103 | }; 104 | -------------------------------------------------------------------------------- /lib/colourfulLog.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const ICONS = { 4 | ok: '\x1b[32m✔\x1b[0m', 5 | error: '\x1b[31m✘\x1b[0m', 6 | warning: '\x1b[33m!\x1b[0m' 7 | }; 8 | 9 | const COLOURS = { 10 | log: '\x1b[2m', 11 | ok: '\x1b[2m', 12 | warning: '\x1b[33m', 13 | error: '\x1b[31m' 14 | }; 15 | 16 | function getIcon(type) { 17 | return ICONS[type] ? ICONS[type] + ' ' : ''; 18 | } 19 | 20 | function getMessage(text, type) { 21 | const color = COLOURS[type] ? COLOURS[type] : ''; 22 | return `${color}${text}\x1b[0m\n`; 23 | } 24 | 25 | function output(text, type) { 26 | process.stdout.write(`${getIcon(type)}${getMessage(text, type)}`); 27 | } 28 | 29 | module.exports = { 30 | log: (text) => output(text), 31 | ok: (text) => output(text, 'ok'), 32 | warning: (text) => output(text, 'warning'), 33 | error: (text) => output(text, 'error') 34 | }; 35 | -------------------------------------------------------------------------------- /lib/common.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const logger = require('./colourfulLog'); 4 | 5 | function getConfigName() { 6 | const configName = process.env.A11Y_CONFIG; 7 | 8 | if (!configName) { 9 | logger.error('No config selected. Use the A11Y_CONFIG environment variable to set one.'); 10 | process.exit(1); 11 | } 12 | 13 | return configName; 14 | } 15 | 16 | function getConfig(configName) { 17 | try { 18 | return require(`../config/${configName}`); 19 | } catch (e) { 20 | logger.error(`Could not find a valid config named ${configName}`); 21 | process.exit(1); 22 | } 23 | 24 | return {}; 25 | } 26 | 27 | function getSignInCredentials() { 28 | const { A11Y_USERNAME: username, A11Y_PASSWORD: password } = process.env; 29 | return { username, password }; 30 | } 31 | 32 | function getSignedInPaths(config) { 33 | const paths = config.signedInPaths || []; 34 | 35 | const { username, password } = getSignInCredentials(); 36 | if (paths.length && (!username || !password)) { 37 | logger.warning('Skipping signed in paths because a username and/or password were not specified. (Use A11Y_USERNAME and A11Y_PASSWORD environment variables to set them)'); 38 | return []; 39 | } 40 | 41 | return paths; 42 | } 43 | 44 | function getLoggingLevel() { 45 | const { A11Y_LOGGING_LEVEL: loggingLevel } = process.env; 46 | 47 | if (loggingLevel === 'verbose') { 48 | return loggingLevel; 49 | } 50 | 51 | return 'silent'; 52 | } 53 | 54 | module.exports = { 55 | getConfigName, 56 | getConfig, 57 | getSignedInPaths, 58 | getSignInCredentials, 59 | getLoggingLevel 60 | }; 61 | -------------------------------------------------------------------------------- /lib/external.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const CDP = require('chrome-remote-interface'); 4 | const lighthouse = require('lighthouse'); 5 | 6 | module.exports = { 7 | CDP, 8 | lighthouse 9 | }; 10 | -------------------------------------------------------------------------------- /lib/lighthouse.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const chromeLauncher = require('chrome-launcher'); 4 | const external = require('./external'); 5 | const fs = require('fs'); 6 | const reportBuilder = require('junit-report-builder'); 7 | const xunitViewer = require('./xunitViewer'); 8 | 9 | const logger = require('./colourfulLog'); 10 | const { 11 | getConfigName, 12 | getConfig, 13 | getSignedInPaths, 14 | getSignInCredentials, 15 | getLoggingLevel 16 | } = require('./common'); 17 | 18 | const request = require('./request'); 19 | 20 | const LIGHTHOUSE_OPTS = { 21 | config: { 22 | extends: 'lighthouse:default', 23 | settings: { 24 | onlyCategories: ['accessibility'] 25 | }, 26 | categories: { 27 | accessibility: { 28 | weight: 1 29 | } 30 | } 31 | }, 32 | flags: { 33 | logLevel: getLoggingLevel(), 34 | output: 'json' 35 | } 36 | }; 37 | 38 | function run() { 39 | const configName = getConfigName(); 40 | const config = getConfig(configName); 41 | const isPretty = process.env.A11Y_PRETTY === 'true'; 42 | const baseUrl = config.baseUrl || 'http://www.bbc.co.uk'; 43 | const domain = baseUrl.replace(/(https?:\/\/)/, ''); 44 | const paths = config.paths || []; 45 | const hide = config.options && config.options.hide || []; 46 | const signedInPaths = getSignedInPaths(config); 47 | 48 | if (!paths.length && !signedInPaths.length) { 49 | logger.error(`No paths listed in the config for ${configName}`); 50 | process.exit(1); 51 | } 52 | 53 | const signedOutTasks = paths.map((path) => () => runTestsForUrl({ url: baseUrl + path, shouldSignIn: false, hide, isPretty })); 54 | const signedInTasks = signedInPaths.map((path) => () => runTestsForUrl({ url: baseUrl + path, shouldSignIn: true, hide, isPretty })); 55 | 56 | const tasks = [...signedOutTasks, ...signedInTasks]; 57 | 58 | logger.log(`Tests will run against: ${domain} ${paths.join(' ')}`); 59 | if (signedInPaths.length) { 60 | logger.log(`Tests will run signed in against: ${domain} ${signedInPaths.join(' ')}`); 61 | } 62 | 63 | return executeSequentially(tasks) 64 | .then(() => { 65 | const output = reportBuilder.build(); 66 | fs.writeFileSync(`${process.cwd()}/lighthouse-report.xml`, output); 67 | if (!isPretty) { 68 | logger.log(output); 69 | } else { 70 | xunitViewer.toConsole(); 71 | } 72 | }); 73 | } 74 | 75 | async function pause(time) { 76 | return new Promise((resolve) => { 77 | setTimeout(resolve, time); 78 | }); 79 | } 80 | 81 | /** 82 | * Wait for the Chrome DevTools Protocol (CDP) to have at least one inspectable 83 | * target. The CDP needs to be ready before attempting to use it. 84 | */ 85 | async function waitForInspectableTarget(chromeInstance, attempt = 1) { 86 | const MAX_NUMBER_OF_ATTEMPTS = 50; 87 | 88 | async function checkEndpointReturnsTargets() { 89 | const response = await request.get(`http://127.0.0.1:${chromeInstance.port}/json/list`); 90 | const targets = JSON.parse(response); 91 | return targets.length > 0; 92 | } 93 | 94 | return new Promise(async (resolve, reject) => { 95 | if (attempt > MAX_NUMBER_OF_ATTEMPTS) { 96 | return reject('Failed to find inspectable target, max attempts to connect reached'); 97 | } 98 | 99 | const isReady = await checkEndpointReturnsTargets(); 100 | 101 | if (!isReady) { 102 | logger.warning('Failed to find inspectable target, retrying...'); 103 | 104 | try { 105 | await pause(300); 106 | await waitForInspectableTarget(chromeInstance, attempt + 1); 107 | 108 | // Need to invoke resolve (or reject) to unblock 109 | // `waitForInspectableTarget` above us in the recursion stack. 110 | return resolve(); 111 | } catch (err) { 112 | return reject(err); 113 | } 114 | } 115 | 116 | resolve(); 117 | }); 118 | } 119 | 120 | async function runTestsForUrl({ url, shouldSignIn, hide, isPretty }) { 121 | logger.log(`Running audit for ${url}`); 122 | 123 | try { 124 | const results = await launchChromeAndRunLighthouse(url, shouldSignIn); 125 | const suiteName = url.replace(/.*?:\/\//g, '').replace('\/', './'); 126 | const suite = reportBuilder.testSuite(); 127 | suite.name(suiteName); 128 | 129 | const { audits, timing: { total: suiteDuration } } = results; 130 | 131 | suite.time(suiteDuration); 132 | 133 | const auditKeys = Object.keys(audits).filter((auditKey) => !audits[auditKey].manual); 134 | const testCaseTime = (suiteDuration / auditKeys.length); 135 | 136 | auditKeys.forEach((auditKey) => { 137 | const { score, manual, description, helpText, details = {}, extendedInfo = {} } = audits[auditKey]; 138 | const testCase = suite.testCase(); 139 | 140 | testCase.className(suiteName); 141 | testCase.name(description); 142 | if (!isPretty) { 143 | testCase.time(testCaseTime); 144 | } 145 | 146 | if (score !== true && !manual) { 147 | const errorMessage = `Error on ${url}\n${helpText}\n\n`; 148 | const erroredElements = getErroredElements({ details, extendedInfo, hide }); 149 | 150 | if (erroredElements.length > 0) { 151 | const errorDetail = getErrorDetail(erroredElements); 152 | testCase.failure(errorMessage + errorDetail); 153 | } 154 | } 155 | }); 156 | } catch (err) { 157 | logger.error(`An error occurred while launching Chrome and running Lighthouse.\nError: ${err}`); 158 | process.exit(1); 159 | } 160 | } 161 | 162 | async function launchChromeAndRunLighthouse(url, shouldSignIn) { 163 | const { flags: lighthouseFlags, config: lightHouseConfig } = LIGHTHOUSE_OPTS; 164 | const chromeOpts = getChromeOpts(); 165 | 166 | let chrome; 167 | 168 | try { 169 | chrome = await chromeLauncher.launch(chromeOpts); 170 | } catch (err) { 171 | return logger.error(`Failed to launch Chrome.\nError: ${err}`); 172 | } 173 | 174 | try { 175 | await completeSetupSteps(chrome, url, shouldSignIn); 176 | } catch (err) { 177 | return logger.error(`Failed to complete setup steps.\nError: ${err}\nDebug: ${{ chrome, url, shouldSignIn }}`); 178 | } 179 | 180 | try { 181 | const flags = { 182 | ...lighthouseFlags, 183 | port: chrome.port 184 | }; 185 | const results = await external.lighthouse(url, flags, lightHouseConfig); 186 | return results; 187 | } catch (err) { 188 | logger.error(`Failed to launch Lighthouse.\nError: ${err}\nDebug: ${{ url, lighthouseFlags, lightHouseConfig }}`); 189 | } finally { 190 | await chrome.kill(); 191 | } 192 | } 193 | 194 | function getChromeOpts() { 195 | const defaultOpts = { 196 | chromeFlags: ['--disable-gpu', '--no-sandbox'], 197 | connectionPollInterval: 500, 198 | maxConnectionRetries: 100, 199 | logLevel: 'error' 200 | }; 201 | 202 | if (process.env.A11Y_HEADLESS) { 203 | return { 204 | ...defaultOpts, 205 | chromeFlags: ['--headless', ...defaultOpts.chromeFlags] 206 | }; 207 | } 208 | 209 | return defaultOpts; 210 | } 211 | 212 | function completeSetupSteps(chrome, url, shouldSignIn) { 213 | if (shouldSignIn) { 214 | return signInToBBCID(chrome, url); 215 | } 216 | 217 | return Promise.resolve(); 218 | } 219 | 220 | async function signInToBBCID(chrome, url) { 221 | let chromeProtocols; 222 | 223 | try { 224 | await waitForInspectableTarget(chrome); 225 | } catch (err) { 226 | return logger.error(`Failed to get inspectable target.\nError: ${err}`); 227 | } 228 | 229 | try { 230 | chromeProtocols = await external.CDP({ port: chrome.port }); 231 | } catch (err) { 232 | return logger.error(`Could not obtain chrome protocols.\nError: ${err}`); 233 | } 234 | 235 | try { 236 | await loadSignInPage(url, chromeProtocols); 237 | } catch (err) { 238 | return logger.error(`Could not load sign in page.\nError: ${err}`); 239 | } 240 | 241 | try { 242 | await completeSignInProcess(chromeProtocols); 243 | } catch (err) { 244 | return logger.error(`Could not complete sign in process.\nError: ${err}`); 245 | } 246 | } 247 | 248 | async function loadSignInPage(url, { Page }) { 249 | const encodedUrl = encodeURIComponent(url); 250 | const signInUrl = `https://account.bbc.com/signin?ptrt=${encodedUrl}`; 251 | 252 | return new Promise(async (resolve) => { 253 | await Page.enable(); 254 | await Page.navigate({ url: signInUrl }); 255 | await Page.loadEventFired(resolve); 256 | }); 257 | } 258 | 259 | function completeSignInProcess({ Page, Runtime }) { 260 | return new Promise(async (resolve) => { 261 | const { username, password } = getSignInCredentials(); 262 | const loginScript = ` 263 | document.getElementById('user-identifier-input').value = '${username}'; 264 | document.getElementById('password-input').value = '${password}'; 265 | document.getElementById('submit-button').click(); 266 | `; 267 | Runtime.evaluate({ expression: loginScript }); 268 | await Page.loadEventFired(resolve); 269 | }); 270 | } 271 | 272 | function executeSequentially(tasks) { 273 | if (tasks && tasks.length > 0) { 274 | const task = tasks.shift(); 275 | return task().then(() => executeSequentially(tasks)); 276 | } 277 | 278 | return Promise.resolve(); 279 | } 280 | 281 | function getErroredElements({ details, extendedInfo, hide }) { 282 | if (details.items) { 283 | return details.items.filter(({ selector }) => !selectorIsHidden(selector, hide)); 284 | } 285 | if (extendedInfo.value && extendedInfo.value.nodes) { 286 | return extendedInfo.value.nodes.filter(({ target }) => !selectorIsHidden(target[0], hide)); 287 | } 288 | return []; 289 | } 290 | 291 | function getErrorDetail(elements) { 292 | return 'Failing elements:\n' + elements.map(({ selector, target, snippet }) => (selector || target[0]) + ' - ' + snippet).join('\n'); 293 | } 294 | 295 | function selectorIsHidden(selector, hide) { 296 | return !!hide.find((hideOption) => selector.includes(hideOption)); 297 | } 298 | 299 | module.exports = { 300 | run 301 | }; 302 | -------------------------------------------------------------------------------- /lib/request.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const http = require('http'); 4 | 5 | function get(url = '') { 6 | return new Promise((resolve, reject) => { 7 | http.get(url, (response) => { 8 | let data = ''; 9 | 10 | response.on('data', (chunk) => { 11 | data += chunk; 12 | }); 13 | 14 | response.on('end', () => { 15 | resolve(data); 16 | }); 17 | 18 | }).on('error', (err) => { 19 | reject(err); 20 | }); 21 | }); 22 | } 23 | 24 | module.exports = { get }; 25 | -------------------------------------------------------------------------------- /lib/xunitViewer.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const xunitViewer = require('xunit-viewer'); 4 | 5 | function toConsole() { 6 | xunitViewer({ 7 | server: false, 8 | results: 'lighthouse-report.xml', 9 | ignore: [], 10 | title: 'Lighthouse a11y audit complete.', 11 | console: true, 12 | output: false 13 | }); 14 | } 15 | 16 | module.exports = { toConsole }; 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@bbc/a11y-tests-web", 3 | "version": "6.6.1", 4 | "description": "Runs automated accessibility tests on a set of pages", 5 | "main": "index.js", 6 | "bin": { 7 | "a11y-tests-web": "./cli.js" 8 | }, 9 | "engines": { 10 | "node": ">=10.0.0" 11 | }, 12 | "scripts": { 13 | "build:bbc-a11y": "node buildBbcA11y.js", 14 | "clean": "node cleanBbcA11y.js", 15 | "start:bbc-a11y": "npm run build:bbc-a11y && bbc-a11y --interactive && npm run clean", 16 | "start:bbc-a11y:headless": "npm run build:bbc-a11y && bbc-a11y && npm run clean", 17 | "start:bbc-a11y:junit": "npm run build:bbc-a11y && bbc-a11y --interactive --reporter ./lib/bbcA11YJUnitReporter.js && npm run clean", 18 | "start:bbc-a11y:junit-headless": "npm run build:bbc-a11y && bbc-a11y --reporter ./lib/bbcA11YJUnitReporter.js && npm run clean", 19 | "start:bbc-a11y:ci": "npm run build:bbc-a11y && bash -c \"docker run --rm --tty --volume $PWD:/ws bbca11y/bbc-a11y-docker --config /ws/a11y.js --reporter /ws/lib/bbcA11YJUnitReporter.js; exit 0;\" && npm run clean", 20 | "start:lighthouse:junit": "node runLighthouse.js", 21 | "start:lighthouse:junit-headless": "A11Y_HEADLESS=true node runLighthouse.js", 22 | "start:lighthouse:headless": "A11Y_HEADLESS=true A11Y_PRETTY=true node runLighthouse.js", 23 | "lint": "npm run clean && eslint .", 24 | "test": "mocha 'test/**/*.js'", 25 | "posttest": "npm run lint", 26 | "coverage": "nyc --reporter=lcov npm test && nyc report" 27 | }, 28 | "author": "Andy Smith", 29 | "license": "Apache-2.0", 30 | "dependencies": { 31 | "bbc-a11y": "^2.4.2", 32 | "chrome-launcher": "^0.13.2", 33 | "chrome-remote-interface": "^0.25.7", 34 | "junit-report-builder": "^1.3.3", 35 | "lighthouse": "^2.9.4", 36 | "xunit-viewer": "^6.3.12", 37 | "yargs": "^14.2.3" 38 | }, 39 | "devDependencies": { 40 | "eslint": "^7.13.0", 41 | "eslint-config-iplayer": "^7.0.0", 42 | "eslint-plugin-mocha": "^4.12.1", 43 | "harp-minify": "^0.4.0", 44 | "mocha": "^7.2.0", 45 | "nock": "^13.1.4", 46 | "nyc": "^15.1.0", 47 | "sinon": "^2.4.0" 48 | }, 49 | "eslintConfig": { 50 | "extends": "iplayer" 51 | }, 52 | "nyc": { 53 | "include": [ 54 | "lib/*" 55 | ] 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /runLighthouse.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { run } = require('./lib/lighthouse'); 4 | run(); 5 | -------------------------------------------------------------------------------- /test/fixtures/inspectableTargets.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "", 4 | "devtoolsFrontendUrl": "/devtools/inspector.html?ws=127.0.0.1:1234/devtools/page/7B5B9E9C373EEDD92AD48AFD3BC85BFF", 5 | "id": "7B5B9E9C373EEDD92AD48AFD3BC85BFF", 6 | "title": "", 7 | "type": "page", 8 | "url": "about:blank", 9 | "webSocketDebuggerUrl": "ws://127.0.0.1:1234/devtools/page/7B5B9E9C373EEDD92AD48AFD3BC85BFF" 10 | } 11 | ] 12 | -------------------------------------------------------------------------------- /test/fixtures/lighthouseReport.json: -------------------------------------------------------------------------------- 1 | { 2 | "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/62.0.3202.94 Safari/537.36", 3 | "lighthouseVersion": "2.6.0", 4 | "generatedTime": "2017-12-06T13:39:59.291Z", 5 | "initialUrl": "https://www.bbc.co.uk/iplayer/episode/p04qh1gk/face-to-face-dame-edith-sitwell", 6 | "url": "https://www.bbc.co.uk/iplayer/episode/p04qh1gk/face-to-face-dame-edith-sitwell", 7 | "audits": { 8 | "aria-roles": { 9 | "score": true, 10 | "displayValue": "", 11 | "rawValue": true, 12 | "scoringMode": "binary", 13 | "name": "aria-roles", 14 | "description": "`[role]` values are valid.", 15 | "helpText": "ARIA roles must have valid values in order to perform their intended accessibility functions. [Learn more](https://dequeuniversity.com/rules/axe/2.2/aria-roles?application=lighthouse).", 16 | "details": { 17 | "type": "list", 18 | "header": { 19 | "type": "text", 20 | "text": "View failing elements" 21 | }, 22 | "items": [] 23 | } 24 | }, 25 | "color-contrast": { 26 | "score": false, 27 | "displayValue": "", 28 | "rawValue": false, 29 | "extendedInfo": { 30 | "value": { 31 | "id": "color-contrast", 32 | "impact": "serious", 33 | "tags": [ 34 | "cat.color", 35 | "wcag2aa", 36 | "wcag143" 37 | ], 38 | "description": "Ensures the contrast between foreground and background colors meets WCAG 2 AA contrast ratio thresholds", 39 | "help": "Elements must have sufficient color contrast", 40 | "helpUrl": "https://dequeuniversity.com/rules/axe/2.4/color-contrast?application=axeAPI", 41 | "nodes": [ 42 | { 43 | "any": null, 44 | "all": null, 45 | "none": null, 46 | "impact": "serious", 47 | "html": "Archive", 48 | "element": null, 49 | "target": [ 50 | "span > strong" 51 | ], 52 | "failureSummary": "Fix any of the following:\n Element has insufficient color contrast of 3.34 (foreground color: #ffffff, background color: #f54997, font size: 8.3pt, font weight: bold)", 53 | "path": "1,HTML,1,BODY,14,DIV,1,DIV,0,DIV,2,DIV,3,DIV,0,DIV,3,DIV,0,DIV,0,DIV,1,DIV,0,DIV,0,DIV,0,DIV,0,SPAN,0,STRONG", 54 | "snippet": "" 55 | } 56 | ] 57 | } 58 | }, 59 | "scoringMode": "binary", 60 | "name": "color-contrast", 61 | "description": "Background and foreground colors do not have a sufficient contrast ratio.", 62 | "helpText": "Low-contrast text is difficult or impossible for many users to read. [Learn more](https://dequeuniversity.com/rules/axe/2.2/color-contrast?application=lighthouse)." 63 | }, 64 | "label": { 65 | "score": false, 66 | "displayValue": "", 67 | "rawValue": false, 68 | "extendedInfo": { 69 | "value": { 70 | "id": "label", 71 | "impact": "critical", 72 | "tags": [ 73 | "cat.forms", 74 | "wcag2a", 75 | "wcag332", 76 | "wcag131", 77 | "section508", 78 | "section508.22.n" 79 | ], 80 | "description": "Ensures every form element has a label", 81 | "help": "Form elements must have labels", 82 | "helpUrl": "https://dequeuniversity.com/rules/axe/2.4/label?application=axeAPI", 83 | "nodes": [ 84 | { 85 | "any": null, 86 | "all": null, 87 | "none": null, 88 | "impact": "critical", 89 | "html": "", 90 | "element": null, 91 | "target": [ 92 | "#orb-modules form > input[type=\"text\"][name=\"q\"]" 93 | ], 94 | "failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty or not visible\n Form element does not have an implicit (wrapped)