├── test ├── unit │ ├── test.js │ ├── util │ │ └── get-score.test.js │ └── express │ │ └── api │ │ ├── post-website.test.js │ │ └── get-website.test.js ├── mocha.opts ├── mocks │ ├── request.js │ ├── queue.js │ ├── app.js │ ├── response.js │ ├── index.js │ ├── config.js │ └── store.js ├── local │ └── before-start.js ├── config │ ├── COPY.json5 │ └── defaults.json5 └── fs-store │ ├── raw1.godaddysites.com │ ├── 2019-01-15T23:41:53.383Z.json │ ├── 2019-01-15T23:42:47.746Z.json │ ├── 2019-01-15T23:52:12.583Z.json │ ├── 2019-01-15T23:52:56.542Z.json │ ├── 2019-01-16T00:02:34.079Z.json │ ├── 2019-01-16T00:03:52.902Z.json │ ├── 2019-01-16T00:06:59.013Z.json │ ├── 2019-01-16T00:08:46.275Z.json │ ├── 2019-01-16T00:09:35.777Z.json │ ├── 2019-01-16T00:11:01.822Z.json │ ├── 2019-01-16T00:15:29.792Z.json │ ├── 2019-01-16T16:11:19.931Z.json │ ├── 2019-01-16T16:13:14.046Z.json │ ├── 2019-01-16T17:19:17.529Z.json │ ├── 2019-01-16T17:21:52.890Z.json │ ├── 2019-01-16T17:22:02.458Z.json │ ├── 2019-01-16T17:22:10.218Z.json │ ├── 2019-01-16T17:22:42.913Z.json │ ├── 2019-01-16T17:37:11.924Z.json │ ├── 2019-01-16T17:58:45.547Z.json │ └── 2019-01-16T21:11:04.685Z.json │ └── www.google.com │ └── 2019-01-16T18:30:37.721Z.json ├── .npmrc ├── src ├── express │ ├── views │ │ ├── partials │ │ │ ├── header.ejs │ │ │ ├── footer.ejs │ │ │ └── head.ejs │ │ └── pages │ │ │ └── index.ejs │ ├── util │ │ ├── get-website.js │ │ └── get-query.js │ ├── auth │ │ ├── basic.js │ │ └── index.js │ ├── api │ │ ├── get-website.js │ │ ├── website │ │ │ └── get-compare.js │ │ └── post-website.js │ ├── index.js │ └── static │ │ └── app.js ├── config │ ├── index.js │ ├── get-configs.js │ ├── default-config.js │ └── get-config-by-id.js ├── util │ ├── get-safe-document.js │ ├── get-score.js │ ├── encrypt.js │ ├── decrypt.js │ ├── get-client.js │ └── convert-doc-to-svg.js ├── lighthouse │ ├── throttling.js │ ├── defaults.js │ └── submit.js ├── http │ └── start.js ├── commands │ ├── init.js │ └── server.js ├── queue │ ├── index.js │ └── process.js ├── cli.js └── store │ └── index.js ├── bin └── lh4u ├── packages ├── lighthouse4u-amqp │ ├── .npmrc │ ├── .gitignore │ ├── .npmignore │ ├── SECURITY.md │ ├── package.json │ ├── LICENSE │ ├── README.md │ └── index.js ├── lighthouse4u-es │ ├── .npmrc │ ├── .gitignore │ ├── .npmignore │ ├── SECURITY.md │ ├── package.json │ ├── README.md │ ├── LICENSE │ ├── default-options.json │ └── index.js ├── lighthouse4u-fs │ ├── .npmrc │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── SECURITY.md │ ├── package.json │ ├── LICENSE │ └── index.js ├── lighthouse4u-s3 │ ├── .npmrc │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── package.json │ ├── SECURITY.md │ ├── LICENSE │ └── index.js └── lighthouse4u-sqs │ ├── .npmrc │ ├── .gitignore │ ├── .npmignore │ ├── README.md │ ├── package.json │ ├── SECURITY.md │ ├── LICENSE │ └── index.js ├── docs └── example.gif ├── .travis.yml ├── renovate.json ├── .eslintrc ├── SECURITY.md ├── .gitignore ├── LICENSE ├── package.json ├── .github └── workflows │ └── codeql.yml ├── README.md └── lh.json /test/unit/test.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry=https://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /src/express/views/partials/header.ejs: -------------------------------------------------------------------------------- 1 |

Lighthouse4u

2 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --recursive 2 | -t 30000 3 | --include unit 4 | -------------------------------------------------------------------------------- /bin/lh4u: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require('../src/cli'); 4 | -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/.npmrc: -------------------------------------------------------------------------------- 1 | registry=http://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /packages/lighthouse4u-es/.npmrc: -------------------------------------------------------------------------------- 1 | registry=http://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/.npmrc: -------------------------------------------------------------------------------- 1 | registry=http://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/.npmrc: -------------------------------------------------------------------------------- 1 | registry=http://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/.npmrc: -------------------------------------------------------------------------------- 1 | registry=http://registry.npmjs.org/ 2 | -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-es/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-es/.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /test/mocks/request.js: -------------------------------------------------------------------------------- 1 | module.exports = () => ({ 2 | query: {} 3 | }); 4 | -------------------------------------------------------------------------------- /docs/example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/godaddy/lighthouse4u/master/docs/example.gif -------------------------------------------------------------------------------- /src/express/views/partials/footer.ejs: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | addons: 3 | chrome: stable 4 | language: node_js 5 | node_js: 6 | - '8' 7 | - '10' 8 | -------------------------------------------------------------------------------- /test/mocks/queue.js: -------------------------------------------------------------------------------- 1 | const { stub } = require('sinon'); 2 | 3 | module.exports = () => ({ 4 | enqueue: stub().resolves() 5 | }); 6 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /test/local/before-start.js: -------------------------------------------------------------------------------- 1 | console.log('My custom handler prior to HTTP Server Start'); 2 | 3 | // optional 4 | // module.exports = argv => console.log('argv:', argv); 5 | -------------------------------------------------------------------------------- /src/config/index.js: -------------------------------------------------------------------------------- 1 | const getConfigById = require('./get-config-by-id'); 2 | const getConfigs = require('./get-configs'); 3 | 4 | module.exports = { getConfigById, getConfigs }; 5 | -------------------------------------------------------------------------------- /test/config/COPY.json5: -------------------------------------------------------------------------------- 1 | { 2 | // COPY THIS FILE to local.json5 and apply your local-only changes 3 | queue: { options: { connect: { options: { url: 'amqp://user:password@host:5672/lh4u' } } } } 4 | } 5 | -------------------------------------------------------------------------------- /src/util/get-safe-document.js: -------------------------------------------------------------------------------- 1 | module.exports = function getSafeDocument(document) { 2 | const { secureHeaders, cipherVector, commands, cookies, ...safeDocument } = document; 3 | 4 | return safeDocument; 5 | }; 6 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "godaddy", 3 | "rules": { 4 | "strict": 0, 5 | "no-sync": 0, 6 | "no-unused-vars": 0, 7 | "id-length": 0, 8 | "no-alert": 0, 9 | "no-shadow": 0, 10 | "no-async-promise-executor": 0 11 | } 12 | } -------------------------------------------------------------------------------- /src/util/get-score.js: -------------------------------------------------------------------------------- 1 | module.exports = function getScore(doc, cat) { 2 | switch (doc.state) { 3 | case 'processed': 4 | return (cat in doc.categories) ? `${(doc.categories[cat].score * 100).toFixed(0)}%` : '?'; 5 | case 'error': 6 | return 'error'; 7 | default: 8 | return 'pending'; 9 | } 10 | }; 11 | -------------------------------------------------------------------------------- /test/mocks/app.js: -------------------------------------------------------------------------------- 1 | const { stub } = require('sinon'); 2 | 3 | module.exports = mocks => { 4 | 5 | const appGet = stub(); 6 | appGet.withArgs('config').returns(mocks.config); 7 | appGet.withArgs('store').returns(mocks.store); 8 | appGet.withArgs('queue').returns(mocks.queue); 9 | return { 10 | get: appGet 11 | }; 12 | 13 | }; 14 | -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-15T23:41:53.383Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547595713382 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-15T23:42:47.746Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547595767745 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-15T23:52:12.583Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547596332581 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-15T23:52:56.542Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547596376542 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T00:02:34.079Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547596954079 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T00:03:52.902Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547597032902 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T00:06:59.013Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547597219011 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T00:08:46.275Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547597326274 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T00:09:35.777Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547597375775 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T00:11:01.822Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547597461819 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T00:15:29.792Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547597729789 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T16:11:19.931Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547655079930 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T16:13:14.046Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547655194045 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T17:19:17.529Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547659157527 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T17:21:52.890Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547659312890 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T17:22:02.458Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547659322457 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T17:22:10.218Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547659330218 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T17:22:42.913Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547659362912 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T17:37:11.924Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547660231924 11 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T17:58:45.547Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "headers": {}, 6 | "group": "web", 7 | "delay": 0, 8 | "delayTime": 0, 9 | "state": "requested", 10 | "createDate": 1547661525547 11 | } -------------------------------------------------------------------------------- /test/mocks/response.js: -------------------------------------------------------------------------------- 1 | const { stub } = require('sinon'); 2 | 3 | module.exports = () => { 4 | const ret = { 5 | render: stub(), 6 | send: stub(), 7 | sendStatus: stub(), 8 | status: stub(), 9 | writeHead: stub(), 10 | setHeader: stub(), 11 | end: stub() 12 | }; 13 | 14 | ret.status.returns(ret); 15 | 16 | return ret; 17 | } 18 | 19 | -------------------------------------------------------------------------------- /src/express/util/get-website.js: -------------------------------------------------------------------------------- 1 | const getQuery = require('./get-query'); 2 | 3 | module.exports = (app, query, { clean = false } = {}) => { 4 | 5 | const store = app.get('store'); 6 | 7 | const { top = 1 } = query; 8 | 9 | const q = getQuery(query); 10 | 11 | if (!q) throw new Error('query param `q` is required'); 12 | 13 | return store.query(q, { top }); 14 | }; 15 | -------------------------------------------------------------------------------- /test/mocks/index.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | const mocks = { 3 | config: require('./config')(), 4 | store: require('./store')(), 5 | queue: require('./queue')(), 6 | request: require('./request')(), 7 | response: require('./response')() 8 | }; 9 | 10 | mocks.app = require('./app')(mocks); 11 | mocks.request.app = mocks.app; 12 | 13 | return mocks; 14 | }; -------------------------------------------------------------------------------- /src/lighthouse/throttling.js: -------------------------------------------------------------------------------- 1 | const { throttling } = require('lighthouse/lighthouse-core/config/constants'); 2 | 3 | // extend LH's throttling with our own 4 | module.exports = Object.assign({ 5 | 6 | }, throttling); 7 | 8 | if (!module.exports.mobile3G && module.exports.mobileSlow4G) { 9 | // backward compatibility since LH renamed 3G 10 | module.exports.mobile3G = module.exports.mobileSlow4G; 11 | } 12 | -------------------------------------------------------------------------------- /src/express/auth/basic.js: -------------------------------------------------------------------------------- 1 | const basicAuth = require('basic-auth'); 2 | 3 | module.exports = (/* app*/) => { 4 | return ({ req/* , res*/ }, options) => { 5 | const creds = basicAuth(req); 6 | if (!creds) return false; 7 | const { name, pass } = creds; 8 | 9 | if (name !== options.user || pass !== options.pass) return false; 10 | 11 | // if we get this far, request is permitted 12 | return true; 13 | }; 14 | }; 15 | -------------------------------------------------------------------------------- /src/util/encrypt.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | 3 | module.exports = function encrypt(input, secretKey, vector) { 4 | const hash = crypto.createHash('sha1'); 5 | hash.update(secretKey); 6 | const key = hash.digest().slice(0, 16).toString('hex'); 7 | const cipher = crypto.createCipheriv('aes-256-cbc', key, vector); 8 | let result = cipher.update(input, 'utf8', 'base64'); 9 | result += cipher.final('base64'); 10 | return result; 11 | }; 12 | -------------------------------------------------------------------------------- /test/mocks/config.js: -------------------------------------------------------------------------------- 1 | const { merge } = require('lodash'); 2 | const { readFileSync } = require('fs'); 3 | const { parse } = require('json5'); 4 | 5 | const defaultConfig = require('../../src/config/default-config'); 6 | const testDefaults = parse(readFileSync('./test/config/defaults.json5', 'utf8')); 7 | const localConfig = parse(readFileSync('./test/config/COPY.json5', 'utf8')); 8 | 9 | module.exports = () => merge({}, defaultConfig, testDefaults, localConfig); 10 | -------------------------------------------------------------------------------- /src/util/decrypt.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | 3 | module.exports = function decrypt(encryptedInput, secretKey, vector) { 4 | const hash = crypto.createHash('sha1'); 5 | hash.update(secretKey); 6 | const key = hash.digest().slice(0, 16).toString('hex'); 7 | const decipher = crypto.createDecipheriv('aes-256-cbc', key, vector); 8 | let result = decipher.update(encryptedInput, 'base64', 'utf8'); 9 | result += decipher.final('utf8'); 10 | return result; 11 | }; 12 | -------------------------------------------------------------------------------- /src/config/get-configs.js: -------------------------------------------------------------------------------- 1 | const getConfig = require('./get-config-by-id'); 2 | 3 | module.exports = argv => { 4 | const configs = typeof argv.config === 'string' ? [argv.config] : argv.config; 5 | 6 | if (configs.length === 0) { 7 | // if no config specified, use environment, or fallback to default 8 | configs.push(argv.configEnv in process.env ? process.env[argv.configEnv] : argv.configDefault); 9 | } 10 | 11 | return Promise.all(configs.map(configName => getConfig(configName, argv))); 12 | }; 13 | -------------------------------------------------------------------------------- /src/express/views/partials/head.ejs: -------------------------------------------------------------------------------- 1 | Lighthouse4u 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /src/http/start.js: -------------------------------------------------------------------------------- 1 | const http = require('http'); 2 | const http2 = require('http2'); 3 | 4 | module.exports = app => { 5 | const { http: { bindings } } = app.get('config'); 6 | 7 | const bindingKeys = Object.keys(bindings); 8 | if (!bindingKeys.length) throw new Error('No http.bindings detected!'); 9 | bindingKeys.forEach(key => { 10 | const { port, ssl } = bindings[key]; 11 | 12 | const server = ssl 13 | ? http2.createServer(ssl, app) 14 | : http.createServer(app) 15 | ; 16 | 17 | server.listen(port); 18 | console.log(`Listening on ${port}...`); 19 | }); 20 | }; 21 | -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse4u-fs 2 | 3 | A File System storage client for [Lighthouse4u](https://github.com/godaddy/lighthouse4u). Recommended for local 4 | testing only. 5 | 6 | ``` 7 | { 8 | store: { 9 | module: 'lighthouse4u-fs', options: { 10 | dir: './my/storage' 11 | } 12 | } 13 | } 14 | ``` 15 | 16 | 17 | ## Configuration Options 18 | 19 | | Option | Type | Default | Desc | 20 | | --- | --- | --- | --- | 21 | | module | `string` | **required** | Set this to `lighthouse4u-fs` | 22 | | options | `object` | **required** | File storage options | 23 | | ->.dir | `string` | **required** | Relative or absolute file path of storage | 24 | -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse4u-s3 2 | 3 | An AWS S3 storage client for [Lighthouse4u](https://github.com/godaddy/lighthouse4u). 4 | 5 | ``` 6 | { 7 | store: { 8 | module: 'lighthouse4u-s3', options: { 9 | region: 'us-east-1', 10 | bucket: 'my-lh4u-bucket' 11 | } 12 | } 13 | } 14 | ``` 15 | 16 | 17 | ## Configuration Options 18 | 19 | | Option | Type | Default | Desc | 20 | | --- | --- | --- | --- | 21 | | module | `string` | **required** | Set this to `lighthouse4u-s3` | 22 | | options | `object` | **required** | S3 storage options | 23 | | ->.region | `string` | **required** | AWS Region | 24 | | ->.bucket | `string` | **required** | AWS S3 Bucket | 25 | -------------------------------------------------------------------------------- /test/config/defaults.json5: -------------------------------------------------------------------------------- 1 | { 2 | http: { 3 | bindings: { 4 | http: { 5 | port: 8080 6 | } 7 | } 8 | }, 9 | store: { 10 | module: './packages/lighthouse4u-fs', 11 | options: { 12 | dir: './test/fs-store' 13 | } 14 | }, 15 | queue: { 16 | module: './packages/lighthouse4u-amqp', 17 | options: { 18 | connect: { options: { url: 'amqp://user:password@host:5672/lh4u' } }, // see COPY.json5 19 | queue: { name: 'lh4u' } 20 | }, 21 | secretKey: 'superSecret!' // for testing only -- this belongs in secure 22 | }, 23 | lighthouse: { 24 | attempts: { 25 | default: 1 26 | }, 27 | samples: { 28 | default: 1 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse4u-sqs 2 | 3 | An AWS Simple Queue Service queue client for [Lighthouse4u](https://github.com/godaddy/lighthouse4u). 4 | 5 | ``` 6 | { 7 | queue: { 8 | module: 'lighthouse4u-sqs', options: { 9 | region: 'us-east-1', 10 | queueUrl: 'https://sqs.us-east-1.amazonaws.com/897234987/lh4u' 11 | } 12 | } 13 | } 14 | ``` 15 | 16 | 17 | ## Configuration Options 18 | 19 | | Option | Type | Default | Desc | 20 | | --- | --- | --- | --- | 21 | | module | `string` | **required** | Set this to `lighthouse4u-sqs` | 22 | | options | `object` | **required** | SQS queue options | 23 | | ->.region | `string` | **required** | AWS Region | 24 | | ->.queueUrl | `string` | **required** | AWS SQS URL | 25 | -------------------------------------------------------------------------------- /test/mocks/store.js: -------------------------------------------------------------------------------- 1 | const { stub } = require('sinon'); 2 | 3 | module.exports = () => { 4 | const doc = { 5 | requestedUrl: 'requestedUrl', 6 | finalUrl: 'finalUrl', 7 | rootDomain: 'rootDomain', 8 | group: 'group', 9 | samples: 1, 10 | attempt: 1, 11 | attempts: 1, 12 | state: 'processed', 13 | createDate: 1234567890, 14 | categories: { 15 | performance: { score: 0.5 }, 16 | pwa: { score: 0.6 }, 17 | accessibility: { score: 0.7 }, 18 | 'best-practices': { score: 0.8 }, 19 | seo: { score: 0.9 }, 20 | } 21 | }; 22 | return { 23 | '$document': doc, 24 | query: stub().resolves(doc), 25 | write: stub().resolves({ 26 | id: 'documentId' 27 | }) 28 | }; 29 | }; 30 | -------------------------------------------------------------------------------- /src/util/get-client.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const fs = require('fs'); 3 | 4 | module.exports = function getClient(config) { 5 | let absPath = path.isAbsolute(config.module) ? config.module // already absolute path 6 | : path.resolve(process.cwd(), config.module) // try explicit first 7 | ; 8 | if (!fs.existsSync(absPath)) { // if not found, attempt to use node_modules 9 | absPath = path.resolve(process.cwd(), 'node_modules/' + config.module); 10 | } 11 | // no need to cache the `Client` module since `require` does this for us 12 | const mod = require(absPath); 13 | const Client = mod.default || mod; // support ES Modules & CommonJS 14 | // always create a unique instance of Client 15 | const instance = new Client(config.options || {}); 16 | 17 | return instance; 18 | }; 19 | -------------------------------------------------------------------------------- /src/util/convert-doc-to-svg.js: -------------------------------------------------------------------------------- 1 | const getScore = require('./get-score'); 2 | 3 | module.exports = (doc, { svgWidth, svgHeight, scale }) => { 4 | return { 5 | state: doc && doc.state, 6 | group: doc && doc.group, 7 | requestedUrl: doc ? doc.requestedUrl : '(NOT FOUND)', 8 | createDate: doc && new Date(doc.createDate).toLocaleString('en-US', { weekday: 'long', year: 'numeric', month: 'long', day: 'numeric' }), 9 | performance: doc && getScore(doc, 'performance'), 10 | accessibility: doc && getScore(doc, 'accessibility'), 11 | bestPractices: doc && getScore(doc, 'best-practices'), 12 | seo: doc && getScore(doc, 'seo'), 13 | pwa: doc && getScore(doc, 'pwa'), 14 | svgWidth: Math.round(svgWidth * parseFloat(scale)), 15 | svgHeight: Math.round(svgHeight * parseFloat(scale)) 16 | }; 17 | }; 18 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | We take security very seriously at GoDaddy. We appreciate your efforts to 4 | responsibly disclose your findings, and will make every effort to acknowledge 5 | your contributions. 6 | 7 | ## Where should I report security issues? 8 | 9 | In order to give the community time to respond and upgrade, we strongly urge you 10 | report all security issues privately. 11 | 12 | To report a security issue in one of our Open Source projects email us directly 13 | at **oss@godaddy.com** and include the word "SECURITY" in the subject line. 14 | 15 | This mail is delivered to our Open Source Security team. 16 | 17 | After the initial reply to your report, the team will keep you informed of the 18 | progress being made towards a fix and announcement, and may ask for additional 19 | information or guidance. -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lighthouse4u-s3", 3 | "version": "1.0.0", 4 | "description": "AWS S3 Storage client for Lighthouse4u", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/godaddy/lighthouse4u.git" 12 | }, 13 | "files": [ 14 | "index.js", 15 | "LICENSE", 16 | "README.md", 17 | "SECURITY.md", 18 | "package.json" 19 | ], 20 | "author": "GoDaddy.com Operating Company LLC", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/godaddy/lighthouse4u/issues" 24 | }, 25 | "homepage": "https://github.com/godaddy/lighthouse4u#readme", 26 | "dependencies": { 27 | "aws-sdk": "^2.384.0" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lighthouse4u-sqs", 3 | "version": "1.0.0", 4 | "description": "AWS SQS Queue client for Lighthouse4u", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/godaddy/lighthouse4u.git" 12 | }, 13 | "files": [ 14 | "index.js", 15 | "LICENSE", 16 | "README.md", 17 | "SECURITY.md", 18 | "package.json" 19 | ], 20 | "author": "GoDaddy.com Operating Company LLC", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/godaddy/lighthouse4u/issues" 24 | }, 25 | "homepage": "https://github.com/godaddy/lighthouse4u#readme", 26 | "dependencies": { 27 | "aws-sdk": "^2.384.0" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | 39 | .DS_Store 40 | .idea/* 41 | lib/* 42 | 43 | tmp/* 44 | 45 | test/config/local.json5 46 | -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | We take security very seriously at GoDaddy. We appreciate your efforts to 4 | responsibly disclose your findings, and will make every effort to acknowledge 5 | your contributions. 6 | 7 | ## Where should I report security issues? 8 | 9 | In order to give the community time to respond and upgrade, we strongly urge you 10 | report all security issues privately. 11 | 12 | To report a security issue in one of our Open Source projects email us directly 13 | at **oss@godaddy.com** and include the word "SECURITY" in the subject line. 14 | 15 | This mail is delivered to our Open Source Security team. 16 | 17 | After the initial reply to your report, the team will keep you informed of the 18 | progress being made towards a fix and announcement, and may ask for additional 19 | information or guidance. -------------------------------------------------------------------------------- /packages/lighthouse4u-es/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | We take security very seriously at GoDaddy. We appreciate your efforts to 4 | responsibly disclose your findings, and will make every effort to acknowledge 5 | your contributions. 6 | 7 | ## Where should I report security issues? 8 | 9 | In order to give the community time to respond and upgrade, we strongly urge you 10 | report all security issues privately. 11 | 12 | To report a security issue in one of our Open Source projects email us directly 13 | at **oss@godaddy.com** and include the word "SECURITY" in the subject line. 14 | 15 | This mail is delivered to our Open Source Security team. 16 | 17 | After the initial reply to your report, the team will keep you informed of the 18 | progress being made towards a fix and announcement, and may ask for additional 19 | information or guidance. -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | We take security very seriously at GoDaddy. We appreciate your efforts to 4 | responsibly disclose your findings, and will make every effort to acknowledge 5 | your contributions. 6 | 7 | ## Where should I report security issues? 8 | 9 | In order to give the community time to respond and upgrade, we strongly urge you 10 | report all security issues privately. 11 | 12 | To report a security issue in one of our Open Source projects email us directly 13 | at **oss@godaddy.com** and include the word "SECURITY" in the subject line. 14 | 15 | This mail is delivered to our Open Source Security team. 16 | 17 | After the initial reply to your report, the team will keep you informed of the 18 | progress being made towards a fix and announcement, and may ask for additional 19 | information or guidance. -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lighthouse4u-fs", 3 | "version": "1.0.0", 4 | "description": "AWS FS Storage client for Lighthouse4u", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/godaddy/lighthouse4u.git" 12 | }, 13 | "files": [ 14 | "index.js", 15 | "LICENSE", 16 | "README.md", 17 | "SECURITY.md", 18 | "package.json" 19 | ], 20 | "author": "GoDaddy.com Operating Company LLC", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/godaddy/lighthouse4u/issues" 24 | }, 25 | "homepage": "https://github.com/godaddy/lighthouse4u#readme", 26 | "dependencies": { 27 | "fs-extra": "^7.0.1" 28 | }, 29 | "devDependencies": {} 30 | } 31 | -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | We take security very seriously at GoDaddy. We appreciate your efforts to 4 | responsibly disclose your findings, and will make every effort to acknowledge 5 | your contributions. 6 | 7 | ## Where should I report security issues? 8 | 9 | In order to give the community time to respond and upgrade, we strongly urge you 10 | report all security issues privately. 11 | 12 | To report a security issue in one of our Open Source projects email us directly 13 | at **oss@godaddy.com** and include the word "SECURITY" in the subject line. 14 | 15 | This mail is delivered to our Open Source Security team. 16 | 17 | After the initial reply to your report, the team will keep you informed of the 18 | progress being made towards a fix and announcement, and may ask for additional 19 | information or guidance. -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | We take security very seriously at GoDaddy. We appreciate your efforts to 4 | responsibly disclose your findings, and will make every effort to acknowledge 5 | your contributions. 6 | 7 | ## Where should I report security issues? 8 | 9 | In order to give the community time to respond and upgrade, we strongly urge you 10 | report all security issues privately. 11 | 12 | To report a security issue in one of our Open Source projects email us directly 13 | at **oss@godaddy.com** and include the word "SECURITY" in the subject line. 14 | 15 | This mail is delivered to our Open Source Security team. 16 | 17 | After the initial reply to your report, the team will keep you informed of the 18 | progress being made towards a fix and announcement, and may ask for additional 19 | information or guidance. -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lighthouse4u-amqp", 3 | "version": "1.0.2", 4 | "description": "AWS AMQP Queue client for Lighthouse4u", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/godaddy/lighthouse4u.git" 12 | }, 13 | "files": [ 14 | "index.js", 15 | "LICENSE", 16 | "README.md", 17 | "SECURITY.md", 18 | "package.json" 19 | ], 20 | "author": "GoDaddy.com Operating Company LLC", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/godaddy/lighthouse4u/issues" 24 | }, 25 | "homepage": "https://github.com/godaddy/lighthouse4u#readme", 26 | "dependencies": { 27 | "amqplib": "^0.7.1" 28 | }, 29 | "devDependencies": {} 30 | } 31 | -------------------------------------------------------------------------------- /src/express/util/get-query.js: -------------------------------------------------------------------------------- 1 | const LEGACY_QUERIES = { 2 | documentId: 'documentId', 3 | rootDomain: 'rootDomain', 4 | domainName: 'domainName', 5 | requestedUrl: 'requestedUrl', 6 | group: 'group' 7 | }; 8 | 9 | module.exports = (query, defaultKey = 'q') => { 10 | if (!query || typeof query === 'string') { 11 | const legacySplit = query.split(':'); 12 | if (legacySplit && legacySplit.length >= 2 && legacySplit[0] in LEGACY_QUERIES) { 13 | // remove key 14 | return query.substr(legacySplit[0].length + 1); 15 | } 16 | return query; 17 | } 18 | 19 | let q = query[defaultKey]; 20 | 21 | if (!q) { // backward compatibility logic 22 | const { documentId, rootDomain, domainName, requestedUrl, group } = query; 23 | q = documentId ? `id:${documentId}` : (rootDomain || domainName || requestedUrl || group); 24 | } 25 | 26 | return q; 27 | }; 28 | -------------------------------------------------------------------------------- /packages/lighthouse4u-es/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lighthouse4u-es", 3 | "version": "1.2.3", 4 | "description": "Elasticsearch Storage client for Lighthouse4u", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/godaddy/lighthouse4u.git" 12 | }, 13 | "files": [ 14 | "index.js", 15 | "default-options.json", 16 | "LICENSE", 17 | "README.md", 18 | "SECURITY.md", 19 | "package.json" 20 | ], 21 | "author": "GoDaddy.com Operating Company LLC", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/godaddy/lighthouse4u/issues" 25 | }, 26 | "homepage": "https://github.com/godaddy/lighthouse4u#readme", 27 | "dependencies": { 28 | "@elastic/elasticsearch": "^8.0.0", 29 | "lodash": "^4.17.11" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/commands/init.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | command: 'init', 3 | desc: 'Initialize Store & Queue', 4 | handler: async argv => { 5 | // ! deps must be moved into function callback to avoid loading deps prior to start events 6 | const { getConfigs } = require('../config'); 7 | const getStorageClient = require('../store'); 8 | const getQueueClient = require('../queue'); 9 | 10 | const configs = await getConfigs(argv); 11 | 12 | const [config] = configs; 13 | 14 | try { 15 | const storage = getStorageClient(config); 16 | await storage.initialize(); 17 | } catch (ex) { 18 | console.error('Storage initialization failed:', ex.stack || ex); 19 | } 20 | 21 | try { 22 | const queue = getQueueClient(config); 23 | await queue.initialize(); 24 | } catch (ex) { 25 | console.error('Queue initialization failed:', ex.stack || ex); 26 | } 27 | 28 | process.exit(); 29 | } 30 | }; 31 | -------------------------------------------------------------------------------- /packages/lighthouse4u-es/README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse4u-es 2 | 3 | An Elasticsearch storage client for [Lighthouse4u](https://github.com/godaddy/lighthouse4u). 4 | 5 | ``` 6 | { 7 | store: { 8 | module: 'lighthouse4u-es', options: { 9 | client: { 10 | node: 'http://localhost:9200' 11 | }, 12 | index: { 13 | name: 'lh4u', 14 | type: 'lh4u' 15 | } 16 | } 17 | } 18 | } 19 | ``` 20 | 21 | 22 | ## Configuration Options 23 | 24 | | Option | Type | Default | Desc | 25 | | --- | --- | --- | --- | 26 | | module | `string` | **required** | Set this to `lighthouse4u-es` | 27 | | options | `object` | **required** | [Elasticsearch driver options](https://www.npmjs.com/package/@elastic/elasticsearch) | 28 | | ->.client | `ESOptions` | [See Defaults](./default-options.json#L2) | Options supplied to driver for connections | 29 | | ->.index | `ESIndexOptions` | [See Defaults](./default-options.json#L12) | Options supplied to driver upon creation of ES index | 30 | -------------------------------------------------------------------------------- /src/commands/server.js: -------------------------------------------------------------------------------- 1 | const { resolve } = require('path'); 2 | 3 | module.exports = { 4 | command: 'server', 5 | desc: 'Start HTTP Server', 6 | handler: async argv => { 7 | // ! deps must be moved into function callback to avoid loading deps prior to start events 8 | 9 | if (argv.beforeStart) { 10 | const beforeStart = require(resolve(argv.beforeStart)); 11 | if (typeof beforeStart === 'function') { 12 | beforeStart(argv); // forward argv 13 | } 14 | } 15 | 16 | process.on('unhandledRejection', err => console.error(err.stack || err)); 17 | 18 | const { getConfigs } = require('../config'); 19 | const [config] = await getConfigs(argv); 20 | 21 | const startListener = require('../http/start'); 22 | const getApp = require('../express'); 23 | 24 | const app = getApp(config); 25 | startListener(app); 26 | if (config.queue.enabled === true) { 27 | // only process queue if enabled 28 | const processQueue = require('../queue/process'); 29 | processQueue(app); 30 | } 31 | } 32 | }; 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 GoDaddy Operating Company, LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/queue/index.js: -------------------------------------------------------------------------------- 1 | const getSafeDocument = require('../util/get-safe-document'); 2 | const getClient = require('../util/get-client'); 3 | 4 | module.exports = class Queue { 5 | constructor(globalConfig) { 6 | this.config = globalConfig; 7 | // use `reader` config if avail, otherwise default to `store` 8 | this.queueConfig = globalConfig.queue; 9 | if (!this.queueConfig) throw new Error('Queue requires `queue` config'); 10 | 11 | this.client = getClient(this.queueConfig); 12 | } 13 | 14 | dequeue(opts) { 15 | return this.client.dequeue(opts); 16 | } 17 | 18 | ack(msg) { 19 | // some queue clients blow up by the existence of our dirty `data` prop 20 | const { data, ...safeProps } = msg; 21 | const safeMsg = { ...safeProps }; 22 | return this.client.ack(safeMsg); 23 | } 24 | 25 | nack(msg) { 26 | // some queue clients blow up by the existence of our dirty `data` prop 27 | const { data, ...safeProps } = msg; 28 | const safeMsg = { ...safeProps }; 29 | return this.client.ack(safeMsg); 30 | } 31 | 32 | enqueue(data, opts) { 33 | return this.client.enqueue(getSafeDocument(data), opts); 34 | } 35 | }; 36 | -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 GoDaddy Operating Company, LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/lighthouse4u-es/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 GoDaddy Operating Company, LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 GoDaddy Operating Company, LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 GoDaddy Operating Company, LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 GoDaddy Operating Company, LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse4u-amqp 2 | 3 | An [AMQP](https://www.npmjs.com/package/amqplib) queue client for [Lighthouse4u](https://github.com/godaddy/lighthouse4u). 4 | 5 | ``` 6 | { 7 | store: { 8 | module: 'lighthouse4u-amqp', options: { 9 | connect: { 10 | options: { 11 | url: 'amqp://user:password@host:5672/lh4u' 12 | } 13 | } 14 | queue: { 15 | name: 'lh4u' 16 | } 17 | } 18 | } 19 | } 20 | ``` 21 | 22 | 23 | ## Configuration Options 24 | 25 | | Option | Type | Default | Desc | 26 | | --- | --- | --- | --- | 27 | | module | `string` | **required** | Set this to `lighthouse4u-s3` | 28 | | options | `object` | **required** | S3 storage options | 29 | | ->.connect | `object` | **required** | AWS Region | 30 | | ->.connect.options | `object` | **required** | [Options](http://www.squaremobius.net/amqp.node/channel_api.html#connect) to connect to AMQP-compatible queue | 31 | | ->.connect.options.url | `string` | **required** | URL of AMQP-compatible queue | 32 | | ->.queue | `object` | **required** | Queue info | 33 | | ->.queue.name | `string` | **required** | Name of the queue | 34 | | ->.queue.options | `object` | optional | [Options](http://www.squaremobius.net/amqp.node/channel_api.html#channel_assertQueue) to initialize channel | 35 | -------------------------------------------------------------------------------- /src/config/default-config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | store: { 3 | options: {} 4 | }, 5 | queue: { 6 | idleDelayMs: 1000, // forced delay between queue checks when queue is empty 7 | enabled: true, 8 | options: {} 9 | }, 10 | http: { 11 | bindings: { 12 | http: { 13 | port: 8994 14 | } 15 | }, 16 | // authRedirect: 'https://some/other/place', 17 | auth: { 18 | /* 19 | basicAuthForMyGroup: { 20 | type: 'basic', 21 | groups: 'myGroup', // or [] for more than one, or '*' for any 22 | options: { 23 | user: 'user', 24 | pass: 'see secure/{env}' 25 | } 26 | }, 27 | customAuthForMyGroup: { 28 | type: 'custom', 29 | groups: 'myGroup', 30 | customPath: './lib/my-auth-handler.js', 31 | options: { 32 | some: 'option' 33 | } 34 | } 35 | */ 36 | }, 37 | routes: { 38 | // '/somePath': './routes/somePath.js' 39 | // OR 40 | // /somePath': { method: 'POST', path: './routes/post-somePath.js' } 41 | }, 42 | staticFiles: { 43 | // '/somePath': { 44 | // root: './path-to-static-folder', 45 | // ... other options: https://expressjs.com/en/4x/api.html#express.static 46 | // } 47 | } 48 | }, 49 | lighthouse: require('../lighthouse/defaults') 50 | }; 51 | -------------------------------------------------------------------------------- /src/express/api/get-website.js: -------------------------------------------------------------------------------- 1 | const getWebsite = require('../util/get-website'); 2 | const convertDocToSVG = require('../../util/convert-doc-to-svg'); 3 | const ReportGenerator = require('lighthouse/report/generator/report-generator.js'); 4 | 5 | const SVG_DEFAULT_WIDTH = 640; 6 | const SVG_DEFAULT_HEIGHT = 180; 7 | 8 | module.exports = async (req, res) => { 9 | 10 | const { format = 'json', scale = 1 } = req.query; 11 | 12 | let data; 13 | 14 | try { 15 | data = await getWebsite(req.app, req.query); 16 | } catch (ex) { 17 | console.error('store.getWebsite.err:', ex.stack || ex); 18 | return void res.sendStatus(400); 19 | } 20 | 21 | const files = data.files ? data.files : [data]; 22 | 23 | if (!files.length) { 24 | return void res.sendStatus(404); 25 | } 26 | 27 | if (format === 'svg') { // send SVG 28 | res.setHeader('Content-Type', 'image/svg+xml'); 29 | const svgOpts = { 30 | svgWidth: SVG_DEFAULT_WIDTH, svgHeight: SVG_DEFAULT_HEIGHT, scale 31 | }; 32 | res.render('pages/website-svg-full', convertDocToSVG(files[0], svgOpts)); 33 | } else if (format === 'reportJson') { 34 | res.send(JSON.parse(files[0]._report)); 35 | } else if (format === 'reportHtml') { 36 | res.send(ReportGenerator.generateReport(JSON.parse(files[0]._report), 'html')); 37 | } else { // else send JSON 38 | res.send(files.map(({ audits, i18n, _report, categoryGroups, ...others }) => ({ ...others, hasReport: !!_report }))); 39 | } 40 | }; 41 | -------------------------------------------------------------------------------- /src/express/api/website/get-compare.js: -------------------------------------------------------------------------------- 1 | const getWebsite = require('../../util/get-website'); 2 | const convertDocToSVG = require('../../../util/convert-doc-to-svg'); 3 | 4 | const SVG_DEFAULT_WIDTH = 640; 5 | const SVG_DEFAULT_HEIGHT = 382; 6 | 7 | module.exports = async (req, res) => { 8 | 9 | const { format = 'json', scale = 1, q1, q2 } = req.query; 10 | 11 | if (!q1 || !q2) { 12 | return void res.sendStatus(400); 13 | } 14 | 15 | let data1, data2; 16 | 17 | try { 18 | data1 = await getWebsite(req.app, q1); 19 | } catch (ex) { 20 | console.error('store.getWebsite.err:', ex.stack || ex); 21 | return void res.sendStatus(400); 22 | } 23 | 24 | try { 25 | data2 = await getWebsite(req.app, q2); 26 | } catch (ex) { 27 | console.error('store.getWebsite.err:', ex.stack || ex); 28 | return void res.sendStatus(400); 29 | } 30 | 31 | const files1 = data1.files ? data1.files : [data1]; 32 | const files2 = data2.files ? data2.files : [data2]; 33 | 34 | const [q1Result] = files1; 35 | const [q2Result] = files2; 36 | 37 | if (format === 'svg') { // send SVG 38 | res.setHeader('Content-Type', 'image/svg+xml'); 39 | const svgOpts = { 40 | svgWidth: SVG_DEFAULT_WIDTH, svgHeight: SVG_DEFAULT_HEIGHT, scale 41 | }; 42 | res.render('pages/website-svg-compare', { 43 | q1: convertDocToSVG(q1Result, svgOpts), 44 | q2: convertDocToSVG(q2Result, svgOpts) 45 | }); 46 | } else { // else send JSON 47 | res.send({ 48 | q1: q1Result, 49 | q2: q2Result 50 | }); 51 | } 52 | }; 53 | -------------------------------------------------------------------------------- /test/unit/util/get-score.test.js: -------------------------------------------------------------------------------- 1 | const { stub } = require('sinon'); 2 | const chai = require('chai'); 3 | const sinon = require('sinon'); 4 | chai.use(require('sinon-chai')); 5 | const { expect } = chai; 6 | const getMocks = require('../../mocks'); 7 | const path = require('path'); 8 | 9 | const libSrc = path.resolve('./src/util/get-score.js'); 10 | 11 | describe('/util/get-score', async () => { 12 | 13 | let mocks, lib, doc, cat; 14 | 15 | beforeEach(() => { 16 | mocks = getMocks(); 17 | doc = { 18 | state: 'processed', 19 | categories: { 20 | performance: { 21 | score: 0.59 22 | } 23 | } 24 | }; 25 | cat = 'performance'; 26 | lib = require(libSrc); 27 | }); 28 | 29 | it('returns 59%', async () => { 30 | const result = lib(doc, cat); 31 | expect(result).to.equal('59%'); 32 | }); 33 | 34 | it('does not handle greater tenths of a percentage', async () => { 35 | doc.categories.performance.score = 0.591; 36 | const result = lib(doc, cat); 37 | expect(result).to.equal('59%'); 38 | }); 39 | 40 | it('error state', async () => { 41 | doc.state = 'error'; 42 | const result = lib(doc, cat); 43 | expect(result).to.equal('error'); 44 | }); 45 | 46 | it('any unknown state is pending', async () => { 47 | doc.state = 'something unknown'; 48 | const result = lib(doc, cat); 49 | expect(result).to.equal('pending'); 50 | }); 51 | 52 | it('unknown category', async () => { 53 | cat = 'something unknown' 54 | const result = lib(doc, cat); 55 | expect(result).to.equal('?'); 56 | }); 57 | 58 | }); 59 | 60 | -------------------------------------------------------------------------------- /src/lighthouse/defaults.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | config: { // See: https://github.com/GoogleChrome/lighthouse/blob/master/lighthouse-core/config/default-config.js 3 | extends: 'lighthouse:default', // extends default 4 | logLevel: 'warn', 5 | chromeFlags: ['--headless', '--disable-gpu', '--no-sandbox'], // no-sandbox is required if running as root 6 | settings: { // See: https://github.com/GoogleChrome/lighthouse/blob/master/lighthouse-core/config/constants.js#L30 7 | output: 'json', 8 | extraHeaders: {}, 9 | throttling: 'mobile3G' // See: https://github.com/GoogleChrome/lighthouse/blob/master/lighthouse-core/config/constants.js#L16 10 | } 11 | }, 12 | validate: { 13 | // 'group-name': './validate/group-name.js' // throws if invalid response 14 | }, 15 | auditMode: 'simple', // how much of the `audits` report to retain (false, 'simple', 'details', 'all') -- it's very verbose: https://github.com/GoogleChrome/lighthouse/blob/master/docs/understanding-results.md#audits 16 | concurrency: 1, // number of concurrent lighthouse tests permitted -- a value greater than 1 may negatively impact accuracy of reports 17 | samples: { // number of tests run before median performance report is determined 18 | default: 1, 19 | range: [1, 5] // [min, max] 20 | }, 21 | attempts: { 22 | default: 2, 23 | range: [1, 10], 24 | delayMsPerExponent: 1000 // 2^attempt*delayPerExponent = 1s, 2s, 4s, 8s, etc 25 | }, 26 | delay: { 27 | default: 0, 28 | range: [0, 1000 * 60 * 60], // 1hr 29 | maxRequeueDelayMs: 1000 * 30 // max time before before delayed messages are requeued if still waiting 30 | } 31 | }; 32 | -------------------------------------------------------------------------------- /src/express/auth/index.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const URL = require('url'); 3 | 4 | const gInstances = {}; 5 | 6 | module.exports = ({ app, config: { http: { auth, authRedirect } } }) => { 7 | const authKeys = Object.keys(auth).filter(key => !!auth[key]); 8 | 9 | return async (req, res, next) => { 10 | 11 | const promises = authKeys.map(key => { 12 | const authConfig = auth[key]; 13 | 14 | const instance = getInstance(app, authConfig); 15 | 16 | return instance({ req, res }, authConfig.options || {}); 17 | }); 18 | 19 | let results; 20 | try { 21 | results = await Promise.all(promises); 22 | } catch (ex) { 23 | return void next(ex); 24 | } 25 | 26 | const groupsAllowed = results.reduce( 27 | (state, result, idx) => state 28 | || (result && (auth[authKeys[idx]].groups || '*')) 29 | || false, 30 | false 31 | ); 32 | 33 | req.groupsAllowed = groupsAllowed; 34 | 35 | if (groupsAllowed) return void next(); 36 | 37 | const { pathname } = URL.parse(req.url); 38 | const isUI = pathname === '/'; 39 | if (isUI && authRedirect) { 40 | // use redirect if provided 41 | res.writeHead(302, { Location: authRedirect }); 42 | return void res.end(); 43 | } 44 | 45 | const err = new Error('Unauthorized'); 46 | err.statusCode = 401; 47 | next(err); 48 | }; 49 | }; 50 | 51 | function getInstance(app, { type, customPath }) { 52 | let instance = gInstances[type]; 53 | if (!instance) { 54 | gInstances[type] = instance = require(customPath ? path.resolve(customPath) : `./${type}`)(app); 55 | } 56 | 57 | return instance; 58 | } 59 | -------------------------------------------------------------------------------- /src/cli.js: -------------------------------------------------------------------------------- 1 | const yargs = require('yargs'); 2 | const path = require('path'); 3 | const pkg = require('../package.json'); 4 | 5 | module.exports = yargs 6 | .commandDir(path.resolve(__dirname, '..', 'src', 'commands')) 7 | .option('before-start', { 8 | describe: 'Custom code to execute before starting HTTP Server', 9 | type: 'string' 10 | }) 11 | .option('config', { 12 | describe: 'One or more configuration files (with or without extension)', 13 | type: 'array', 14 | default: [] 15 | }) 16 | .option('config-dir', { 17 | describe: 'Directory of configuration files', 18 | type: 'string', 19 | default: 'config' 20 | }) 21 | .option('config-env', { 22 | describe: 'Environment variable used to detect configuration filename (ex: "development", "production", etc)', 23 | type: 'string', 24 | default: 'NODE_ENV' 25 | }) 26 | .option('config-default', { 27 | describe: 'Default configuration to use if environment is not available (ex: "local")', 28 | type: 'string', 29 | default: 'local' 30 | }) 31 | .option('config-base', { 32 | describe: 'Configuration to use as defaults for all environment configurations (ex: "defaults")', 33 | type: 'string' 34 | }) 35 | .option('config-exts', { 36 | describe: 'Supported extensions to detect for with configuration files', 37 | type: 'array', 38 | default: ['.json', '.json5', '.js'] 39 | }) 40 | .option('secure-config', { 41 | alias: 'secure-dir', 42 | describe: 'Directory of secure configuration files', 43 | type: 'string' 44 | }) 45 | .option('secure-file', { 46 | describe: 'File (or files if different per configuration) to ' + 47 | 'load that holds the secret required to decrypt secure configuration files', 48 | type: 'string' 49 | }) 50 | .help() 51 | .epilogue(`Lighthouse4u v${pkg.version}`) 52 | .demandCommand() 53 | .argv 54 | ; 55 | -------------------------------------------------------------------------------- /src/store/index.js: -------------------------------------------------------------------------------- 1 | const getSafeDocument = require('../util/get-safe-document'); 2 | const getClient = require('../util/get-client'); 3 | 4 | module.exports = class Storage { 5 | constructor(globalConfig) { 6 | this.config = globalConfig; 7 | // use `reader` config if avail, otherwise default to `store` 8 | this.readerConfig = this.config.reader || this.config.store; 9 | // use `writer` config if avail, otherwise default to `store` 10 | this.writerConfig = this.config.writer || this.config.store; 11 | if (!this.readerConfig) throw new Error('Storage requires either `reader` or `store` config'); 12 | if (!this.writerConfig) throw new Error('Storage requires either `writer` or `store` config'); 13 | 14 | this.readerClient = getClient(this.readerConfig); 15 | this.writerClient = getClient(this.writerConfig); 16 | } 17 | 18 | write(data, opts) { 19 | if (/^id:/.test(data.id)) data.id = data.id.substr(3); // remove prefix 20 | return this.writerClient.write(getSafeDocument(data), opts) 21 | .then(doc => { 22 | doc.id = `id:${doc.id}`; // prefix id's before forwarding 23 | return doc; 24 | }) 25 | ; 26 | } 27 | 28 | query(query, opts) { 29 | const isReadQuery = /^id:/.test(query); 30 | const isGroup = !/\./.test(query); 31 | if (isReadQuery) query = query.substr(3); // remove prefix before passing on to client 32 | else if (!isGroup && !/^http(s)?:/i.test(query)) query = `http://${query}`; // fully qualify URL, scheme is non-critical other than for parsing 33 | return this.readerClient[isReadQuery ? 'read' : 'find'](query, opts).then(data => { 34 | if (!data) return data; 35 | 36 | const mod = f => { f.id = `id:${f.id}`; }; 37 | if (data.files && Array.isArray(data.files)) { // files is array 38 | data.files.forEach(mod); 39 | } else { // data is file 40 | mod(data); 41 | } 42 | 43 | return data; 44 | }); 45 | } 46 | }; 47 | -------------------------------------------------------------------------------- /src/express/index.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const path = require('path'); 3 | const Storage = require('../store'); 4 | const Queue = require('../queue'); 5 | 6 | module.exports = config => { 7 | 8 | const app = express(); 9 | 10 | app.set('config', config); 11 | app.set('store', new Storage(config)); 12 | app.set('queue', new Queue(config)); 13 | app.set('view engine', 'ejs'); 14 | app.set('views', path.join(__dirname, 'views')); 15 | 16 | // mount `http.auth` if any provided 17 | if (Object.keys(config.http.auth).length > 0) { 18 | app.use(require('./auth')({ app, config })); 19 | } 20 | 21 | app.use('/static', express.static(path.join(__dirname, 'static'))); 22 | app.use('/api', express.json()); 23 | app.get('/', (req, res) => res.render('pages/index')); 24 | app.get('/api/website', require('./api/get-website')); 25 | app.get('/api/website/compare', require('./api/website/get-compare')); 26 | app.post('/api/website', require('./api/post-website')); 27 | 28 | // mount `http.staticFiles` if any provided 29 | Object.keys(config.http.staticFiles).forEach(route => { 30 | const staticOptions = config.http.staticFiles[route]; 31 | app.use(route, express.static(staticOptions.root, staticOptions)); 32 | }); 33 | 34 | // mount `http.routes` if any provided 35 | Object.keys(config.http.routes).forEach(key => { 36 | const route = config.http.routes[key]; 37 | const routeOptions = typeof route === 'string' ? {} : (route.options || route); 38 | const routePath = path.resolve(route.path || route); 39 | const routeMethod = route.method || 'get'; 40 | const routeModule = require(routePath)(app, routeOptions); 41 | const routeKey = route.route || key; 42 | app[routeMethod](routeKey, routeModule); 43 | }); 44 | 45 | app.use((err, req, res, next) => { 46 | 47 | console.error(err.stack || err); 48 | 49 | if (!res.headersSent) { 50 | res.status(err.statusCode || 500).send('Oops!'); 51 | } 52 | 53 | }); 54 | 55 | return app; 56 | 57 | }; 58 | -------------------------------------------------------------------------------- /test/unit/express/api/post-website.test.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'); 2 | const sinon = require('sinon'); 3 | chai.use(require('sinon-chai')); 4 | const { expect } = chai; 5 | const getMocks = require('../../../mocks'); 6 | const path = require('path'); 7 | const proxyquire = require('proxyquire').noPreserveCache(); 8 | 9 | const libSrc = path.resolve('./src/express/api/post-website.js'); 10 | 11 | describe('/express/api/post-website', async () => { 12 | 13 | let lib, mocks, meta; 14 | 15 | beforeEach(() => { 16 | mocks = getMocks(); 17 | meta = { identifier: 'test' }; 18 | mocks.request.body = { 19 | 'url': 'https://www.google.com/', 20 | 'throttling': 'mobile3G', 21 | 'group': 'web', 22 | 'headers': {}, 23 | 'secureHeaders': { secretKey: 'secretValue' }, 24 | 'cookies': [{ cookie1: 'c1', cookie2: { domain: 'domain', url: 'url', value: 'c2' } }], 25 | 'commands': [{ command: 'command' }], 26 | 'meta': meta 27 | }; 28 | lib = proxyquire(libSrc, mocks); 29 | }); 30 | 31 | it('successful', async () => { 32 | await lib(mocks.request, mocks.response); 33 | expect(mocks.store.write).to.be.calledOnce; 34 | expect(mocks.store.write.args[0][0].requestedUrl).to.equal('https://www.google.com/'); 35 | expect(mocks.store.write.args[0][0].domainName).to.equal('www.google.com'); 36 | expect(mocks.store.write.args[0][0].rootDomain).to.equal('google.com'); 37 | expect(mocks.response.sendStatus).to.not.be.called; 38 | expect(mocks.response.send).to.be.calledOnce; 39 | expect(mocks.response.send.args[0][0].id).to.equal('documentId'); 40 | }); 41 | 42 | it('fail to write', async () => { 43 | mocks.store.write.rejects('oops!'); 44 | await lib(mocks.request, mocks.response); 45 | expect(mocks.store.write).to.be.calledOnce; 46 | expect(mocks.response.sendStatus).to.be.calledOnce; 47 | expect(mocks.response.sendStatus).to.be.calledWith(500); 48 | }); 49 | 50 | it('passes through meta', async () => { 51 | await lib(mocks.request, mocks.response); 52 | expect(mocks.response.send.args[0][0].meta).to.equal(meta); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lighthouse4u", 3 | "version": "1.10.1", 4 | "description": "LH4U provides Google Lighthouse as a service, surfaced by both a friendly UI+API, and backed by Elastic Search for all your query and visualization needs", 5 | "main": "./src/cli.js", 6 | "bin": { 7 | "lh4u": "./bin/lh4u", 8 | "lighthouse4u": "./bin/lh4u" 9 | }, 10 | "scripts": { 11 | "eslint": "eslint-godaddy -c .eslintrc src/", 12 | "cover": "nyc mocha test/", 13 | "pretest": "npm run eslint", 14 | "report": "nyc report --reporter=lcov", 15 | "init": "./bin/lh4u init --config local --config-dir ./test/config --config-base defaults", 16 | "start": "./bin/lh4u server --config local --config-dir ./test/config --config-base defaults --before-start ./test/local/before-start.js", 17 | "test": "npm run cover && npm run report" 18 | }, 19 | "engines": { 20 | "node": ">=8.8" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "git+https://github.com/godaddy/lighthouse4u.git" 25 | }, 26 | "keywords": [ 27 | "lighthouse", 28 | "google", 29 | "chrome", 30 | "devtools" 31 | ], 32 | "files": [ 33 | "bin", 34 | "src", 35 | "LICENSE", 36 | "README.md", 37 | "SECURITY.md", 38 | "package.json" 39 | ], 40 | "author": "GoDaddy.com Operating Company LLC", 41 | "license": "MIT", 42 | "dependencies": { 43 | "basic-auth": "^2.0.1", 44 | "chrome-launcher": "^0.15.0", 45 | "config-shield": "^0.2.3", 46 | "cross-fetch": "^3.1.4", 47 | "ejs": "^3.1.6", 48 | "express": "^4.19.2", 49 | "json5": "^2.2.0", 50 | "lighthouse": "^9.6.8", 51 | "lodash": "^4.17.21", 52 | "papaparse": "^5.3.1", 53 | "tldjs": "^2.3.1", 54 | "yargs": "^15.4.1" 55 | }, 56 | "devDependencies": { 57 | "amqplib": "^0.10.2", 58 | "chai": "^4.3.4", 59 | "eslint": "^8.0.0", 60 | "eslint-config-godaddy": "^4.0.1", 61 | "eslint-plugin-json": "^2.1.2", 62 | "eslint-plugin-mocha": "^7.0.1", 63 | "fs-extra": "^9.1.0", 64 | "mocha": "^7.2.0", 65 | "nyc": "^15.1.0", 66 | "proxyquire": "^2.1.3", 67 | "sinon": "^9.2.4", 68 | "sinon-chai": "^3.7.0" 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /packages/lighthouse4u-sqs/index.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | 3 | module.exports = class QueueSQS { 4 | constructor({ region, queueUrl } = {}) { 5 | if (region) { 6 | AWS.config.update({ region }); 7 | } 8 | this.sqs = new AWS.SQS({ }); 9 | this.QueueUrl = queueUrl; 10 | } 11 | 12 | initialize() { 13 | // no special initialization support 14 | return Promise.resolve(); 15 | } 16 | 17 | async dequeue() { 18 | const { QueueUrl } = this; 19 | const params = { QueueUrl, MaxNumberOfMessages: 1, WaitTimeSeconds: 1 }; 20 | return new Promise((resolve, reject) => { 21 | this.sqs.receiveMessage(params, (err, data) => { 22 | if (err) return void reject(err); 23 | 24 | if (!data.Messages) return resolve(); 25 | 26 | data.Messages.forEach(msg => { 27 | // msg.data is only required LH4U property 28 | msg.data = JSON.parse(msg.Body); 29 | }); 30 | 31 | resolve(data.Messages[0]); // first only 32 | }); 33 | }); 34 | } 35 | 36 | ack({ ReceiptHandle }) { 37 | const { QueueUrl } = this; 38 | const params = { 39 | QueueUrl, 40 | ReceiptHandle 41 | }; 42 | 43 | return new Promise((resolve, reject) => { 44 | this.sqs.deleteMessage(params, (err) => { 45 | if (err) return void reject(err); 46 | 47 | resolve(); 48 | }) 49 | }); 50 | } 51 | 52 | nack({ ReceiptHandle }) { 53 | const { QueueUrl } = this; 54 | const params = { 55 | QueueUrl, 56 | ReceiptHandle, 57 | VisibilityTimeout: 0 // make available for re-processing immediately 58 | }; 59 | 60 | return new Promise((resolve, reject) => { 61 | this.sqs.changeMessageVisibility(params, (err) => { 62 | if (err) return void reject(err); 63 | 64 | resolve(); 65 | }) 66 | }); 67 | } 68 | 69 | enqueue(data) { 70 | const { QueueUrl } = this; 71 | const MessageBody = JSON.stringify(data, null, 2); 72 | const params = { 73 | QueueUrl, 74 | MessageBody, 75 | DelaySeconds: 0 76 | }; 77 | 78 | return new Promise((resolve, reject) => { 79 | this.sqs.sendMessage(params, (err, data) => { 80 | if (err) return void reject(err); 81 | 82 | resolve(data); 83 | }) 84 | }); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /packages/lighthouse4u-amqp/index.js: -------------------------------------------------------------------------------- 1 | const amqplib = require('amqplib'); 2 | 3 | module.exports = class QueueSQS { 4 | constructor(opts) { 5 | if (!opts) throw new Error('Options required'); 6 | if (!opts.connect || !opts.connect.options) throw new Error('`connect.options` required'); 7 | if (!opts.queue || !opts.queue.name) throw new Error('`queue.name` required'); 8 | 9 | this.options = opts; 10 | this.connect(); 11 | } 12 | 13 | async connect(forceReconnect) { 14 | if (!forceReconnect && this.connection && this.channel) return this.channel; 15 | 16 | this.connection = await amqplib.connect(this.options.connect.options.url || this.options.connect.options); 17 | this.channel = await this.connection.createChannel(); 18 | await this.channel.assertQueue(this.options.queue.name, this.options.queue.options || {}); 19 | 20 | return this.channel; 21 | } 22 | 23 | initialize() { 24 | // no special initialization support 25 | return Promise.resolve(); 26 | } 27 | 28 | async dequeue() { 29 | let channel, msg; 30 | 31 | try { 32 | channel = await this.connect(); 33 | msg = await channel.get(this.options.queue.name, { noAck: false }); 34 | } catch (ex) { 35 | this.connect(true); // reconnect 36 | throw ex; 37 | } 38 | 39 | if (!msg) return; 40 | 41 | let data; 42 | try { 43 | data = JSON.parse(msg.content.toString()); 44 | } catch (ex) { 45 | channel.ack(msg); 46 | console.warn('RMQP.get returned invalid message', msg, ex.stack); 47 | return; 48 | } 49 | if (!msg.content || !data) { 50 | channel.ack(msg); 51 | console.warn('RMQP.get returned an invalid message', msg); 52 | return; 53 | } 54 | 55 | msg.data = data; 56 | 57 | return msg; 58 | } 59 | 60 | async ack(msg) { 61 | const channel = await this.connect(); 62 | 63 | channel.ack(msg); 64 | 65 | return Promise.resolve(); 66 | } 67 | 68 | async nack(msg) { 69 | const channel = await this.connect(); 70 | 71 | channel.nack(msg); 72 | 73 | return Promise.resolve(); 74 | } 75 | 76 | async enqueue(data) { 77 | try { 78 | const channel = await this.connect(); 79 | 80 | if (!channel.sendToQueue(this.options.queue.name, Buffer.from(JSON.stringify(data)))) { 81 | throw new Error('Failed to write to queue'); 82 | } 83 | } catch (ex) { 84 | this.connect(true); // reconnect 85 | throw ex; 86 | } 87 | 88 | return Promise.resolve(); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /packages/lighthouse4u-es/default-options.json: -------------------------------------------------------------------------------- 1 | { 2 | "client": { 3 | "node": "http://localhost:9200", 4 | "log": { 5 | "type": "stdio", 6 | "levels": [ 7 | "error", 8 | "warning" 9 | ] 10 | } 11 | }, 12 | "index": { 13 | "name": "lh4u", 14 | "type": "lh4u", 15 | "settings": { 16 | "number_of_shards": 3, 17 | "number_of_replicas": 2 18 | }, 19 | "mappings": { 20 | "lh4u": { 21 | "properties": { 22 | "state": { 23 | "type": "string", 24 | "index": "not_analyzed" 25 | }, 26 | "requestedUrl": { 27 | "type": "string", 28 | "index": "not_analyzed" 29 | }, 30 | "finalUrl": { 31 | "type": "string", 32 | "index": "not_analyzed" 33 | }, 34 | "domainName": { 35 | "type": "string", 36 | "index": "not_analyzed" 37 | }, 38 | "rootDomain": { 39 | "type": "string", 40 | "index": "not_analyzed" 41 | }, 42 | "group": { 43 | "type": "string", 44 | "index": "not_analyzed" 45 | }, 46 | "categories": { 47 | "type": "object", 48 | "properties": { 49 | "accessibility": { 50 | "type": "object", 51 | "properties": { 52 | "score": { 53 | "type": "double" 54 | } 55 | } 56 | }, 57 | "best-practices": { 58 | "type": "object", 59 | "properties": { 60 | "score": { 61 | "type": "double" 62 | } 63 | } 64 | }, 65 | "performance": { 66 | "type": "object", 67 | "properties": { 68 | "score": { 69 | "type": "double" 70 | } 71 | } 72 | }, 73 | "pwa": { 74 | "type": "object", 75 | "properties": { 76 | "score": { 77 | "type": "double" 78 | } 79 | } 80 | }, 81 | "seo": { 82 | "type": "object", 83 | "properties": { 84 | "score": { 85 | "type": "double" 86 | } 87 | } 88 | } 89 | } 90 | }, 91 | "createDate": { 92 | "type": "date" 93 | } 94 | } 95 | } 96 | } 97 | } 98 | } -------------------------------------------------------------------------------- /packages/lighthouse4u-es/index.js: -------------------------------------------------------------------------------- 1 | const URL = require('url'); 2 | const elasticsearch = require('@elastic/elasticsearch'); 3 | const { defaultsDeep } = require('lodash'); 4 | const defaultOptions = require('./default-options.json'); 5 | 6 | module.exports = class StoreES { 7 | constructor(options) { 8 | this.config = defaultsDeep({}, options, defaultOptions); 9 | this.client = new elasticsearch.Client(this.config.client); 10 | } 11 | 12 | initialize() { 13 | const mapping = { 14 | index: this.config.index.name, 15 | type: this.config.index.type, 16 | body: { 17 | settings: this.config.index.settings, 18 | mappings: this.config.index.mappings 19 | } 20 | }; 21 | 22 | const promise = this.client.indices.create(mapping); 23 | console.log('Elasticsearch index created:', mapping); 24 | return promise; 25 | } 26 | 27 | read(Key, { etag } = {}) { 28 | return this.client.get({ 29 | index: this.config.index.name, 30 | type: this.config.index.type, 31 | id: Key 32 | }).then(({ body }) => { 33 | const ret = body._source; 34 | 35 | ret.id = Key; 36 | 37 | return ret; 38 | }); 39 | } 40 | 41 | list(query, { resumeKey, maxCount = 10, order = 'DESC' } = {}) { 42 | const q = convertQueryToES(query); 43 | return this.client.search({ 44 | index: this.config.index.name, 45 | type: this.config.index.type, 46 | q, 47 | body: { 48 | size: maxCount, 49 | sort: [{ createDate: { order: order.toLowerCase() }}] 50 | } 51 | }).then(({ body }) => { 52 | const { hits } = body; 53 | const files = hits.hits.map(hit => { 54 | const ret = hit._source; 55 | ret.id = hit._id; 56 | 57 | return ret; 58 | }); 59 | 60 | return { 61 | resumeKey: null, // TODO 62 | files 63 | } 64 | }); 65 | } 66 | 67 | async find(url, opts) { 68 | return this.list(url, opts); 69 | } 70 | 71 | write(data, { meta = {} } = {}) { 72 | return this.client.index({ 73 | index: this.config.index.name, 74 | type: this.config.index.type, 75 | id: data.id, 76 | body: data 77 | }).then(({ body }) => { 78 | data.id = body._id; 79 | 80 | return data; 81 | }); 82 | } 83 | } 84 | 85 | function convertQueryToES(val) { 86 | const protoMatch = /^https?:\/\/(.*)/i.exec(val); 87 | const protoLess = protoMatch ? protoMatch[1] : val; 88 | if (/\//.test(protoLess)) return `requestedUrl:"${val}"`; 89 | else if (/\.(.*)\./.test(val)) return `domainName:"${protoLess}"`; 90 | else if (/\./.test(val)) return `rootDomain:"${protoLess}"`; 91 | else if (/id:/.test(protoLess)) return protoLess; // passthrough if ID 92 | return `group:"${protoLess}"`; 93 | } 94 | -------------------------------------------------------------------------------- /src/express/views/pages/index.ejs: -------------------------------------------------------------------------------- 1 | 2 | <%- include('../partials/head') %> 3 | 4 | 5 |
6 |
<%- include('../partials/header') %>
7 | 8 |
9 |
10 | Run Lighthouse 11 |
12 |
13 | Docs 14 |
23 | 24 |
25 | 26 | 27 | 28 |
29 | 30 |
31 | 32 |
33 |
34 | Website Info 35 |
36 |
37 |
38 |
39 | 40 | 41 | Query by document (via `id:{myDocumentId}`), or URL, or partial URL. 42 |
43 |
44 | 45 | 46 | No more than 1 result will be returned if querying by document ID. 47 |
48 |
49 | 50 | 51 |
52 |
53 |
54 |
55 |
56 | SVG JSON Lighthouse Report Lighthouse JSON
57 | 58 | 59 |
60 |
61 | 62 | 63 |
64 | 65 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '39 0 * * 1' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'javascript' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /test/unit/express/api/get-website.test.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'); 2 | const sinon = require('sinon'); 3 | chai.use(require('sinon-chai')); 4 | const { expect } = chai; 5 | const getMocks = require('../../../mocks'); 6 | const path = require('path'); 7 | const proxyquire = require('proxyquire').noPreserveCache(); 8 | 9 | const libSrc = path.resolve('./src/express/api/get-website.js'); 10 | 11 | describe('/express/api/get-website', async () => { 12 | 13 | let lib, mocks; 14 | 15 | beforeEach(() => { 16 | mocks = getMocks(); 17 | lib = proxyquire(libSrc, mocks); 18 | }); 19 | 20 | it('get by q', async () => { 21 | mocks.request.query.q = 'documentId'; 22 | await lib(mocks.request, mocks.response); 23 | expect(mocks.store.query).to.be.calledOnce; 24 | expect(mocks.response.send).to.be.calledOnce; 25 | }); 26 | 27 | it('fail to get by q', async () => { 28 | expect(mocks.store.query).to.not.be.called; 29 | await lib(mocks.request, mocks.response); 30 | expect(mocks.store.query).to.not.be.called; 31 | expect(mocks.response.sendStatus).to.be.calledOnce; 32 | expect(mocks.response.sendStatus).to.be.calledWith(400); 33 | }); 34 | 35 | it('get by domainName', async () => { 36 | mocks.request.query.domainName = 'domainName'; 37 | await lib(mocks.request, mocks.response); 38 | expect(mocks.response.send).to.be.calledOnce; 39 | }); 40 | 41 | it('get SVG', async () => { 42 | mocks.request.query.q = 'documentId'; 43 | mocks.request.query.format = 'svg'; 44 | await lib(mocks.request, mocks.response); 45 | expect(mocks.store.query).to.be.calledOnce; 46 | expect(mocks.store.query.args[0][0]).to.equal('documentId'); 47 | expect(mocks.response.render).to.be.calledOnce; 48 | expect(mocks.response.render.args[0][1].requestedUrl).to.equal('requestedUrl'); 49 | expect(mocks.response.render.args[0][1].state).to.equal('processed'); 50 | expect(mocks.response.setHeader).to.be.calledOnce; 51 | }); 52 | 53 | it('get SVG in pending state', async () => { 54 | mocks.request.query.q = 'documentId'; 55 | mocks.request.query.format = 'svg'; 56 | mocks.store['$document'].state = 'pending'; 57 | await lib(mocks.request, mocks.response); 58 | expect(mocks.store.query).to.be.calledOnce; 59 | expect(mocks.response.render).to.be.calledOnce; 60 | expect(mocks.store.query.args[0][0]).to.equal('documentId'); 61 | expect(mocks.response.render.args[0][1].requestedUrl).to.equal('requestedUrl'); 62 | expect(mocks.response.render.args[0][1].state).to.equal('pending'); 63 | expect(mocks.response.setHeader).to.be.calledOnce; 64 | }); 65 | 66 | it('get SVG in error state', async () => { 67 | mocks.request.query.q = 'documentId'; 68 | mocks.request.query.format = 'svg'; 69 | mocks.store['$document'].state = 'error'; 70 | await lib(mocks.request, mocks.response); 71 | expect(mocks.store.query).to.be.calledOnce; 72 | expect(mocks.store.query.args[0][0]).to.equal('documentId'); 73 | expect(mocks.response.render).to.be.calledOnce; 74 | expect(mocks.response.render.args[0][1].requestedUrl).to.equal('requestedUrl'); 75 | expect(mocks.response.render.args[0][1].state).to.equal('error'); 76 | expect(mocks.response.setHeader).to.be.calledOnce; 77 | }); 78 | 79 | }); 80 | -------------------------------------------------------------------------------- /packages/lighthouse4u-fs/index.js: -------------------------------------------------------------------------------- 1 | const URL = require('url'); 2 | const fs = require('fs'); 3 | const path = require('path'); 4 | const { ensureDir } = require('fs-extra'); 5 | 6 | module.exports = class StoreS3 { 7 | constructor({ dir } = {}) { 8 | this.dir = dir; 9 | } 10 | 11 | initialize() { 12 | // no special initialization support 13 | return Promise.resolve(); 14 | } 15 | 16 | read(relPath) { 17 | const absPath = path.resolve(this.dir, relPath); 18 | 19 | return new Promise((resolve, reject) => { 20 | fs.readFile(absPath, 'utf8', (err, json) => { 21 | if (err) return void reject(err); 22 | 23 | const data = JSON.parse(json); 24 | data.id = relPath; 25 | 26 | resolve(data); 27 | }); 28 | }); 29 | } 30 | 31 | list(url, { /*resumeKey, */maxCount = 10 } = {}) { 32 | const relDir = urlToDir(url); 33 | const absPath = path.resolve(this.dir, relDir); 34 | 35 | return new Promise(resolve => { 36 | fs.readdir(absPath, {}, (err, dirFiles) => { 37 | if (err) return void resolve({ files: [] }); 38 | 39 | // !!! No support for recursion or resuming (resumeKey) -- overkill for this client since it's for dev only at present 40 | 41 | // crazy, but sort+reverse for DESC order is faster than custom sorter 42 | const files = dirFiles.sort().reverse().slice(0, maxCount).map(f => ({ id: path.join(relDir, f) })); 43 | 44 | resolve({ 45 | files 46 | }); 47 | }); 48 | }); 49 | } 50 | 51 | async find(url, opts) { 52 | const ls = await this.list(url, opts); 53 | // read all docs 54 | await Promise.all(ls.files.map(async f => { 55 | const data = await this.read(f.id); 56 | 57 | // copy props 58 | Object.assign(f, data); 59 | 60 | return f; // resolve value is unused 61 | })); 62 | 63 | return ls; 64 | } 65 | 66 | write(data) { 67 | const relPath = data.id ? data.id : urlToKey(data.requestedUrl); 68 | const absPath = path.resolve(this.dir, relPath); 69 | const json = JSON.stringify(data, null, 2); 70 | return new Promise(async (resolve, reject) => { 71 | await ensureDir(path.dirname(absPath)); 72 | fs.writeFile(absPath, json, 'utf8', err => { 73 | if (err) return void reject(err); 74 | 75 | data.id = relPath; 76 | data.size = json.length; 77 | 78 | resolve(data); 79 | }) 80 | }); 81 | } 82 | } 83 | 84 | function urlToDir(url) { 85 | const urlInfo = URL.parse(url); 86 | return `${urlInfo.hostname}${urlToPath(urlInfo.pathname)}${urlInfo.pathname[urlInfo.pathname.length - 1] === '/' ? '' : `/`}` 87 | } 88 | 89 | function urlToKey(url) { 90 | // it's important that the key can be ordered by time 91 | const lexicalTime = new Date().toISOString(); 92 | const urlInfo = URL.parse(url); 93 | return `${urlInfo.hostname}${urlToPath(urlInfo.pathname)}${urlInfo.pathname[urlInfo.pathname.length - 1] === '/' ? lexicalTime : `/${lexicalTime}`}.json` 94 | } 95 | 96 | function urlToPath(key) { 97 | // Note: replace all non-alphanumeric characters as precaution across *nix/windows 98 | return key.replace(/[^a-zA-Z0-9\\\/]/g, '_'); 99 | } 100 | -------------------------------------------------------------------------------- /src/queue/process.js: -------------------------------------------------------------------------------- 1 | const submitWebsite = require('../lighthouse/submit'); 2 | const getSafeDocument = require('../util/get-safe-document'); 3 | 4 | module.exports = async app => { 5 | main(app); 6 | }; 7 | 8 | async function main(app) { 9 | const config = app.get('config'); 10 | const store = app.get('store'); 11 | const queue = app.get('queue'); 12 | 13 | const { idleDelayMs } = config.queue; 14 | getNextMessage({ app, config, queue, store, idleDelayMs }); 15 | } 16 | 17 | async function getNextMessage(options) { 18 | const { idleDelayMs, queue, config } = options; 19 | const { lighthouse } = config; 20 | 21 | const msg = await queue.dequeue(); 22 | 23 | if (!msg) return void setTimeout(getNextMessage.bind(null, options), idleDelayMs); 24 | 25 | const { data } = msg; 26 | 27 | let delayTime = data.delayTime; 28 | if (!delayTime || delayTime <= Date.now()) { 29 | await processMessage(options, msg, data); 30 | 31 | delayTime = data.state === 'retry' && data.delayTime; 32 | } 33 | 34 | if (delayTime) { // future message 35 | // message is delayed 36 | const waitBeforeRequeue = Math.min(lighthouse.delay.maxRequeueDelayMs, delayTime - Date.now()); 37 | setTimeout(() => { 38 | // queue the modified request 39 | queue.enqueue(data); 40 | 41 | // drop the old msg 42 | queue.ack(msg); 43 | }, waitBeforeRequeue).unref(); // no need to hold ref, msg will be requeued if connection broken 44 | 45 | // notice ^ is non-blocking, we'll continue to process other messages even during delay 46 | } 47 | 48 | getNextMessage(options); 49 | } 50 | 51 | async function processMessage(options, msg, data) { 52 | const { config, queue, store } = options; 53 | const { lighthouse } = config; 54 | 55 | const attempts = Math.min( 56 | Math.max(data.attempts || lighthouse.attempts.default, lighthouse.attempts.range[0]), 57 | lighthouse.attempts.range[1] 58 | ); 59 | 60 | data.attempt = data.attempt || 1; 61 | 62 | try { 63 | const results = await submitWebsite(data.requestedUrl, config, data); 64 | data.state = 'processed'; 65 | data = Object.assign(data, results); 66 | delete data.headers; // no longer needed 67 | 68 | // update store and ACK msg from queue 69 | try { 70 | data = await store.write(data); 71 | queue && queue.ack(msg); 72 | } catch (ex) { 73 | console.error('failed to write to store!', ex.stack || ex); 74 | 75 | // retry 76 | queue && queue.nack(msg); 77 | } 78 | } catch (ex) { 79 | console.error(`lighthouse failed! attempt ${data.attempt} of ${attempts}`, getSafeDocument(data), ex.stack || ex); 80 | 81 | // retry unless out of attempts 82 | if (data.attempt >= attempts) { 83 | data.state = 'error'; 84 | data.errorMessage = (ex.stack || ex).toString(); 85 | } else { 86 | data.attempt++; 87 | data.state = 'retry'; 88 | // calc time before next attempt using exponential backoff 89 | data.delayTime = Date.now() + Math.pow(2, data.attempt) * config.lighthouse.attempts.delayMsPerExponent; 90 | } 91 | 92 | // save failure state to store 93 | try { 94 | data = await store.write(data); 95 | } catch (ex2) { 96 | console.error('failed to write to store!', getSafeDocument(data), ex2.stack || ex2); 97 | 98 | // nothing more to do 99 | } 100 | 101 | if (data.state === 'error') { // we've given up 102 | queue && queue.ack(msg); 103 | } 104 | } 105 | 106 | return data; 107 | } 108 | 109 | module.exports.processMessage = processMessage; 110 | -------------------------------------------------------------------------------- /src/config/get-config-by-id.js: -------------------------------------------------------------------------------- 1 | const configShield = require('config-shield'); 2 | const path = require('path'); 3 | const fs = require('fs'); 4 | const json5 = require('json5'); 5 | const { merge } = require('lodash'); 6 | const { promisify } = require('util'); 7 | const defaultConfig = require('./default-config'); 8 | 9 | const readFile = promisify(fs.readFile); 10 | const cshieldLoad = configShield.load.bind(configShield); 11 | 12 | const gConfigs = {}; 13 | 14 | module.exports = async (configName, argv) => { 15 | const configDir = path.resolve(argv.configDir); 16 | 17 | if (argv.secureConfig && !argv.secureFile) throw new Error('secure-config requires associated secure-file. secure-secret not yet supported.'); 18 | 19 | const baseConfigPromise = !argv.configBase ? {} : loadConfig(configDir, argv.configBase, argv); 20 | const envConfigPromise = loadConfig(configDir, configName, argv); 21 | let secureConfigPromise; 22 | if (argv.secureConfig) { 23 | const ext = path.extname(configName); 24 | const configId = path.basename(configName, ext); 25 | 26 | secureConfigPromise = new Promise(async (resolve, reject) => { 27 | const cshield = await cshieldLoad({ 28 | instance: configId, 29 | configPath: path.join(argv.secureConfig, `${configId}.json`), 30 | privateKeyPath: argv.secureFile 31 | }); 32 | 33 | const config = {}; 34 | // merge secure config into base config 35 | cshield.getKeys().forEach(k => { 36 | config[k] = cshield.getProp(k); 37 | }); 38 | 39 | resolve(config); 40 | }); 41 | } else { 42 | secureConfigPromise = {}; // resolve to empty object 43 | } 44 | 45 | return new Promise((resolve, reject) => { 46 | 47 | Promise.all([baseConfigPromise, envConfigPromise, secureConfigPromise]) 48 | .then(([baseConfig, envConfig, secureConfig]) => { 49 | 50 | const finalConfig = merge({}, defaultConfig, baseConfig, envConfig, secureConfig); 51 | 52 | if (typeof finalConfig.httpHandler === 'string') { 53 | finalConfig.httpHandler = require(path.resolve(finalConfig.httpHandler)); // app-relative path 54 | } 55 | 56 | resolve(finalConfig); 57 | }) 58 | .catch(err => reject(err)) 59 | ; 60 | 61 | }); 62 | }; 63 | 64 | async function loadConfig(configDir, configName, argv) { 65 | // IKNOWRIGHT: 66 | // there are some minor blocking calls in here, but they should be reasonable 67 | // being they are once-per-config calls. 68 | 69 | let ext = path.extname(configName); 70 | const configId = path.basename(configName, ext); 71 | const gConfig = gConfigs[configId]; 72 | if (gConfig) return gConfig; // return global object if already avail 73 | let absPath = path.join(configDir, configName); 74 | 75 | if (!ext) { 76 | // auto-detect 77 | for (let i = 0; i < argv.configExts.length; i++) { 78 | ext = argv.configExts[i]; 79 | if (fs.existsSync(absPath + ext)) { // found 80 | absPath += ext; 81 | break; 82 | } else { // not found 83 | ext = null; 84 | } 85 | } 86 | if (!ext) { // ext not detected 87 | throw new Error(`Configuration not found: ${absPath}`); 88 | } 89 | } 90 | 91 | let o; 92 | 93 | if (!/\.json5?/.test(ext)) { 94 | // perform require on commonJS 95 | 96 | o = require(absPath); 97 | o.id = configId; 98 | gConfigs[absPath] = o; // store in global object 99 | 100 | return o; 101 | } 102 | 103 | // for json/json5 files, utilize json5 loader 104 | 105 | const data = await readFile(absPath, 'utf8'); 106 | 107 | o = json5.parse(data); 108 | o.id = configId; 109 | 110 | gConfigs[configId] = o; // store in global object 111 | 112 | return o; 113 | } 114 | -------------------------------------------------------------------------------- /packages/lighthouse4u-s3/index.js: -------------------------------------------------------------------------------- 1 | const URL = require('url'); 2 | const AWS = require('aws-sdk'); 3 | 4 | module.exports = class StoreS3 { 5 | constructor({ region, bucket } = {}) { 6 | if (region) { 7 | AWS.config.update({ region }); 8 | } 9 | this.s3 = new AWS.S3({ }); 10 | this.Bucket = bucket; 11 | } 12 | 13 | initialize() { 14 | // no special initialization support 15 | return Promise.resolve(); 16 | } 17 | 18 | read(Key, { etag } = {}) { 19 | const { Bucket } = this; 20 | const params = { Bucket, Key: decodeURIComponent(Key) }; 21 | if (etag) params.IfNoneMatch = etag; 22 | return new Promise((resolve, reject) => { 23 | this.s3.getObject(params, (err, res) => { 24 | if (err) return void reject(err); 25 | 26 | const data = JSON.parse(res.Body); 27 | 28 | data.id = Key; 29 | data.etag = res.ETag; 30 | 31 | resolve(data); 32 | }); 33 | }); 34 | } 35 | 36 | list(url, { resumeKey, maxCount = 10, order = 'DESC' } = {}) { 37 | const Prefix = urlToS3Dir(url); 38 | const { Bucket } = this; 39 | const params = { 40 | Bucket, 41 | Delimiter: '/', 42 | EncodingType: 'url', 43 | FetchOwner: false, 44 | ContinuationToken: resumeKey, 45 | MaxKeys: maxCount, 46 | Prefix 47 | }; 48 | 49 | return new Promise((resolve, reject) => { 50 | this.s3.listObjectsV2(params, (err, data) => { 51 | if (err) return void reject(err); 52 | 53 | const files = data.Contents.map(f => ({ 54 | id: f.Key, 55 | etag: f.ETag 56 | })); 57 | 58 | resolve({ 59 | resumeKey: data.IsTruncated ? data.NextContinuationToken : undefined, 60 | files 61 | }); 62 | }); 63 | }); 64 | } 65 | 66 | async find(url, opts) { 67 | const ls = await this.list(url, opts); 68 | 69 | // read all docs 70 | await Promise.all(ls.files.map(async f => { 71 | const data = await this.read(f.id); 72 | 73 | // copy props 74 | Object.assign(f, data); 75 | 76 | return f; // resolve value is unused 77 | })); 78 | 79 | return ls; 80 | } 81 | 82 | write(data, { meta = {} } = {}) { 83 | const { Bucket } = this; 84 | const Key = data.id ? data.id : urlToS3Key(data.requestedUrl); 85 | const Body = JSON.stringify(data, null, 2); 86 | const Metadata = meta; 87 | const ContentType = 'application/json' 88 | const params = { Bucket, Key, Body, Metadata, ContentType }; 89 | 90 | return new Promise((resolve, reject) => { 91 | this.s3.upload(params, (err, res) => { 92 | if (err) return void reject(err); 93 | 94 | data.id = Key; 95 | data.etag = res.ETag; 96 | 97 | resolve(data); 98 | }) 99 | }); 100 | 101 | } 102 | } 103 | 104 | function urlToS3Dir(url) { 105 | const urlInfo = URL.parse(url); 106 | return `${urlInfo.hostname}${encodeSpecialCharacters(urlInfo.pathname)}${urlInfo.pathname[urlInfo.pathname.length - 1] === '/' ? '' : `/`}` 107 | } 108 | 109 | function urlToS3Key(url) { 110 | // it's important that the key can be lexical (for byte order) and shrinking to permit returning newest results first 111 | // S3 doesn't permit returning descending order w/o a query layer 112 | const decrementingKey = encodeSpecialCharacters(Buffer.from(`${9999999999999 - Date.now()}`).toString('base64')); // ugly hack since we cannot use a human readable timestamp 113 | const urlInfo = URL.parse(url); 114 | return `${urlInfo.hostname}${encodeSpecialCharacters(urlInfo.pathname)}${urlInfo.pathname[urlInfo.pathname.length - 1] === '/' ? decrementingKey : `/${decrementingKey}`}.json` 115 | } 116 | 117 | /* PULLED FROM knox 118 | https://github.com/Automattic/knox/blob/master/lib/client.js#L64-L70 119 | */ 120 | function encodeSpecialCharacters(filename) { 121 | // Note: these characters are valid in URIs, but S3 does not like them for 122 | // some reason. 123 | return encodeURI(filename).replace(/[!'()#*+? ]/g, function (char) { 124 | return '%' + char.charCodeAt(0).toString(16); 125 | }); 126 | } 127 | -------------------------------------------------------------------------------- /src/express/static/app.js: -------------------------------------------------------------------------------- 1 | const lighthouseOptions = document.getElementById('lighthouseOptions'); 2 | const websiteInfo = document.getElementById('websiteInfo'); 3 | const queryByValue = document.getElementById('queryByValue'); 4 | const submitWebsite = document.getElementById('submitWebsite'); 5 | const getWebsite = document.getElementById('getWebsite'); 6 | const cancelGetWebsite = document.getElementById('cancelGetWebsite'); 7 | const queryTop = document.getElementById('queryTop'); 8 | const websiteSVG = document.getElementById('websiteSVG'); 9 | const websiteIFrame = document.getElementById('websiteIFrame'); 10 | const svgURL = document.getElementById('svgURL'); 11 | const jsonURL = document.getElementById('jsonURL'); 12 | const reportURL = document.getElementById('reportURL'); 13 | const reportJSONURL = document.getElementById('reportJSONURL'); 14 | const useBatch = document.getElementById('useBatch'); 15 | const batchFile = document.getElementById('batchFile'); 16 | 17 | let batchFileText; 18 | const batchFileReader = new FileReader(); 19 | batchFileReader.onload = e => { 20 | batchFileText = e.target.result; 21 | }; 22 | batchFile.addEventListener('change', evt => { 23 | batchFileReader.readAsText(evt.target.files[0]); 24 | }, false); 25 | 26 | function onSubmitWebsite() { 27 | const lhOptions = JSON.parse(lighthouseOptions.value); 28 | if (useBatch.checked) { 29 | if (!batchFileText) { 30 | return void alert('No batch file provided, try again'); 31 | } 32 | 33 | lhOptions.batch = batchFileText; 34 | } 35 | const lhOptionsJSON = JSON.stringify(lhOptions); 36 | submitWebsite.disabled = getWebsite.disabled = true; 37 | fetch('api/website', { 38 | method: 'POST', 39 | headers: { 40 | 'Content-Type': 'application/json; charset=utf-8' 41 | }, 42 | body: lhOptionsJSON 43 | }) 44 | .then(res => res.status !== 200 ? new Error(`Server returned ${res.status}`) : res.json()) 45 | .then(body => { 46 | if (body instanceof Error) throw body; 47 | 48 | submitWebsite.disabled = getWebsite.disabled = null; 49 | 50 | const arr = prettifyWebsites(Array.isArray(body) ? body : [body]); 51 | queryByValue.value = arr[0].id; 52 | websiteInfo.innerText = JSON.stringify(arr, null, 2); 53 | 54 | updateCards(arr); 55 | window.scrollTo(0, document.body.scrollHeight); 56 | }) 57 | .catch(err => { 58 | console.error('Oops!', err.stack || err); 59 | submitWebsite.disabled = getWebsite.disabled = null; 60 | }) 61 | ; 62 | } 63 | 64 | const QUERY_TYPE_LABELS = { 65 | requestedUrl: 'Requested URL', 66 | domainName: 'Domain Name', 67 | rootDomain: 'Root Domain Name', 68 | group: 'Group Name', 69 | documentId: 'Document ID' 70 | }; 71 | 72 | function getQueryURL(query) { 73 | return `api/website?top=${queryTop.value}&q=${encodeURIComponent(query || queryByValue.value)}`; 74 | } 75 | 76 | function updateCards(arr) { 77 | if (!Array.isArray(arr) || !arr.length || !arr[0]) { 78 | websiteSVG.style.display = 'none'; 79 | websiteIFrame.style.display = 'none'; 80 | svgURL.style.display = 'none'; 81 | jsonURL.style.display = 'none'; 82 | reportURL.style.display = 'none'; 83 | reportJSONURL.style.display = 'none'; 84 | return; 85 | } 86 | 87 | const first = arr[0]; 88 | 89 | const queryURL = getQueryURL(first.id); 90 | 91 | svgURL.href = `${queryURL}&format=svg&scale=1`; 92 | svgURL.style.display = ''; 93 | jsonURL.href = `${queryURL}&format=json`; 94 | jsonURL.style.display = ''; 95 | 96 | if (first.hasReport) { 97 | websiteSVG.style.display = 'none'; 98 | reportURL.href = `${queryURL}&format=reportHtml`; 99 | reportURL.style.display = ''; 100 | websiteIFrame.src = `${reportURL.href}&cache=${Date.now()}`; // only use cache busting on IMG itself, not the link to copy 101 | websiteIFrame.style.display = ''; 102 | reportJSONURL.href = `${queryURL}&format=reportJson`; 103 | reportJSONURL.style.display = ''; 104 | } else { 105 | websiteSVG.src = `${svgURL.href}&cache=${Date.now()}`; // only use cache busting on IMG itself, not the link to copy 106 | websiteSVG.style.display = ''; 107 | reportURL.style.display = 'none'; 108 | reportJSONURL.style.display = 'none'; 109 | websiteIFrame.style.display = 'none'; 110 | } 111 | } 112 | 113 | function onGetWebsite() { 114 | submitWebsite.disabled = getWebsite.disabled = true; 115 | const queryURL = getQueryURL(); 116 | fetch(queryURL) 117 | .then(res => res.status !== 200 ? new Error(`Server returned ${res.status}`) : res.json()) 118 | .then(body => { 119 | if (body instanceof Error) throw body; 120 | 121 | submitWebsite.disabled = getWebsite.disabled = null; 122 | 123 | websiteInfo.innerText = JSON.stringify(prettifyWebsites(body), null, 2); 124 | 125 | updateCards(body); 126 | window.scrollTo(0, document.body.scrollHeight); 127 | }) 128 | .catch(err => { 129 | console.error('Oops!', err.stack || err); 130 | websiteInfo.innerText = `Oops! ${(err.stack && err.stack.message) || err}`; 131 | submitWebsite.disabled = getWebsite.disabled = null; 132 | window.scrollTo(0, document.body.scrollHeight); 133 | }) 134 | ; 135 | } 136 | 137 | function toggleUseBatch() { 138 | batchFile.style.display = useBatch.checked ? null : 'none'; 139 | } 140 | 141 | const query = document.location.search.substr(1).split('&') 142 | .reduce((ctx, k) => { 143 | const s = k.split('='); ctx[s[0]] = s[1]; return ctx; 144 | }, {}) 145 | ; 146 | updateCards(); 147 | if (query.query) { 148 | queryByValue.value = query.query; 149 | onGetWebsite(); 150 | } 151 | 152 | function prettifyWebsites(o) { 153 | return o.map(r => Object.keys(r).reduce((state, k) => { 154 | const v = r[k]; 155 | const t = typeof v; 156 | if (k === 'meta' || t === 'string' || t === 'number' || t === 'boolean') { 157 | state[k] = v; 158 | } 159 | return state; 160 | }, {})); 161 | } 162 | -------------------------------------------------------------------------------- /src/express/api/post-website.js: -------------------------------------------------------------------------------- 1 | const { getDomain, getSubdomain } = require('tldjs'); 2 | const encrypt = require('../../util/encrypt'); 3 | const getWebsite = require('../util/get-website'); 4 | const processQueue = require('../../queue/process'); 5 | const { processMessage } = processQueue; 6 | const crypto = require('crypto'); 7 | const papa = require('papaparse'); 8 | 9 | module.exports = async (req, res) => { 10 | const { report, queue: canQueue = true, batch, wait, url, headers, secureHeaders, commands, cookies, auditMode, samples, attempts, hostOverride, delay: delayStr, group = 'unknown', meta } = req.body; 11 | 12 | let documentRequests; 13 | 14 | if (!url && !batch) return void res.status(400).send('`url` OR `batch` required'); 15 | 16 | if (batch) { 17 | if (typeof batch === 'object') { 18 | documentRequests = batch; 19 | } else if (batch[0] === '[') { 20 | // appears to be JSON 21 | documentRequests = JSON.parse(batch); 22 | } else { 23 | // assume CSV 24 | documentRequests = papa.parse(batch).data; 25 | } 26 | 27 | // use 2nd row to detect cols in case of header row 28 | const { urlCol, groupCol } = (documentRequests.length > 1 ? documentRequests[1] : documentRequests[0]).reduce((state, val, idx) => { 29 | if (state.urlCol < 0) { 30 | if (/\./.test(val) === true) { 31 | // dumb detector -- we're expecting domain or url left/most cols 32 | state.urlCol = idx; 33 | } 34 | } else if (state.groupCol < 0) { 35 | if (typeof val === 'string') { 36 | // for now we assume col following url is group, if one is provided 37 | state.groupCol = idx; 38 | } 39 | } 40 | 41 | return state; 42 | }, { 43 | urlCol: -1, 44 | groupCol: -1 45 | }); 46 | 47 | documentRequests = documentRequests.map(row => { 48 | const v = row[urlCol]; 49 | const isUrl = /\./.test(v); 50 | const url = isUrl && (/^https?:\/\//.test(v) ? v : `http://${v}`); 51 | return { 52 | url, 53 | group: `${group}${row[groupCol] || ''}` 54 | }; 55 | }).filter(row => typeof row.url === 'string'); 56 | } 57 | 58 | if (!documentRequests) { 59 | documentRequests = [{ 60 | url, 61 | group 62 | }]; 63 | } 64 | 65 | const config = req.app.get('config'); 66 | const store = req.app.get('store'); 67 | const queue = req.app.get('queue'); 68 | const { lighthouse } = config; 69 | const { secretKey } = config.queue; 70 | 71 | let secureHeadersEncrypted, cipherVector; 72 | if (secureHeaders) { 73 | if (!secretKey) { 74 | return void res.status(400).send('`secureHeaders` feature not enabled'); 75 | } 76 | 77 | cipherVector = Buffer.from(crypto.randomBytes(8)).toString('hex'); 78 | secureHeadersEncrypted = encrypt(JSON.stringify(secureHeaders), secretKey, cipherVector); 79 | } 80 | let commandsEncrypted; 81 | if (commands) { 82 | if (!secretKey) { 83 | return void res.status(400).send('`commands` feature not enabled'); 84 | } 85 | 86 | cipherVector = cipherVector || Buffer.from(crypto.randomBytes(8)).toString('hex'); 87 | commandsEncrypted = encrypt(JSON.stringify(commands), secretKey, cipherVector); 88 | } 89 | let cookiesEncrypted; 90 | if (cookies) { 91 | if (!secretKey) { 92 | return void res.status(400).send('`cookies` feature not enabled'); 93 | } 94 | 95 | cipherVector = cipherVector || Buffer.from(crypto.randomBytes(8)).toString('hex'); 96 | cookiesEncrypted = encrypt(JSON.stringify(cookies), secretKey, cipherVector); 97 | } 98 | 99 | const delay = Math.min( 100 | Math.max(parseInt(delayStr, 10) || lighthouse.delay.default, lighthouse.delay.range[0]), 101 | lighthouse.delay.range[1] 102 | ); 103 | const delayTime = delay && (Date.now() + delay); // time from now 104 | 105 | const groups = (typeof req.groupsAllowed === 'string') ? [req.groupsAllowed] : req.groupsAllowed; 106 | 107 | try { 108 | const documents = documentRequests.map(({ url, group }) => { 109 | 110 | if (req.groupsAllowed && (typeof req.groupsAllowed !== 'string' || req.groupsAllowed !== '*')) { 111 | // verify the requested group is allowed 112 | const isAllowed = groups.reduce((isAllowed, allowedGroup) => { 113 | return isAllowed || group === allowedGroup; 114 | }, false); 115 | if (!isAllowed) { 116 | const err = new Error('Group not authorized'); 117 | err.status = 401; 118 | throw err; 119 | } 120 | } 121 | 122 | const rootDomain = getDomain(url); 123 | const subDomain = getSubdomain(url); 124 | const domainName = subDomain ? `${subDomain}.${rootDomain}` : rootDomain; 125 | 126 | return { 127 | requestedUrl: url, 128 | domainName, 129 | rootDomain, 130 | headers, 131 | secureHeaders: secureHeadersEncrypted, 132 | commands: commandsEncrypted, 133 | cookies: cookiesEncrypted, 134 | cipherVector, 135 | report, 136 | group, 137 | auditMode, 138 | samples, 139 | attempts, 140 | hostOverride, 141 | delay, 142 | delayTime, 143 | state: 'requested', 144 | createDate: Date.now(), 145 | meta 146 | }; 147 | 148 | }); 149 | 150 | const documentPromises = documents.map(async doc => { 151 | const { id } = await store.write(doc); 152 | 153 | doc.id = id; 154 | 155 | if (canQueue) { // do not queue until the document has been indexed 156 | await queue.enqueue(doc); 157 | } else { 158 | // process inline 159 | doc = await processMessage({ config, store }, null, doc); 160 | } 161 | 162 | return doc; 163 | }); 164 | 165 | let storedDocs = await Promise.all(documentPromises); 166 | 167 | if (wait) { 168 | const processedDocs = await waitForProcessedDocs(req.app, storedDocs, wait); 169 | if (processedDocs) { 170 | storedDocs = processedDocs; 171 | } else { 172 | // if timeout, respond with partial to signal delta 173 | res.status(206); // partial 174 | } 175 | } 176 | 177 | res.send(batch ? storedDocs : storedDocs[0]); // forward document(s) back to caller 178 | } catch (ex) { 179 | console.error('indexWebsite.error:', ex.stack || ex); 180 | res.sendStatus(ex.status || 500); 181 | } 182 | 183 | }; 184 | 185 | async function waitForProcessedDocs(app, documents, timeout) { 186 | const start = Date.now(); 187 | 188 | do { 189 | documents = await Promise.all(documents.map(doc => { 190 | if (doc.state === 'processed' || doc.state === 'error') return doc; // nothing more to do 191 | 192 | return getWebsite(app, { q: doc.id }).then(({ audits, i18n, _report, categoryGroups, ...others }) => ({ ...others, hasReport: !!_report })); 193 | })); 194 | 195 | if (!documents.find(doc => (doc.state !== 'processed' && doc.state !== 'error'))) { 196 | // return final completed states if available 197 | return documents; 198 | } 199 | 200 | // sleep... pulling sucks, but more ideal for server to handle than client 201 | await new Promise(resolve => setTimeout(resolve, 1000)); 202 | } while ((Date.now() - start) < timeout); 203 | } 204 | -------------------------------------------------------------------------------- /src/lighthouse/submit.js: -------------------------------------------------------------------------------- 1 | const { parse } = require('url'); 2 | const lighthouse = require('lighthouse'); 3 | const ChromeProtocol = require('lighthouse/lighthouse-core/gather/connections/cri.js'); 4 | const chromeLauncher = require('chrome-launcher'); 5 | const throttling = require('./throttling'); 6 | const decrypt = require('../util/decrypt'); 7 | const URL = require('url'); 8 | const path = require('path'); 9 | const _ = require('lodash'); 10 | const fetch = require('cross-fetch'); 11 | 12 | const OPTIONS_TO_CONFIG = { 13 | throttling: 'settings.throttling', 14 | headers: 'settings.extraHeaders' 15 | }; 16 | 17 | // FORMAT: https://github.com/GoogleChrome/lighthouse/blob/master/docs/understanding-results.md#properties 18 | 19 | const ALLOWED_KEYS = { 20 | userAgent: true, 21 | environment: true, 22 | lighthouseVersion: true, 23 | fetchTime: true, 24 | requestedUrl: true, 25 | finalUrl: true, 26 | runWarnings: false, 27 | audits: true, 28 | configSettings: true, 29 | categories: true, 30 | categoryGroups: true, 31 | timing: true, 32 | i18n: false 33 | }; 34 | 35 | module.exports = async (url, { lighthouse: baseConfig, queue, launcher }, options) => { 36 | const config = Object.assign({}, baseConfig.config); 37 | const { hostOverride, group, secureHeaders, cipherVector, commands, cookies, meta } = options; 38 | 39 | // only pull over whitelisted options 40 | Object.keys(options).forEach(optionKey => { 41 | const mapToKey = OPTIONS_TO_CONFIG[optionKey]; 42 | if (!mapToKey) return; // ignore unless whitelisted 43 | // copy whitelisted option to the new location 44 | _.set(config, mapToKey, options[optionKey]); 45 | }); 46 | 47 | let decryptedHeaders; 48 | if (secureHeaders) { 49 | const { secretKey } = queue; 50 | if (!secretKey) throw new Error('`secureHeaders` is not allowed without `queue.secretKey` being set'); 51 | 52 | decryptedHeaders = JSON.parse(decrypt(secureHeaders, secretKey, cipherVector)); 53 | config.settings.extraHeaders = Object.assign(config.settings.extraHeaders, decryptedHeaders); 54 | } 55 | let decryptedCommands; 56 | if (commands) { 57 | const { secretKey } = queue; 58 | if (!secretKey) throw new Error('`commands` is not allowed without `queue.secretKey` being set'); 59 | 60 | decryptedCommands = JSON.parse(decrypt(commands, secretKey, cipherVector)); 61 | } 62 | let decryptedCookies; 63 | if (cookies) { 64 | const { secretKey } = queue; 65 | if (!secretKey) throw new Error('`cookies` is not allowed without `queue.secretKey` being set'); 66 | 67 | decryptedCookies = JSON.parse(decrypt(cookies, secretKey, cipherVector)); 68 | } 69 | 70 | // throw if non-200 71 | // unfortunately not everyone supports HEAD, so we must perform a full GET... 72 | const res = await fetch(url, { headers: config.settings.extraHeaders || {} }); 73 | if (res.status >= 400) throw new Error(`requestedUrl '${url}' returned status ${res.status}`); 74 | 75 | const validateHandlerPath = baseConfig.validate && baseConfig.validate[group]; 76 | const validateHandler = validateHandlerPath && require(path.resolve(validateHandlerPath)); 77 | validateHandler && validateHandler({ res }); 78 | 79 | let throttlingPreset; 80 | 81 | if (typeof config.settings.throttling === 'string') { 82 | // resolve throttling 83 | throttlingPreset = config.settings.throttling; 84 | config.settings.throttling = throttling[config.settings.throttling]; 85 | } 86 | 87 | const auditMode = options.auditMode || baseConfig.auditMode; 88 | const report = typeof options.report === 'boolean' ? options.report : baseConfig.report; 89 | const samples = Math.min(Math.max(options.samples || baseConfig.samples.default, baseConfig.samples.range[0]), baseConfig.samples.range[1]); 90 | 91 | const results = []; 92 | for (let sample = 0; sample < samples; sample++) { 93 | results[sample] = await getLighthouseResult(url, config, { report, launcher, auditMode, throttlingPreset, hostOverride, commands: decryptedCommands, cookies: decryptedCookies, meta }); 94 | } 95 | 96 | // take the top result 97 | const result = results.sort((a, b) => b.categories.performance.score - a.categories.performance.score)[0]; 98 | 99 | const headers = result.configSettings && result.configSettings.extraHeaders; 100 | if (decryptedHeaders && headers) { 101 | // scrub secure headers 102 | Object.keys(headers).forEach(key => { 103 | if (key in decryptedHeaders) { 104 | // if secure header, scrub it 105 | headers[key] = '__private__'; 106 | } 107 | }); 108 | } 109 | 110 | // todo: permit configurable categories.. for example score = sum of each category 111 | return result; 112 | }; 113 | 114 | async function executeCommand(connection, instances, { id, command, options, waitFor, waitTimeout }) { 115 | if (!command) throw new Error('lighthouse.connection.sendCommand requires `command`'); 116 | id = id || command; 117 | const waiter = waitFor ? createCommandWaiter(connection, { waitFor, waitTimeout }) : null; 118 | const result = await connection.sendCommand(command, options); 119 | if (waiter) { 120 | await waiter; 121 | } 122 | if (result.result && result.result.subtype === 'error') throw new Error(result.result.description); 123 | instances[id] = result; 124 | return result; 125 | } 126 | 127 | function createCommandWaiter(connection, { waitFor, waitTimeout = 30000 }) { 128 | return new Promise((resolve, reject) => { 129 | let timer; 130 | 131 | const listener = event => { 132 | if (event.method === waitFor) { 133 | clearTimeout(timer); 134 | connection._eventEmitter.removeAllListeners(); 135 | resolve(); 136 | } 137 | }; 138 | 139 | timer = setTimeout(() => { 140 | connection._eventEmitter.removeAllListeners(); 141 | reject(new Error(`Timed out waiting for ${waitFor} after ${waitTimeout}ms`)); 142 | }, waitTimeout); 143 | 144 | connection.on('protocolevent', listener); 145 | }); 146 | } 147 | 148 | function getCommandsFromCookies(cookies, { url }) { 149 | const commands = [{ 150 | command: 'Page.enable' // required for event tracking 151 | }]; 152 | 153 | const urlInfo = parse(url); 154 | 155 | let currentUrl; 156 | cookies.forEach(cookie => { 157 | const key = Object.keys(cookie)[0]; 158 | const v = cookie[key]; 159 | const value = typeof v === 'object' ? v : { 160 | value: v, 161 | secure: /^https/.test(url) 162 | }; 163 | value.url = value.url || url; // use request url unless an explicit url is provided 164 | value.domain = value.domain || urlInfo.hostname.split('.').splice(-2, 2).join('.'); 165 | 166 | if (value.url !== currentUrl) { 167 | commands.push({ 168 | command: 'Page.navigate', 169 | waitFor: 'Page.loadEventFired', 170 | options: { 171 | url: value.url 172 | } 173 | }); 174 | 175 | currentUrl = value.url; 176 | } 177 | 178 | const expressionDomain = value.domain ? `; domain=${value.domain}` : ''; 179 | const expressionPath = value.path ? `; path=${value.path}` : ''; 180 | const expressionSecure = value.secure ? '; secure' : ''; 181 | const expressionHttpOnly = value.httpOnly ? '; httpOnly' : ''; 182 | const expression = `document.cookie="${key}=${value.value}${expressionDomain}${expressionPath}${expressionSecure}${expressionHttpOnly}"`; 183 | 184 | commands.push({ 185 | command: 'Runtime.evaluate', 186 | options: { 187 | expression 188 | } 189 | }); 190 | }); 191 | 192 | return commands; 193 | } 194 | 195 | async function getLighthouseResult(url, config, { launcher, auditMode, throttlingPreset, hostOverride, report, commands = [], cookies = [], meta }) { 196 | const chromeOptions = { chromeFlags: config.chromeFlags }; 197 | if (hostOverride) { 198 | const { host } = URL.parse(url); 199 | // bug in chrome preventing headless support of `host-rules`: https://bugs.chromium.org/p/chromium/issues/detail?id=798793 200 | chromeOptions.chromeFlags = chromeOptions.chromeFlags.concat([ 201 | `--host-rules=MAP ${host} ${hostOverride}`, '--ignore-certificate-errors' 202 | ]); 203 | } 204 | 205 | const chrome = await (launcher || chromeLauncher.launch)(chromeOptions); 206 | chromeOptions.port = chrome.port; 207 | return new Promise(async (resolve, reject) => { 208 | // a workaround for what appears to a bug with lighthouse that prevents graceful failure 209 | const timer = setTimeout(() => { 210 | chrome.kill(); // force cleanup 211 | reject(new Error('timeout!')); 212 | }, 60000).unref(); 213 | 214 | try { 215 | const connection = new ChromeProtocol(chromeOptions.port, chromeOptions.hostname); 216 | await connection.connect(); 217 | 218 | // process cookies 219 | let cookieCommands; 220 | if (cookies.length > 0) { 221 | cookieCommands = getCommandsFromCookies(cookies, { url }); 222 | commands = cookieCommands.concat(commands); 223 | } 224 | 225 | // process commands 226 | const instances = {}; 227 | for (var cmdI = 0; cmdI < commands.length; cmdI++) { 228 | await executeCommand(connection, instances, commands[cmdI]); 229 | } 230 | 231 | const results = await lighthouse(url, chromeOptions, config, connection); 232 | clearTimeout(timer); 233 | const { lhr } = results; 234 | 235 | // track preset if any 236 | lhr.configSettings.throttlingPreset = throttlingPreset; 237 | 238 | // allowed keys 239 | Object.keys(lhr).forEach(resultKey => { 240 | if (!ALLOWED_KEYS[resultKey]) lhr[resultKey] = null; 241 | }); 242 | 243 | if (report) { 244 | // store FULL report (200-300KB) 245 | lhr._report = results.report; 246 | } 247 | 248 | const { categories } = lhr; 249 | Object.keys(categories).forEach(catKey => { 250 | const cat = categories[catKey]; 251 | // no need for `auditRefs` 252 | delete cat.auditRefs; 253 | }); 254 | 255 | // process `audits` 256 | 257 | let auditV; 258 | if (!auditMode) { 259 | lhr.audits = null; 260 | } else { 261 | Object.keys(lhr.audits || {}).forEach(k => { 262 | auditV = lhr.audits[k]; 263 | delete auditV.displayValue; 264 | if (auditMode === 'simple' || auditMode === 'details') { 265 | delete auditV.id; 266 | delete auditV.title; 267 | delete auditV.description; 268 | } 269 | if (auditMode === 'simple') { 270 | delete auditV.details; 271 | } 272 | }); 273 | } 274 | 275 | if (lhr.timing) { 276 | // not yet supported 277 | delete lhr.timing.entries; 278 | } 279 | 280 | if (meta) { 281 | lhr.meta = meta; 282 | } 283 | 284 | // use results.lhr for the JS-consumeable output 285 | // https://github.com/GoogleChrome/lighthouse/blob/master/typings/lhr.d.ts 286 | // use results.report for the HTML/JSON/CSV output as a string 287 | // use results.artifacts for the trace/screenshots/other specific case you need (rarer) 288 | await chrome.kill(); 289 | resolve(lhr); 290 | } catch (ex) { 291 | clearTimeout(timer); 292 | await chrome.kill(); 293 | reject(ex); 294 | } 295 | }); 296 | } 297 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse4u 2 | 3 | [![travis-ci](https://travis-ci.org/godaddy/lighthouse4u.svg?branch=master)](https://travis-ci.org/godaddy/lighthouse4u) 4 | 5 | LH4U provides [Google Lighthouse](https://github.com/GoogleChrome/lighthouse) as a service, surfaced by both a friendly UI+API, and backed by various 6 | [storage](#storage-clients) and [queue](#queue-clients) clients to support a wide range of architectures. 7 | 8 | 9 | ![UI](./docs/example.gif) 10 | 11 | ## Usage 12 | 13 | Start server via CLI: 14 | 15 | ``` 16 | npm i -g lighthouse4u 17 | lh4u server --config local --config-dir ./app/config --config-base defaults --secure-config ./app/config/secure --secure-file some-private.key 18 | ``` 19 | 20 | Or locally to this repo via `npm start` (you'll need to create `test/config/local.json5` by copying `test/config/COPY.json5`). 21 | 22 | 23 | ## Architectures 24 | 25 | With the release of v1, we've decoupled compute and storage into multiple tiers to permit a diverse set of 26 | architectures. Not only are custom [storage](#storage-clients) and [queue](#queue-clients) clients 27 | supported, but compute can even be run serverless. 28 | 29 | ### Queue Architecture 30 | 31 | This most closely resembles that of `v0.7`, but without the restrictions of being locked into Elasticsearch and RabbitMQ. 32 | 33 | ``` 34 | { 35 | reader: { 36 | module: 'lighthouse4u-elasticsearch', options: { /* ... */ } 37 | }, 38 | writer: { 39 | module: 'lighthouse4u-elasticsearch', options: { /* ... */ } 40 | }, 41 | queue: { // rabbit 42 | module: 'lighthouse4u-amqp', options: { /* ... */ } 43 | } 44 | } 45 | ``` 46 | 47 | **Note:** The `lighthouse4u-elasticsearch` storage client is not yet available. Happy to take contributions! 48 | 49 | 50 | ### Serverless Architecture 51 | 52 | Same as above, you're not tied to Elasticsearch. Use whatever 53 | [storage](#storage-clients) and [queue](#queue-clients) client fits your needs. Due to the nature 54 | of AWS Lambda, your Lighthouse "runner" (triggered via SQS) is separate from the Lighthouse4u UI+API. 55 | 56 | Serverless example via [AWS Lambda](https://github.com/godaddy/lighthouse4u-lambda) 57 | 58 | 59 | 60 | ## Configuration Options 61 | 62 | | Option | Type | Default | Desc | 63 | | --- | --- | --- | --- | 64 | | store|reader|writer | | | All options connected with storage. Reads will goto `reader` if provided, otherwise default to `store`. Writes will goto `writer` if provided, otherwise default to `store`. | 65 | | ->.module | `string` | **required** | Path of client module to import. See [storage clients](#storage-clients) | 66 | | ->.options | `object` | | Collection of options supplied to storage client | 67 | | queue | | | Queue configuration | 68 | | ->.module | `string` | **required** | Path of client module to import. See [queue clients](#queue-clients) | 69 | | ->.idleDelayMs | `number` | `1000` | Time (in MS) between queue checks when queue is empty | 70 | | ->.enabled | `boolean` | `true` | Consuming queue may be disabled | 71 | | ->.options | `object` | | Collection of options supplied to queue client | 72 | | http | | | HTTP(S) Server options | 73 | | ->.bindings | `Hash` | | An object of named bindings | 74 | | ->.bindings.{name} | `HttpBinding` | | With `{name}` being the name of the binding -- can be anything, ex: `myMagicBinding`. See value to `false` if you want to disable this binding. | 75 | | ->.bindings.{name}.port | `number` | `8994` | Port to bind to | 76 | | ->.bindings.{name}.ssl | [TLS Options](https://nodejs.org/dist/latest-v10.x/docs/api/tls.html#tls_new_tls_tlssocket_socket_options) | `undefined` | Required to bind HTTPS/2 | 77 | | ->.auth | | | Zero or more authorization options | 78 | | ->.auth.{authName} | `HttpAuth` | | A set of auth options keyed by any custom `{authName}` | 79 | | ->.auth.{authName}.type | `basic|custom` | **required** | Type of auth, be it built-in Basic auth, or custom provided by `customPath` module | 80 | | ->.auth.{authName}.groups | `string|array` | `*` | **required** | | 81 | | ->.auth.{authName}.customPath | `string` | | Path of the custom module | 82 | | ->.auth.{authName}.options | | | Options provided to the auth module | 83 | | ->.authRedirect | | `undefined` | URL to redirect UI to if auth is enabled and fails | 84 | | ->.routes | | | Optional set of custom routes | 85 | | ->.routes.{name} | `string|HttpRoute` | | If value of type string that'll be used as path | 86 | | ->.routes.{name}.path | `string` | | Path to resolve to connect middleware, relative to current working directory | 87 | | ->.routes.{name}.method | `string` | `GET` | Method to bind to | 88 | | ->.routes.{name}.route | `string` | Uses `{name}` if undefined | Route to map, ala `/api/test` | 89 | | ->.staticFiles | | | Optional to bind routes to static assets | 90 | | ->.staticFiles.{route} | [Express Static](https://expressjs.com/en/starter/static-files.html) | | Options to supply to static middleware | 91 | | lighthouse | | | Options related to Lighthouse usage | 92 | | ->.config | [LighthouseOptions](https://github.com/GoogleChrome/lighthouse/blob/master/lighthouse-core/config/default-config.js) | | Google Lighthouse configuration options | 93 | | ->.config.extends | `string` | `lighthouse:default` | What options to default to | 94 | | ->.config.logLevel | `string` | `warn` | Level of logging | 95 | | ->.config.chromeFlags | `array` | `[ '--headless', '--disable-gpu', '--no-sandbox' ]` | Array of CLI arguments | 96 | | ->.config.settings | [LighthouseSettings](https://github.com/GoogleChrome/lighthouse/blob/master/lighthouse-core/config/constants.js#L30) | [See Defaults](./src/config/default-config.js#L90) | Settings applied to Lighthouse | 97 | | ->.validate | `hash` | | Zero or more validators | 98 | | ->.validate.{groupName} | `string` | | Path of validation module used to determine if the response is coming from the intended server. Useful in cases where you only want to measure results coming from an intended infrastructure | 99 | | ->.concurrency | `number` | `1` | Number of concurrently processed tasks permitted | 100 | | ->.report | `bool` | `true` | Save the full report, viewable in LH viewer | 101 | | ->.samples | | | After N lighthouse samples are taken, only the best result is recorded | 102 | | ->.samples.default | `number` | `3` | Default number of lighthouse tests performed | 103 | | ->.samples.range | `tuple` | `[1, 5]` | Minimum and maximum samples taken before returning result | 104 | | ->.attempts | | | Number of attempts at running Google Lighthouse before giving up due to failures | 105 | | ->.attempts.default | `number` | `2` | Default number of attempts before giving up | 106 | | ->.attempts.range | `tuple` | `[1, 10]` | Minimum and maximum attempts before giving up | 107 | | ->.attempts.delayMsPerExponent | `number` | `1000` | Exponential backoff after failure | 108 | | ->.delay | | | Time (in ms) before a test is executed | 109 | | ->.delay.default | `number` | `0` | Default time to wait before test can be run | 110 | | ->.delay.range | `tuple` | `[0, 1000 * 60 * 60]` | Minimum and maximum time before test can be run | 111 | | ->.delay.delayMsPerExponent | `number` | `1000 * 30` | Maximum time before delayed messages will be requeued | 112 | 113 | 114 | ## API 115 | 116 | ### API - `GET /api/website` 117 | 118 | Fetch zero or more website results matching the criteria. 119 | 120 | #### Query String Options 121 | 122 | | Option | Type | Default | Desc | 123 | | --- | --- | --- | --- | 124 | | format | `string` | `json` | Format of results, be it `json`, `svg`, `reportHtml`, or `reportJson` | 125 | | scale | `number` | `1` | Scale of `svg` | 126 | | q | `string` | optional | Query by URL or document ID | 127 | | top | `number` | `1` | Maximum records to return. Only applicable for `json` format | 128 | 129 | 130 | ### API - `GET /api/website/compare` 131 | 132 | Similar to the single website GET, but instead compares results from `q1` with `q2`. 133 | 134 | #### Query String Options 135 | 136 | | Option | Type | Default | Desc | 137 | | --- | --- | --- | --- | 138 | | format | `string` | `json` | Format of results, be it `json` or `svg` | 139 | | scale | `number` | `1` | Scale of `svg` | 140 | | q1 | `string` | **required** | First query by URL or document ID | 141 | | q2 | `string` | **required** | Second query by URL or document ID | 142 | 143 | ### API - `POST /api/website` 144 | 145 | Submit to process website with Google Lighthouse. 146 | 147 | #### JSON Body Options 148 | 149 | | Option | Type | Default | Desc | 150 | | --- | --- | --- | --- | 151 | | url | `string` | **required** | URL to process via Google Lighthouse | 152 | | wait | `number` | optional | Block returning until job is complete, or the `wait` time (in ms) has elapsed, resulting in a partial 206 response | 153 | | headers | `hash` | optional | Collection of HTTP headers to supply in request | 154 | | secureHeaders | hash | optional | Same use as `headers`, but stored securely in queue, and never persisted to ElasticSearch | 155 | | samples | `number` | (See `options.lighthouse.samples`) | Number of samples to take before recording result | 156 | | attempts | `number` | (See `options.lighthouse.attempts`) | Number of failure attempts before giving up | 157 | | hostOverride | `string` | optional | Map host of request to an explicit IP. [Not yet supported by Chrome in Headless mode](https://bugs.chromium.org/p/chromium/issues/detail?id=798793) | 158 | | delay | `number` | (See `options.lighthouse.delay`) | Delay (in milliseconds) before test will be performed. Useful if the intended service or domain is not expected to be available for some time | 159 | | group | `string` | `unknown` | Group to categorize result to. Useful when searching/filtering on different groups of results | 160 | | report | `bool` | `true` | If the full report should be stored, allowing for viewing in LH Viewer | 161 | | auditMode | `false|'simple'|'details'|'all'` | `'simple'` | How much of the [audits](https://github.com/GoogleChrome/lighthouse/blob/master/docs/understanding-results.md#properties) data to persist to storage | 162 | | cookies | `array<{name=string|Cookie}>` | optional | Auto-translates cookies to set to the required `commands` | 163 | | cookies[idx].value | `string` | **required** | Value to set for cookie | 164 | | cookies[idx].domain | `string` | optional | Domain to apply cookie to. Defaults to root of `requestedUrl` domain | 165 | | cookies[idx].url | `string` | optional | If the page required to apply cookie to differs from the `requestedUrl` you can set that here | 166 | | cookies[idx].path | `string` | optional | Path to apply to cookie | 167 | | cookies[idx].secure | `boolean` | optional | Is cookie secured | 168 | | cookies[idx].httpOnly | `boolean` | optional | HTTP only | 169 | | commands | `array` | optional | [DevTool Commands](https://chromedevtools.github.io/devtools-protocol/) to be executed prior to running Lighthouse | 170 | | commands[idx].command | `string` | **required** | [DevTool Command](https://chromedevtools.github.io/devtools-protocol/) to execute | 171 | | commands[idx].options | `object` | **required** | [DevTool Command Options](https://chromedevtools.github.io/devtools-protocol/) to supply command | 172 | | meta | `object` | optional | Object containing metadata that will be attached to final lighthouse results | 173 | 174 | 175 | ### Command 176 | 177 | 178 | ## Storage Clients 179 | 180 | * [AWS S3](https://github.com/godaddy/lighthouse4u/tree/master/packages/lighthouse4u-s3) - Store and basic query 181 | support via S3 buckets. 182 | * [File System](https://github.com/godaddy/lighthouse4u/tree/master/packages/lighthouse4u-fs) - Not recommended 183 | for production usage. Good for local testing. 184 | * [Elasticsearch](https://github.com/godaddy/lighthouse4u/tree/master/packages/lighthouse4u-es) - Store and query 185 | support via Elasticsearch. 186 | 187 | 188 | ## Queue Clients 189 | 190 | * [AWS SQS](https://github.com/godaddy/lighthouse4u/tree/master/packages/lighthouse4u-sqs) - Support for 191 | Simple Queue Service. 192 | * [AMQP](https://github.com/godaddy/lighthouse4u/tree/master/packages/lighthouse4u-amqp) - Support for 193 | RabbitMQ and other AMQP compatible queues. 194 | 195 | 196 | ## Secure Configuration 197 | 198 | Optionally you may run LH4U with the `--secure-config {secureConfigPath}` and `--secure-file {securePrivateFile}` powered by [Config Shield](https://github.com/godaddy/node-config-shield). 199 | 200 | ``` 201 | npm i -g config-shield 202 | cshield ./app/config/secure some-private.key 203 | > help 204 | > set queue { options: { url: 'amqp://lh4u_user:someSuperSecretPassword@rmq.on-my-domain.com:5672/lh4u' } } 205 | > get queue 206 | : { "options": { "url": "amqp://lh4u_user:someSuperSecretPassword@rmq.on-my-domain.com:5672/lh4u" } } 207 | > save 208 | > exit 209 | ``` 210 | 211 | The example above will allow you to merge sensitive credentials with your existing public configuration. Only options defined in the secure config will be deeply merged. 212 | -------------------------------------------------------------------------------- /lh.json: -------------------------------------------------------------------------------- 1 | { 2 | "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/70.0.3538.110 Safari/537.36", 3 | "environment": { 4 | "networkUserAgent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/66.0.3359.30 Mobile Safari/537.36", 5 | "hostUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/70.0.3538.110 Safari/537.36", 6 | "benchmarkIndex": 1248 7 | }, 8 | "lighthouseVersion": "3.1.1", 9 | "fetchTime": "2018-12-01T15:29:33.347Z", 10 | "requestedUrl": "http://www.allibrinkerhoff.com/", 11 | "finalUrl": "https://allibrinkerhoff.com/", 12 | "runWarnings": null, 13 | "audits": { 14 | "is-on-https": { 15 | "score": 0, 16 | "scoreDisplayMode": "binary", 17 | "rawValue": false 18 | }, 19 | "redirects-http": { 20 | "score": 1, 21 | "scoreDisplayMode": "binary", 22 | "rawValue": true 23 | }, 24 | "service-worker": { 25 | "score": 0, 26 | "scoreDisplayMode": "binary", 27 | "rawValue": false 28 | }, 29 | "works-offline": { 30 | "score": 0, 31 | "scoreDisplayMode": "binary", 32 | "rawValue": false, 33 | "warnings": [ 34 | "You may be not loading offline because your test URL (http://www.allibrinkerhoff.com/) was redirected to \"https://allibrinkerhoff.com/\". Try testing the second URL directly." 35 | ] 36 | }, 37 | "viewport": { 38 | "score": 1, 39 | "scoreDisplayMode": "binary", 40 | "rawValue": true, 41 | "warnings": [] 42 | }, 43 | "without-javascript": { 44 | "score": 1, 45 | "scoreDisplayMode": "binary", 46 | "rawValue": true 47 | }, 48 | "first-contentful-paint": { 49 | "score": 0.55, 50 | "scoreDisplayMode": "numeric", 51 | "rawValue": 3814.5359999999996 52 | }, 53 | "first-meaningful-paint": { 54 | "score": 0.37, 55 | "scoreDisplayMode": "numeric", 56 | "rawValue": 4594.563999999999 57 | }, 58 | "load-fast-enough-for-pwa": { 59 | "score": 1, 60 | "scoreDisplayMode": "binary", 61 | "rawValue": 9193.85 62 | }, 63 | "speed-index": { 64 | "score": 0.55, 65 | "scoreDisplayMode": "numeric", 66 | "rawValue": 5489.316725309338 67 | }, 68 | "screenshot-thumbnails": { 69 | "score": null, 70 | "scoreDisplayMode": "informative", 71 | "rawValue": true 72 | }, 73 | "final-screenshot": { 74 | "score": null, 75 | "scoreDisplayMode": "informative", 76 | "rawValue": true 77 | }, 78 | "estimated-input-latency": { 79 | "score": 1, 80 | "scoreDisplayMode": "numeric", 81 | "rawValue": 32.4 82 | }, 83 | "errors-in-console": { 84 | "score": 1, 85 | "scoreDisplayMode": "binary", 86 | "rawValue": 0 87 | }, 88 | "time-to-first-byte": { 89 | "score": 1, 90 | "scoreDisplayMode": "binary", 91 | "rawValue": 25.631 92 | }, 93 | "first-cpu-idle": { 94 | "score": 0.33, 95 | "scoreDisplayMode": "numeric", 96 | "rawValue": 8014.955000000001 97 | }, 98 | "interactive": { 99 | "score": 0.33, 100 | "scoreDisplayMode": "numeric", 101 | "rawValue": 9193.85 102 | }, 103 | "user-timings": { 104 | "score": null, 105 | "scoreDisplayMode": "not-applicable", 106 | "rawValue": true 107 | }, 108 | "critical-request-chains": { 109 | "score": null, 110 | "scoreDisplayMode": "informative", 111 | "rawValue": false 112 | }, 113 | "redirects": { 114 | "score": 0.46, 115 | "scoreDisplayMode": "numeric", 116 | "rawValue": 1121.415 117 | }, 118 | "webapp-install-banner": { 119 | "score": 0, 120 | "scoreDisplayMode": "binary", 121 | "rawValue": false, 122 | "explanation": "Failures: No manifest was fetched,\nSite does not register a service worker." 123 | }, 124 | "splash-screen": { 125 | "score": 0, 126 | "scoreDisplayMode": "binary", 127 | "rawValue": false, 128 | "explanation": "Failures: No manifest was fetched." 129 | }, 130 | "themed-omnibox": { 131 | "score": 0, 132 | "scoreDisplayMode": "binary", 133 | "rawValue": false, 134 | "explanation": "Failures: No manifest was fetched." 135 | }, 136 | "manifest-short-name-length": { 137 | "score": null, 138 | "scoreDisplayMode": "not-applicable", 139 | "rawValue": true 140 | }, 141 | "content-width": { 142 | "score": 1, 143 | "scoreDisplayMode": "binary", 144 | "rawValue": true, 145 | "explanation": "" 146 | }, 147 | "image-aspect-ratio": { 148 | "score": 1, 149 | "scoreDisplayMode": "binary", 150 | "rawValue": true, 151 | "warnings": [] 152 | }, 153 | "deprecations": { 154 | "score": 1, 155 | "scoreDisplayMode": "binary", 156 | "rawValue": true 157 | }, 158 | "mainthread-work-breakdown": { 159 | "score": 0.12, 160 | "scoreDisplayMode": "numeric", 161 | "rawValue": 7534.343999999999 162 | }, 163 | "bootup-time": { 164 | "score": 0.85, 165 | "scoreDisplayMode": "numeric", 166 | "rawValue": 1569.124 167 | }, 168 | "uses-rel-preload": { 169 | "score": 1, 170 | "scoreDisplayMode": "numeric", 171 | "rawValue": 0 172 | }, 173 | "uses-rel-preconnect": { 174 | "score": 0.71, 175 | "scoreDisplayMode": "numeric", 176 | "rawValue": 369.93399999999997 177 | }, 178 | "font-display": { 179 | "score": 0, 180 | "scoreDisplayMode": "binary", 181 | "rawValue": false 182 | }, 183 | "network-requests": { 184 | "score": null, 185 | "scoreDisplayMode": "informative", 186 | "rawValue": 82 187 | }, 188 | "metrics": { 189 | "score": null, 190 | "scoreDisplayMode": "informative", 191 | "rawValue": 9193.85 192 | }, 193 | "pwa-cross-browser": { 194 | "score": null, 195 | "scoreDisplayMode": "manual", 196 | "rawValue": false 197 | }, 198 | "pwa-page-transitions": { 199 | "score": null, 200 | "scoreDisplayMode": "manual", 201 | "rawValue": false 202 | }, 203 | "pwa-each-page-has-url": { 204 | "score": null, 205 | "scoreDisplayMode": "manual", 206 | "rawValue": false 207 | }, 208 | "accesskeys": { 209 | "score": null, 210 | "scoreDisplayMode": "not-applicable", 211 | "rawValue": true 212 | }, 213 | "aria-allowed-attr": { 214 | "score": 1, 215 | "scoreDisplayMode": "binary", 216 | "rawValue": true 217 | }, 218 | "aria-required-attr": { 219 | "score": 1, 220 | "scoreDisplayMode": "binary", 221 | "rawValue": true 222 | }, 223 | "aria-required-children": { 224 | "score": 1, 225 | "scoreDisplayMode": "binary", 226 | "rawValue": true 227 | }, 228 | "aria-required-parent": { 229 | "score": 1, 230 | "scoreDisplayMode": "binary", 231 | "rawValue": true 232 | }, 233 | "aria-roles": { 234 | "score": 1, 235 | "scoreDisplayMode": "binary", 236 | "rawValue": true 237 | }, 238 | "aria-valid-attr-value": { 239 | "score": 1, 240 | "scoreDisplayMode": "binary", 241 | "rawValue": true 242 | }, 243 | "aria-valid-attr": { 244 | "score": 1, 245 | "scoreDisplayMode": "binary", 246 | "rawValue": true 247 | }, 248 | "audio-caption": { 249 | "score": null, 250 | "scoreDisplayMode": "not-applicable", 251 | "rawValue": true 252 | }, 253 | "button-name": { 254 | "score": 1, 255 | "scoreDisplayMode": "binary", 256 | "rawValue": true 257 | }, 258 | "bypass": { 259 | "score": 1, 260 | "scoreDisplayMode": "binary", 261 | "rawValue": true 262 | }, 263 | "color-contrast": { 264 | "score": 0, 265 | "scoreDisplayMode": "binary", 266 | "rawValue": false 267 | }, 268 | "definition-list": { 269 | "score": null, 270 | "scoreDisplayMode": "not-applicable", 271 | "rawValue": true 272 | }, 273 | "dlitem": { 274 | "score": null, 275 | "scoreDisplayMode": "not-applicable", 276 | "rawValue": true 277 | }, 278 | "document-title": { 279 | "score": 1, 280 | "scoreDisplayMode": "binary", 281 | "rawValue": true 282 | }, 283 | "duplicate-id": { 284 | "score": 1, 285 | "scoreDisplayMode": "binary", 286 | "rawValue": true 287 | }, 288 | "frame-title": { 289 | "score": 0, 290 | "scoreDisplayMode": "binary", 291 | "rawValue": false 292 | }, 293 | "html-has-lang": { 294 | "score": 1, 295 | "scoreDisplayMode": "binary", 296 | "rawValue": true 297 | }, 298 | "html-lang-valid": { 299 | "score": 1, 300 | "scoreDisplayMode": "binary", 301 | "rawValue": true 302 | }, 303 | "image-alt": { 304 | "score": 1, 305 | "scoreDisplayMode": "binary", 306 | "rawValue": true 307 | }, 308 | "input-image-alt": { 309 | "score": null, 310 | "scoreDisplayMode": "not-applicable", 311 | "rawValue": true 312 | }, 313 | "label": { 314 | "score": 1, 315 | "scoreDisplayMode": "binary", 316 | "rawValue": true 317 | }, 318 | "layout-table": { 319 | "score": null, 320 | "scoreDisplayMode": "not-applicable", 321 | "rawValue": true 322 | }, 323 | "link-name": { 324 | "score": 0, 325 | "scoreDisplayMode": "binary", 326 | "rawValue": false 327 | }, 328 | "list": { 329 | "score": 1, 330 | "scoreDisplayMode": "binary", 331 | "rawValue": true 332 | }, 333 | "listitem": { 334 | "score": 1, 335 | "scoreDisplayMode": "binary", 336 | "rawValue": true 337 | }, 338 | "meta-refresh": { 339 | "score": null, 340 | "scoreDisplayMode": "not-applicable", 341 | "rawValue": true 342 | }, 343 | "meta-viewport": { 344 | "score": 1, 345 | "scoreDisplayMode": "binary", 346 | "rawValue": true 347 | }, 348 | "object-alt": { 349 | "score": null, 350 | "scoreDisplayMode": "not-applicable", 351 | "rawValue": true 352 | }, 353 | "tabindex": { 354 | "score": null, 355 | "scoreDisplayMode": "not-applicable", 356 | "rawValue": true 357 | }, 358 | "td-headers-attr": { 359 | "score": null, 360 | "scoreDisplayMode": "not-applicable", 361 | "rawValue": true 362 | }, 363 | "th-has-data-cells": { 364 | "score": null, 365 | "scoreDisplayMode": "not-applicable", 366 | "rawValue": true 367 | }, 368 | "valid-lang": { 369 | "score": null, 370 | "scoreDisplayMode": "not-applicable", 371 | "rawValue": true 372 | }, 373 | "video-caption": { 374 | "score": null, 375 | "scoreDisplayMode": "not-applicable", 376 | "rawValue": true 377 | }, 378 | "video-description": { 379 | "score": null, 380 | "scoreDisplayMode": "not-applicable", 381 | "rawValue": true 382 | }, 383 | "custom-controls-labels": { 384 | "score": null, 385 | "scoreDisplayMode": "manual", 386 | "rawValue": false 387 | }, 388 | "custom-controls-roles": { 389 | "score": null, 390 | "scoreDisplayMode": "manual", 391 | "rawValue": false 392 | }, 393 | "focus-traps": { 394 | "score": null, 395 | "scoreDisplayMode": "manual", 396 | "rawValue": false 397 | }, 398 | "focusable-controls": { 399 | "score": null, 400 | "scoreDisplayMode": "manual", 401 | "rawValue": false 402 | }, 403 | "heading-levels": { 404 | "score": null, 405 | "scoreDisplayMode": "manual", 406 | "rawValue": false 407 | }, 408 | "interactive-element-affordance": { 409 | "score": null, 410 | "scoreDisplayMode": "manual", 411 | "rawValue": false 412 | }, 413 | "logical-tab-order": { 414 | "score": null, 415 | "scoreDisplayMode": "manual", 416 | "rawValue": false 417 | }, 418 | "managed-focus": { 419 | "score": null, 420 | "scoreDisplayMode": "manual", 421 | "rawValue": false 422 | }, 423 | "offscreen-content-hidden": { 424 | "score": null, 425 | "scoreDisplayMode": "manual", 426 | "rawValue": false 427 | }, 428 | "use-landmarks": { 429 | "score": null, 430 | "scoreDisplayMode": "manual", 431 | "rawValue": false 432 | }, 433 | "visual-order-follows-dom": { 434 | "score": null, 435 | "scoreDisplayMode": "manual", 436 | "rawValue": false 437 | }, 438 | "uses-long-cache-ttl": { 439 | "score": 0.13, 440 | "scoreDisplayMode": "numeric", 441 | "rawValue": 507337.5946797763 442 | }, 443 | "total-byte-weight": { 444 | "score": 0.09, 445 | "scoreDisplayMode": "numeric", 446 | "rawValue": 6199100 447 | }, 448 | "offscreen-images": { 449 | "score": 0.88, 450 | "scoreDisplayMode": "numeric", 451 | "rawValue": 150, 452 | "warnings": [] 453 | }, 454 | "render-blocking-resources": { 455 | "score": 0.45, 456 | "scoreDisplayMode": "numeric", 457 | "rawValue": 1154 458 | }, 459 | "unminified-css": { 460 | "score": 1, 461 | "scoreDisplayMode": "numeric", 462 | "rawValue": 0 463 | }, 464 | "unminified-javascript": { 465 | "score": 0.58, 466 | "scoreDisplayMode": "numeric", 467 | "rawValue": 610, 468 | "warnings": [] 469 | }, 470 | "unused-css-rules": { 471 | "score": 0.44, 472 | "scoreDisplayMode": "numeric", 473 | "rawValue": 1230 474 | }, 475 | "uses-webp-images": { 476 | "score": 0, 477 | "scoreDisplayMode": "numeric", 478 | "rawValue": 11950, 479 | "warnings": [] 480 | }, 481 | "uses-optimized-images": { 482 | "score": 0, 483 | "scoreDisplayMode": "numeric", 484 | "rawValue": 5600, 485 | "warnings": [] 486 | }, 487 | "uses-text-compression": { 488 | "score": 1, 489 | "scoreDisplayMode": "numeric", 490 | "rawValue": 0 491 | }, 492 | "uses-responsive-images": { 493 | "score": 0, 494 | "scoreDisplayMode": "numeric", 495 | "rawValue": 6360, 496 | "warnings": [] 497 | }, 498 | "efficient-animated-content": { 499 | "score": 1, 500 | "scoreDisplayMode": "numeric", 501 | "rawValue": 0 502 | }, 503 | "appcache-manifest": { 504 | "score": 1, 505 | "scoreDisplayMode": "binary", 506 | "rawValue": true 507 | }, 508 | "doctype": { 509 | "score": 1, 510 | "scoreDisplayMode": "binary", 511 | "rawValue": true 512 | }, 513 | "dom-size": { 514 | "score": 0.87, 515 | "scoreDisplayMode": "numeric", 516 | "rawValue": 867 517 | }, 518 | "external-anchors-use-rel-noopener": { 519 | "score": 0, 520 | "scoreDisplayMode": "binary", 521 | "rawValue": false, 522 | "warnings": [] 523 | }, 524 | "geolocation-on-start": { 525 | "score": 1, 526 | "scoreDisplayMode": "binary", 527 | "rawValue": true 528 | }, 529 | "no-document-write": { 530 | "score": 1, 531 | "scoreDisplayMode": "binary", 532 | "rawValue": true 533 | }, 534 | "no-vulnerable-libraries": { 535 | "score": 0, 536 | "scoreDisplayMode": "binary", 537 | "rawValue": false 538 | }, 539 | "no-websql": { 540 | "score": 1, 541 | "scoreDisplayMode": "binary", 542 | "rawValue": true 543 | }, 544 | "notification-on-start": { 545 | "score": 1, 546 | "scoreDisplayMode": "binary", 547 | "rawValue": true 548 | }, 549 | "password-inputs-can-be-pasted-into": { 550 | "score": 1, 551 | "scoreDisplayMode": "binary", 552 | "rawValue": true 553 | }, 554 | "uses-http2": { 555 | "score": 1, 556 | "scoreDisplayMode": "binary", 557 | "rawValue": true 558 | }, 559 | "uses-passive-event-listeners": { 560 | "score": 0, 561 | "scoreDisplayMode": "binary", 562 | "rawValue": false 563 | }, 564 | "meta-description": { 565 | "score": 1, 566 | "scoreDisplayMode": "binary", 567 | "rawValue": true 568 | }, 569 | "http-status-code": { 570 | "score": 1, 571 | "scoreDisplayMode": "binary", 572 | "rawValue": true 573 | }, 574 | "font-size": { 575 | "score": 0, 576 | "scoreDisplayMode": "binary", 577 | "rawValue": false, 578 | "explanation": "77.28% of text is too small." 579 | }, 580 | "link-text": { 581 | "score": 1, 582 | "scoreDisplayMode": "binary", 583 | "rawValue": true 584 | }, 585 | "is-crawlable": { 586 | "score": 1, 587 | "scoreDisplayMode": "binary", 588 | "rawValue": true 589 | }, 590 | "robots-txt": { 591 | "score": 1, 592 | "scoreDisplayMode": "binary", 593 | "rawValue": true 594 | }, 595 | "hreflang": { 596 | "score": 1, 597 | "scoreDisplayMode": "binary", 598 | "rawValue": true 599 | }, 600 | "plugins": { 601 | "score": 1, 602 | "scoreDisplayMode": "binary", 603 | "rawValue": true 604 | }, 605 | "canonical": { 606 | "score": 1, 607 | "scoreDisplayMode": "binary", 608 | "rawValue": true 609 | }, 610 | "mobile-friendly": { 611 | "score": null, 612 | "scoreDisplayMode": "manual", 613 | "rawValue": false 614 | }, 615 | "structured-data": { 616 | "score": null, 617 | "scoreDisplayMode": "manual", 618 | "rawValue": false 619 | } 620 | }, 621 | "configSettings": { 622 | "output": "json", 623 | "maxWaitForLoad": 45000, 624 | "throttlingMethod": "simulate", 625 | "throttling": { 626 | "rttMs": 150, 627 | "throughputKbps": 1638.4, 628 | "requestLatencyMs": 562.5, 629 | "downloadThroughputKbps": 1474.5600000000002, 630 | "uploadThroughputKbps": 675, 631 | "cpuSlowdownMultiplier": 4 632 | }, 633 | "auditMode": false, 634 | "gatherMode": false, 635 | "disableStorageReset": false, 636 | "disableDeviceEmulation": false, 637 | "locale": "en-US", 638 | "blockedUrlPatterns": null, 639 | "additionalTraceCategories": null, 640 | "extraHeaders": {}, 641 | "onlyAudits": null, 642 | "onlyCategories": null, 643 | "skipAudits": null 644 | }, 645 | "categories": { 646 | "performance": { 647 | "title": "Performance", 648 | "id": "performance", 649 | "score": 0.44 650 | }, 651 | "pwa": { 652 | "title": "Progressive Web App", 653 | "description": "These checks validate the aspects of a Progressive Web App, as specified by the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist).", 654 | "manualDescription": "These checks are required by the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist) but are not automatically checked by Lighthouse. They do not affect your score but it's important that you verify them manually.", 655 | "id": "pwa", 656 | "score": 0.5 657 | }, 658 | "accessibility": { 659 | "title": "Accessibility", 660 | "description": "These checks highlight opportunities to [improve the accessibility of your web app](https://developers.google.com/web/fundamentals/accessibility). Only a subset of accessibility issues can be automatically detected so manual testing is also encouraged.", 661 | "manualDescription": "These items address areas which an automated testing tool cannot cover. Learn more in our guide on [conducting an accessibility review](https://developers.google.com/web/fundamentals/accessibility/how-to-review).", 662 | "id": "accessibility", 663 | "score": 0.81 664 | }, 665 | "best-practices": { 666 | "title": "Best Practices", 667 | "id": "best-practices", 668 | "score": 0.73 669 | }, 670 | "seo": { 671 | "title": "SEO", 672 | "description": "These checks ensure that your page is optimized for search engine results ranking. There are additional factors Lighthouse does not check that may affect your search ranking. [Learn more](https://support.google.com/webmasters/answer/35769).", 673 | "manualDescription": "Run these additional validators on your site to check additional SEO best practices.", 674 | "id": "seo", 675 | "score": 0.91 676 | } 677 | }, 678 | "categoryGroups": null, 679 | "timing": { 680 | "total": 22396 681 | }, 682 | "i18n": null 683 | } -------------------------------------------------------------------------------- /test/fs-store/www.google.com/2019-01-16T18:30:37.721Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://www.google.com/", 3 | "domainName": "www.google.com", 4 | "rootDomain": "google.com", 5 | "group": "web", 6 | "delay": 0, 7 | "delayTime": 0, 8 | "state": "processed", 9 | "createDate": 1547663437720, 10 | "id": "www.google.com/2019-01-16T18:30:37.721Z.json", 11 | "attempt": 1, 12 | "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/71.0.3578.98 Safari/537.36", 13 | "environment": { 14 | "networkUserAgent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/71.0.3559.0 Mobile Safari/537.36", 15 | "hostUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/71.0.3578.98 Safari/537.36", 16 | "benchmarkIndex": 1272 17 | }, 18 | "lighthouseVersion": "3.2.1", 19 | "fetchTime": "2019-01-16T18:30:39.518Z", 20 | "finalUrl": "https://www.google.com/", 21 | "runWarnings": null, 22 | "runtimeError": null, 23 | "audits": { 24 | "is-on-https": { 25 | "score": 1, 26 | "scoreDisplayMode": "binary", 27 | "rawValue": true 28 | }, 29 | "redirects-http": { 30 | "score": 1, 31 | "scoreDisplayMode": "binary", 32 | "rawValue": true 33 | }, 34 | "service-worker": { 35 | "score": 0, 36 | "scoreDisplayMode": "binary", 37 | "rawValue": false 38 | }, 39 | "works-offline": { 40 | "score": 0, 41 | "scoreDisplayMode": "binary", 42 | "rawValue": false, 43 | "warnings": [] 44 | }, 45 | "viewport": { 46 | "score": 1, 47 | "scoreDisplayMode": "binary", 48 | "rawValue": true, 49 | "warnings": [] 50 | }, 51 | "without-javascript": { 52 | "score": 1, 53 | "scoreDisplayMode": "binary", 54 | "rawValue": true 55 | }, 56 | "first-contentful-paint": { 57 | "score": 1, 58 | "scoreDisplayMode": "numeric", 59 | "rawValue": 932.4559999999999 60 | }, 61 | "first-meaningful-paint": { 62 | "score": 1, 63 | "scoreDisplayMode": "numeric", 64 | "rawValue": 977.4559999999999 65 | }, 66 | "load-fast-enough-for-pwa": { 67 | "score": 1, 68 | "scoreDisplayMode": "binary", 69 | "rawValue": 3013.1934999999994 70 | }, 71 | "speed-index": { 72 | "score": 1, 73 | "scoreDisplayMode": "numeric", 74 | "rawValue": 932.4559999999999 75 | }, 76 | "screenshot-thumbnails": { 77 | "score": null, 78 | "scoreDisplayMode": "informative", 79 | "rawValue": true 80 | }, 81 | "final-screenshot": { 82 | "score": null, 83 | "scoreDisplayMode": "informative", 84 | "rawValue": true 85 | }, 86 | "estimated-input-latency": { 87 | "score": 1, 88 | "scoreDisplayMode": "numeric", 89 | "rawValue": 15.666666666666668 90 | }, 91 | "errors-in-console": { 92 | "score": 1, 93 | "scoreDisplayMode": "binary", 94 | "rawValue": 0 95 | }, 96 | "time-to-first-byte": { 97 | "score": 1, 98 | "scoreDisplayMode": "binary", 99 | "rawValue": 112.38699999999999 100 | }, 101 | "first-cpu-idle": { 102 | "score": 0.97, 103 | "scoreDisplayMode": "numeric", 104 | "rawValue": 2767.8459999999995 105 | }, 106 | "interactive": { 107 | "score": 0.96, 108 | "scoreDisplayMode": "numeric", 109 | "rawValue": 3013.1934999999994 110 | }, 111 | "user-timings": { 112 | "score": null, 113 | "scoreDisplayMode": "not-applicable", 114 | "rawValue": true 115 | }, 116 | "critical-request-chains": { 117 | "score": null, 118 | "scoreDisplayMode": "informative", 119 | "rawValue": false 120 | }, 121 | "redirects": { 122 | "score": 1, 123 | "scoreDisplayMode": "numeric", 124 | "rawValue": 0 125 | }, 126 | "webapp-install-banner": { 127 | "score": 0, 128 | "scoreDisplayMode": "binary", 129 | "rawValue": false, 130 | "explanation": "Failures: No manifest was fetched,\nSite does not register a service worker." 131 | }, 132 | "splash-screen": { 133 | "score": 0, 134 | "scoreDisplayMode": "binary", 135 | "rawValue": false, 136 | "explanation": "Failures: No manifest was fetched." 137 | }, 138 | "themed-omnibox": { 139 | "score": 0, 140 | "scoreDisplayMode": "binary", 141 | "rawValue": false, 142 | "explanation": "Failures: No manifest was fetched,\nNo `` tag found." 143 | }, 144 | "manifest-short-name-length": { 145 | "score": null, 146 | "scoreDisplayMode": "not-applicable", 147 | "rawValue": true 148 | }, 149 | "content-width": { 150 | "score": 1, 151 | "scoreDisplayMode": "binary", 152 | "rawValue": true, 153 | "explanation": "" 154 | }, 155 | "image-aspect-ratio": { 156 | "score": 1, 157 | "scoreDisplayMode": "binary", 158 | "rawValue": true, 159 | "warnings": [] 160 | }, 161 | "deprecations": { 162 | "score": 1, 163 | "scoreDisplayMode": "binary", 164 | "rawValue": true 165 | }, 166 | "mainthread-work-breakdown": { 167 | "score": 1, 168 | "scoreDisplayMode": "numeric", 169 | "rawValue": 998.8160000000005 170 | }, 171 | "bootup-time": { 172 | "score": 1, 173 | "scoreDisplayMode": "numeric", 174 | "rawValue": 464.12800000000004 175 | }, 176 | "uses-rel-preload": { 177 | "score": 1, 178 | "scoreDisplayMode": "numeric", 179 | "rawValue": 0 180 | }, 181 | "uses-rel-preconnect": { 182 | "score": 0.75, 183 | "scoreDisplayMode": "numeric", 184 | "rawValue": 295.73799992656706 185 | }, 186 | "font-display": { 187 | "score": 1, 188 | "scoreDisplayMode": "binary", 189 | "rawValue": true 190 | }, 191 | "network-requests": { 192 | "score": null, 193 | "scoreDisplayMode": "informative", 194 | "rawValue": 18 195 | }, 196 | "metrics": { 197 | "score": null, 198 | "scoreDisplayMode": "informative", 199 | "rawValue": 3013.1934999999994 200 | }, 201 | "pwa-cross-browser": { 202 | "score": null, 203 | "scoreDisplayMode": "manual", 204 | "rawValue": false 205 | }, 206 | "pwa-page-transitions": { 207 | "score": null, 208 | "scoreDisplayMode": "manual", 209 | "rawValue": false 210 | }, 211 | "pwa-each-page-has-url": { 212 | "score": null, 213 | "scoreDisplayMode": "manual", 214 | "rawValue": false 215 | }, 216 | "accesskeys": { 217 | "score": null, 218 | "scoreDisplayMode": "not-applicable", 219 | "rawValue": true 220 | }, 221 | "aria-allowed-attr": { 222 | "score": 1, 223 | "scoreDisplayMode": "binary", 224 | "rawValue": true 225 | }, 226 | "aria-required-attr": { 227 | "score": 1, 228 | "scoreDisplayMode": "binary", 229 | "rawValue": true 230 | }, 231 | "aria-required-children": { 232 | "score": 1, 233 | "scoreDisplayMode": "binary", 234 | "rawValue": true 235 | }, 236 | "aria-required-parent": { 237 | "score": 1, 238 | "scoreDisplayMode": "binary", 239 | "rawValue": true 240 | }, 241 | "aria-roles": { 242 | "score": 1, 243 | "scoreDisplayMode": "binary", 244 | "rawValue": true 245 | }, 246 | "aria-valid-attr-value": { 247 | "score": 1, 248 | "scoreDisplayMode": "binary", 249 | "rawValue": true 250 | }, 251 | "aria-valid-attr": { 252 | "score": 1, 253 | "scoreDisplayMode": "binary", 254 | "rawValue": true 255 | }, 256 | "audio-caption": { 257 | "score": null, 258 | "scoreDisplayMode": "not-applicable", 259 | "rawValue": true 260 | }, 261 | "button-name": { 262 | "score": 1, 263 | "scoreDisplayMode": "binary", 264 | "rawValue": true 265 | }, 266 | "bypass": { 267 | "score": 1, 268 | "scoreDisplayMode": "binary", 269 | "rawValue": true 270 | }, 271 | "color-contrast": { 272 | "score": 0, 273 | "scoreDisplayMode": "binary", 274 | "rawValue": false 275 | }, 276 | "definition-list": { 277 | "score": null, 278 | "scoreDisplayMode": "not-applicable", 279 | "rawValue": true 280 | }, 281 | "dlitem": { 282 | "score": null, 283 | "scoreDisplayMode": "not-applicable", 284 | "rawValue": true 285 | }, 286 | "document-title": { 287 | "score": 1, 288 | "scoreDisplayMode": "binary", 289 | "rawValue": true 290 | }, 291 | "duplicate-id": { 292 | "score": 1, 293 | "scoreDisplayMode": "binary", 294 | "rawValue": true 295 | }, 296 | "frame-title": { 297 | "score": null, 298 | "scoreDisplayMode": "not-applicable", 299 | "rawValue": true 300 | }, 301 | "html-has-lang": { 302 | "score": 1, 303 | "scoreDisplayMode": "binary", 304 | "rawValue": true 305 | }, 306 | "html-lang-valid": { 307 | "score": 1, 308 | "scoreDisplayMode": "binary", 309 | "rawValue": true 310 | }, 311 | "image-alt": { 312 | "score": 1, 313 | "scoreDisplayMode": "binary", 314 | "rawValue": true 315 | }, 316 | "input-image-alt": { 317 | "score": null, 318 | "scoreDisplayMode": "not-applicable", 319 | "rawValue": true 320 | }, 321 | "label": { 322 | "score": 1, 323 | "scoreDisplayMode": "binary", 324 | "rawValue": true 325 | }, 326 | "layout-table": { 327 | "score": null, 328 | "scoreDisplayMode": "not-applicable", 329 | "rawValue": true 330 | }, 331 | "link-name": { 332 | "score": 0, 333 | "scoreDisplayMode": "binary", 334 | "rawValue": false 335 | }, 336 | "list": { 337 | "score": null, 338 | "scoreDisplayMode": "not-applicable", 339 | "rawValue": true 340 | }, 341 | "listitem": { 342 | "score": null, 343 | "scoreDisplayMode": "not-applicable", 344 | "rawValue": true 345 | }, 346 | "meta-refresh": { 347 | "score": null, 348 | "scoreDisplayMode": "not-applicable", 349 | "rawValue": true 350 | }, 351 | "meta-viewport": { 352 | "score": 1, 353 | "scoreDisplayMode": "binary", 354 | "rawValue": true 355 | }, 356 | "object-alt": { 357 | "score": null, 358 | "scoreDisplayMode": "not-applicable", 359 | "rawValue": true 360 | }, 361 | "tabindex": { 362 | "score": 1, 363 | "scoreDisplayMode": "binary", 364 | "rawValue": true 365 | }, 366 | "td-headers-attr": { 367 | "score": null, 368 | "scoreDisplayMode": "not-applicable", 369 | "rawValue": true 370 | }, 371 | "th-has-data-cells": { 372 | "score": null, 373 | "scoreDisplayMode": "not-applicable", 374 | "rawValue": true 375 | }, 376 | "valid-lang": { 377 | "score": null, 378 | "scoreDisplayMode": "not-applicable", 379 | "rawValue": true 380 | }, 381 | "video-caption": { 382 | "score": null, 383 | "scoreDisplayMode": "not-applicable", 384 | "rawValue": true 385 | }, 386 | "video-description": { 387 | "score": null, 388 | "scoreDisplayMode": "not-applicable", 389 | "rawValue": true 390 | }, 391 | "custom-controls-labels": { 392 | "score": null, 393 | "scoreDisplayMode": "manual", 394 | "rawValue": false 395 | }, 396 | "custom-controls-roles": { 397 | "score": null, 398 | "scoreDisplayMode": "manual", 399 | "rawValue": false 400 | }, 401 | "focus-traps": { 402 | "score": null, 403 | "scoreDisplayMode": "manual", 404 | "rawValue": false 405 | }, 406 | "focusable-controls": { 407 | "score": null, 408 | "scoreDisplayMode": "manual", 409 | "rawValue": false 410 | }, 411 | "heading-levels": { 412 | "score": null, 413 | "scoreDisplayMode": "manual", 414 | "rawValue": false 415 | }, 416 | "interactive-element-affordance": { 417 | "score": null, 418 | "scoreDisplayMode": "manual", 419 | "rawValue": false 420 | }, 421 | "logical-tab-order": { 422 | "score": null, 423 | "scoreDisplayMode": "manual", 424 | "rawValue": false 425 | }, 426 | "managed-focus": { 427 | "score": null, 428 | "scoreDisplayMode": "manual", 429 | "rawValue": false 430 | }, 431 | "offscreen-content-hidden": { 432 | "score": null, 433 | "scoreDisplayMode": "manual", 434 | "rawValue": false 435 | }, 436 | "use-landmarks": { 437 | "score": null, 438 | "scoreDisplayMode": "manual", 439 | "rawValue": false 440 | }, 441 | "visual-order-follows-dom": { 442 | "score": null, 443 | "scoreDisplayMode": "manual", 444 | "rawValue": false 445 | }, 446 | "uses-long-cache-ttl": { 447 | "score": 1, 448 | "scoreDisplayMode": "numeric", 449 | "rawValue": 0 450 | }, 451 | "total-byte-weight": { 452 | "score": 1, 453 | "scoreDisplayMode": "numeric", 454 | "rawValue": 295578 455 | }, 456 | "offscreen-images": { 457 | "score": 1, 458 | "scoreDisplayMode": "numeric", 459 | "rawValue": 0, 460 | "warnings": [] 461 | }, 462 | "render-blocking-resources": { 463 | "score": 1, 464 | "scoreDisplayMode": "numeric", 465 | "rawValue": 0 466 | }, 467 | "unminified-css": { 468 | "score": 1, 469 | "scoreDisplayMode": "numeric", 470 | "rawValue": 0 471 | }, 472 | "unminified-javascript": { 473 | "score": 1, 474 | "scoreDisplayMode": "numeric", 475 | "rawValue": 0, 476 | "warnings": [] 477 | }, 478 | "unused-css-rules": { 479 | "score": 1, 480 | "scoreDisplayMode": "numeric", 481 | "rawValue": 0 482 | }, 483 | "uses-webp-images": { 484 | "score": 1, 485 | "scoreDisplayMode": "numeric", 486 | "rawValue": 0, 487 | "warnings": [] 488 | }, 489 | "uses-optimized-images": { 490 | "score": 1, 491 | "scoreDisplayMode": "numeric", 492 | "rawValue": 0, 493 | "warnings": [] 494 | }, 495 | "uses-text-compression": { 496 | "score": 1, 497 | "scoreDisplayMode": "numeric", 498 | "rawValue": 0 499 | }, 500 | "uses-responsive-images": { 501 | "score": 1, 502 | "scoreDisplayMode": "numeric", 503 | "rawValue": 0, 504 | "warnings": [] 505 | }, 506 | "efficient-animated-content": { 507 | "score": 1, 508 | "scoreDisplayMode": "numeric", 509 | "rawValue": 0 510 | }, 511 | "appcache-manifest": { 512 | "score": 1, 513 | "scoreDisplayMode": "binary", 514 | "rawValue": true 515 | }, 516 | "doctype": { 517 | "score": 1, 518 | "scoreDisplayMode": "binary", 519 | "rawValue": true 520 | }, 521 | "dom-size": { 522 | "score": 1, 523 | "scoreDisplayMode": "numeric", 524 | "rawValue": 434 525 | }, 526 | "external-anchors-use-rel-noopener": { 527 | "score": 1, 528 | "scoreDisplayMode": "binary", 529 | "rawValue": true, 530 | "warnings": [] 531 | }, 532 | "geolocation-on-start": { 533 | "score": 1, 534 | "scoreDisplayMode": "binary", 535 | "rawValue": true 536 | }, 537 | "no-document-write": { 538 | "score": 1, 539 | "scoreDisplayMode": "binary", 540 | "rawValue": true 541 | }, 542 | "no-vulnerable-libraries": { 543 | "score": 1, 544 | "scoreDisplayMode": "binary", 545 | "rawValue": true 546 | }, 547 | "js-libraries": { 548 | "score": 1, 549 | "scoreDisplayMode": "binary", 550 | "rawValue": true 551 | }, 552 | "no-websql": { 553 | "score": 1, 554 | "scoreDisplayMode": "binary", 555 | "rawValue": true 556 | }, 557 | "notification-on-start": { 558 | "score": 1, 559 | "scoreDisplayMode": "binary", 560 | "rawValue": true 561 | }, 562 | "password-inputs-can-be-pasted-into": { 563 | "score": 1, 564 | "scoreDisplayMode": "binary", 565 | "rawValue": true 566 | }, 567 | "uses-http2": { 568 | "score": 1, 569 | "scoreDisplayMode": "binary", 570 | "rawValue": true 571 | }, 572 | "uses-passive-event-listeners": { 573 | "score": 0, 574 | "scoreDisplayMode": "binary", 575 | "rawValue": false 576 | }, 577 | "meta-description": { 578 | "score": 0, 579 | "scoreDisplayMode": "binary", 580 | "rawValue": false 581 | }, 582 | "http-status-code": { 583 | "score": 1, 584 | "scoreDisplayMode": "binary", 585 | "rawValue": true 586 | }, 587 | "font-size": { 588 | "score": 1, 589 | "scoreDisplayMode": "binary", 590 | "rawValue": true 591 | }, 592 | "link-text": { 593 | "score": 0, 594 | "scoreDisplayMode": "binary", 595 | "rawValue": false 596 | }, 597 | "is-crawlable": { 598 | "score": 1, 599 | "scoreDisplayMode": "binary", 600 | "rawValue": true 601 | }, 602 | "robots-txt": { 603 | "score": 1, 604 | "scoreDisplayMode": "binary", 605 | "rawValue": true 606 | }, 607 | "hreflang": { 608 | "score": 1, 609 | "scoreDisplayMode": "binary", 610 | "rawValue": true 611 | }, 612 | "plugins": { 613 | "score": 1, 614 | "scoreDisplayMode": "binary", 615 | "rawValue": true 616 | }, 617 | "canonical": { 618 | "score": null, 619 | "scoreDisplayMode": "not-applicable", 620 | "rawValue": true 621 | }, 622 | "mobile-friendly": { 623 | "score": null, 624 | "scoreDisplayMode": "manual", 625 | "rawValue": false 626 | }, 627 | "structured-data": { 628 | "score": null, 629 | "scoreDisplayMode": "manual", 630 | "rawValue": false 631 | } 632 | }, 633 | "configSettings": { 634 | "output": "json", 635 | "maxWaitForLoad": 45000, 636 | "throttlingMethod": "simulate", 637 | "throttling": { 638 | "rttMs": 150, 639 | "throughputKbps": 1638.4, 640 | "requestLatencyMs": 562.5, 641 | "downloadThroughputKbps": 1474.5600000000002, 642 | "uploadThroughputKbps": 675, 643 | "cpuSlowdownMultiplier": 4 644 | }, 645 | "auditMode": false, 646 | "gatherMode": false, 647 | "disableStorageReset": false, 648 | "disableDeviceEmulation": false, 649 | "emulatedFormFactor": "mobile", 650 | "locale": "en-US", 651 | "blockedUrlPatterns": null, 652 | "additionalTraceCategories": null, 653 | "extraHeaders": {}, 654 | "onlyAudits": null, 655 | "onlyCategories": null, 656 | "skipAudits": null, 657 | "throttlingPreset": "mobile3G" 658 | }, 659 | "categories": { 660 | "performance": { 661 | "title": "Performance", 662 | "id": "performance", 663 | "score": 0.98 664 | }, 665 | "pwa": { 666 | "title": "Progressive Web App", 667 | "description": "These checks validate the aspects of a Progressive Web App, as specified by the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist).", 668 | "manualDescription": "These checks are required by the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist) but are not automatically checked by Lighthouse. They do not affect your score but it's important that you verify them manually.", 669 | "id": "pwa", 670 | "score": 0.58 671 | }, 672 | "accessibility": { 673 | "title": "Accessibility", 674 | "description": "These checks highlight opportunities to [improve the accessibility of your web app](https://developers.google.com/web/fundamentals/accessibility). Only a subset of accessibility issues can be automatically detected so manual testing is also encouraged.", 675 | "manualDescription": "These items address areas which an automated testing tool cannot cover. Learn more in our guide on [conducting an accessibility review](https://developers.google.com/web/fundamentals/accessibility/how-to-review).", 676 | "id": "accessibility", 677 | "score": 0.84 678 | }, 679 | "best-practices": { 680 | "title": "Best Practices", 681 | "id": "best-practices", 682 | "score": 0.93 683 | }, 684 | "seo": { 685 | "title": "SEO", 686 | "description": "These checks ensure that your page is optimized for search engine results ranking. There are additional factors Lighthouse does not check that may affect your search ranking. [Learn more](https://support.google.com/webmasters/answer/35769).", 687 | "manualDescription": "Run these additional validators on your site to check additional SEO best practices.", 688 | "id": "seo", 689 | "score": 0.8 690 | } 691 | }, 692 | "categoryGroups": null, 693 | "timing": { 694 | "total": 5576 695 | }, 696 | "i18n": null 697 | } -------------------------------------------------------------------------------- /test/fs-store/raw1.godaddysites.com/2019-01-16T21:11:04.685Z.json: -------------------------------------------------------------------------------- 1 | { 2 | "requestedUrl": "https://raw1.godaddysites.com/", 3 | "domainName": "raw1.godaddysites.com", 4 | "rootDomain": "godaddysites.com", 5 | "group": "web", 6 | "delay": 0, 7 | "delayTime": 0, 8 | "state": "processed", 9 | "createDate": 1547673064685, 10 | "id": "raw1.godaddysites.com/2019-01-16T21:11:04.685Z.json", 11 | "attempt": 1, 12 | "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/71.0.3578.98 Safari/537.36", 13 | "environment": { 14 | "networkUserAgent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/71.0.3559.0 Mobile Safari/537.36", 15 | "hostUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/71.0.3578.98 Safari/537.36", 16 | "benchmarkIndex": 1238 17 | }, 18 | "lighthouseVersion": "3.2.1", 19 | "fetchTime": "2019-01-16T21:11:29.418Z", 20 | "finalUrl": "https://raw1.godaddysites.com/", 21 | "runWarnings": null, 22 | "runtimeError": null, 23 | "audits": { 24 | "is-on-https": { 25 | "score": 1, 26 | "scoreDisplayMode": "binary", 27 | "rawValue": true 28 | }, 29 | "redirects-http": { 30 | "score": 1, 31 | "scoreDisplayMode": "binary", 32 | "rawValue": true 33 | }, 34 | "service-worker": { 35 | "score": 0, 36 | "scoreDisplayMode": "binary", 37 | "rawValue": false 38 | }, 39 | "works-offline": { 40 | "score": 0, 41 | "scoreDisplayMode": "binary", 42 | "rawValue": false, 43 | "warnings": [] 44 | }, 45 | "viewport": { 46 | "score": 1, 47 | "scoreDisplayMode": "binary", 48 | "rawValue": true, 49 | "warnings": [] 50 | }, 51 | "without-javascript": { 52 | "score": 1, 53 | "scoreDisplayMode": "binary", 54 | "rawValue": true 55 | }, 56 | "first-contentful-paint": { 57 | "score": 0.75, 58 | "scoreDisplayMode": "numeric", 59 | "rawValue": 3020.366 60 | }, 61 | "first-meaningful-paint": { 62 | "score": 0.71, 63 | "scoreDisplayMode": "numeric", 64 | "rawValue": 3160.3430000000003 65 | }, 66 | "load-fast-enough-for-pwa": { 67 | "score": 0, 68 | "scoreDisplayMode": "binary", 69 | "rawValue": 14932.026000000002, 70 | "explanation": "Your page loads too slowly and is not interactive within 10 seconds. Look at the opportunities and diagnostics in the \"Performance\" section to learn how to improve." 71 | }, 72 | "speed-index": { 73 | "score": 0.18, 74 | "scoreDisplayMode": "numeric", 75 | "rawValue": 8499.893545988596 76 | }, 77 | "screenshot-thumbnails": { 78 | "score": null, 79 | "scoreDisplayMode": "informative", 80 | "rawValue": true 81 | }, 82 | "final-screenshot": { 83 | "score": null, 84 | "scoreDisplayMode": "informative", 85 | "rawValue": true 86 | }, 87 | "estimated-input-latency": { 88 | "score": 0, 89 | "scoreDisplayMode": "numeric", 90 | "rawValue": 452 91 | }, 92 | "errors-in-console": { 93 | "score": 0, 94 | "scoreDisplayMode": "binary", 95 | "rawValue": 5 96 | }, 97 | "time-to-first-byte": { 98 | "score": 1, 99 | "scoreDisplayMode": "binary", 100 | "rawValue": 147.471 101 | }, 102 | "first-cpu-idle": { 103 | "score": 0.14, 104 | "scoreDisplayMode": "numeric", 105 | "rawValue": 10690.729 106 | }, 107 | "interactive": { 108 | "score": 0.08, 109 | "scoreDisplayMode": "numeric", 110 | "rawValue": 14932.026000000002 111 | }, 112 | "user-timings": { 113 | "score": null, 114 | "scoreDisplayMode": "informative", 115 | "rawValue": false 116 | }, 117 | "critical-request-chains": { 118 | "score": null, 119 | "scoreDisplayMode": "informative", 120 | "rawValue": false 121 | }, 122 | "redirects": { 123 | "score": 1, 124 | "scoreDisplayMode": "numeric", 125 | "rawValue": 0 126 | }, 127 | "webapp-install-banner": { 128 | "score": 0, 129 | "scoreDisplayMode": "binary", 130 | "rawValue": false, 131 | "explanation": "Failures: No manifest was fetched,\nSite does not register a service worker." 132 | }, 133 | "splash-screen": { 134 | "score": 0, 135 | "scoreDisplayMode": "binary", 136 | "rawValue": false, 137 | "explanation": "Failures: No manifest was fetched." 138 | }, 139 | "themed-omnibox": { 140 | "score": 0, 141 | "scoreDisplayMode": "binary", 142 | "rawValue": false, 143 | "explanation": "Failures: No manifest was fetched,\nNo `` tag found." 144 | }, 145 | "manifest-short-name-length": { 146 | "score": null, 147 | "scoreDisplayMode": "not-applicable", 148 | "rawValue": true 149 | }, 150 | "content-width": { 151 | "score": 1, 152 | "scoreDisplayMode": "binary", 153 | "rawValue": true, 154 | "explanation": "" 155 | }, 156 | "image-aspect-ratio": { 157 | "score": 1, 158 | "scoreDisplayMode": "binary", 159 | "rawValue": true, 160 | "warnings": [] 161 | }, 162 | "deprecations": { 163 | "score": 1, 164 | "scoreDisplayMode": "binary", 165 | "rawValue": true 166 | }, 167 | "mainthread-work-breakdown": { 168 | "score": 0.18, 169 | "scoreDisplayMode": "numeric", 170 | "rawValue": 6561.147999999995 171 | }, 172 | "bootup-time": { 173 | "score": 0.5, 174 | "scoreDisplayMode": "numeric", 175 | "rawValue": 3473.499999999999 176 | }, 177 | "uses-rel-preload": { 178 | "score": 1, 179 | "scoreDisplayMode": "numeric", 180 | "rawValue": 0 181 | }, 182 | "uses-rel-preconnect": { 183 | "score": 0.67, 184 | "scoreDisplayMode": "numeric", 185 | "rawValue": 438.288 186 | }, 187 | "font-display": { 188 | "score": 1, 189 | "scoreDisplayMode": "binary", 190 | "rawValue": true 191 | }, 192 | "network-requests": { 193 | "score": null, 194 | "scoreDisplayMode": "informative", 195 | "rawValue": 62 196 | }, 197 | "metrics": { 198 | "score": null, 199 | "scoreDisplayMode": "informative", 200 | "rawValue": 14932.026000000002 201 | }, 202 | "pwa-cross-browser": { 203 | "score": null, 204 | "scoreDisplayMode": "manual", 205 | "rawValue": false 206 | }, 207 | "pwa-page-transitions": { 208 | "score": null, 209 | "scoreDisplayMode": "manual", 210 | "rawValue": false 211 | }, 212 | "pwa-each-page-has-url": { 213 | "score": null, 214 | "scoreDisplayMode": "manual", 215 | "rawValue": false 216 | }, 217 | "accesskeys": { 218 | "score": null, 219 | "scoreDisplayMode": "not-applicable", 220 | "rawValue": true 221 | }, 222 | "aria-allowed-attr": { 223 | "score": 1, 224 | "scoreDisplayMode": "binary", 225 | "rawValue": true 226 | }, 227 | "aria-required-attr": { 228 | "score": null, 229 | "scoreDisplayMode": "not-applicable", 230 | "rawValue": true 231 | }, 232 | "aria-required-children": { 233 | "score": null, 234 | "scoreDisplayMode": "not-applicable", 235 | "rawValue": true 236 | }, 237 | "aria-required-parent": { 238 | "score": null, 239 | "scoreDisplayMode": "not-applicable", 240 | "rawValue": true 241 | }, 242 | "aria-roles": { 243 | "score": null, 244 | "scoreDisplayMode": "not-applicable", 245 | "rawValue": true 246 | }, 247 | "aria-valid-attr-value": { 248 | "score": 1, 249 | "scoreDisplayMode": "binary", 250 | "rawValue": true 251 | }, 252 | "aria-valid-attr": { 253 | "score": 1, 254 | "scoreDisplayMode": "binary", 255 | "rawValue": true 256 | }, 257 | "audio-caption": { 258 | "score": null, 259 | "scoreDisplayMode": "not-applicable", 260 | "rawValue": true 261 | }, 262 | "button-name": { 263 | "score": 1, 264 | "scoreDisplayMode": "binary", 265 | "rawValue": true 266 | }, 267 | "bypass": { 268 | "score": 1, 269 | "scoreDisplayMode": "binary", 270 | "rawValue": true 271 | }, 272 | "color-contrast": { 273 | "score": 0, 274 | "scoreDisplayMode": "binary", 275 | "rawValue": false 276 | }, 277 | "definition-list": { 278 | "score": null, 279 | "scoreDisplayMode": "not-applicable", 280 | "rawValue": true 281 | }, 282 | "dlitem": { 283 | "score": null, 284 | "scoreDisplayMode": "not-applicable", 285 | "rawValue": true 286 | }, 287 | "document-title": { 288 | "score": 1, 289 | "scoreDisplayMode": "binary", 290 | "rawValue": true 291 | }, 292 | "duplicate-id": { 293 | "score": 1, 294 | "scoreDisplayMode": "binary", 295 | "rawValue": true 296 | }, 297 | "frame-title": { 298 | "score": 0, 299 | "scoreDisplayMode": "binary", 300 | "rawValue": false 301 | }, 302 | "html-has-lang": { 303 | "score": 1, 304 | "scoreDisplayMode": "binary", 305 | "rawValue": true 306 | }, 307 | "html-lang-valid": { 308 | "score": 1, 309 | "scoreDisplayMode": "binary", 310 | "rawValue": true 311 | }, 312 | "image-alt": { 313 | "score": 0, 314 | "scoreDisplayMode": "binary", 315 | "rawValue": false 316 | }, 317 | "input-image-alt": { 318 | "score": null, 319 | "scoreDisplayMode": "not-applicable", 320 | "rawValue": true 321 | }, 322 | "label": { 323 | "score": 0, 324 | "scoreDisplayMode": "binary", 325 | "rawValue": false 326 | }, 327 | "layout-table": { 328 | "score": 1, 329 | "scoreDisplayMode": "binary", 330 | "rawValue": true 331 | }, 332 | "link-name": { 333 | "score": 1, 334 | "scoreDisplayMode": "binary", 335 | "rawValue": true 336 | }, 337 | "list": { 338 | "score": 1, 339 | "scoreDisplayMode": "binary", 340 | "rawValue": true 341 | }, 342 | "listitem": { 343 | "score": 1, 344 | "scoreDisplayMode": "binary", 345 | "rawValue": true 346 | }, 347 | "meta-refresh": { 348 | "score": null, 349 | "scoreDisplayMode": "not-applicable", 350 | "rawValue": true 351 | }, 352 | "meta-viewport": { 353 | "score": 1, 354 | "scoreDisplayMode": "binary", 355 | "rawValue": true 356 | }, 357 | "object-alt": { 358 | "score": null, 359 | "scoreDisplayMode": "not-applicable", 360 | "rawValue": true 361 | }, 362 | "tabindex": { 363 | "score": 1, 364 | "scoreDisplayMode": "binary", 365 | "rawValue": true 366 | }, 367 | "td-headers-attr": { 368 | "score": 1, 369 | "scoreDisplayMode": "binary", 370 | "rawValue": true 371 | }, 372 | "th-has-data-cells": { 373 | "score": null, 374 | "scoreDisplayMode": "not-applicable", 375 | "rawValue": true 376 | }, 377 | "valid-lang": { 378 | "score": null, 379 | "scoreDisplayMode": "not-applicable", 380 | "rawValue": true 381 | }, 382 | "video-caption": { 383 | "score": null, 384 | "scoreDisplayMode": "not-applicable", 385 | "rawValue": true 386 | }, 387 | "video-description": { 388 | "score": null, 389 | "scoreDisplayMode": "not-applicable", 390 | "rawValue": true 391 | }, 392 | "custom-controls-labels": { 393 | "score": null, 394 | "scoreDisplayMode": "manual", 395 | "rawValue": false 396 | }, 397 | "custom-controls-roles": { 398 | "score": null, 399 | "scoreDisplayMode": "manual", 400 | "rawValue": false 401 | }, 402 | "focus-traps": { 403 | "score": null, 404 | "scoreDisplayMode": "manual", 405 | "rawValue": false 406 | }, 407 | "focusable-controls": { 408 | "score": null, 409 | "scoreDisplayMode": "manual", 410 | "rawValue": false 411 | }, 412 | "heading-levels": { 413 | "score": null, 414 | "scoreDisplayMode": "manual", 415 | "rawValue": false 416 | }, 417 | "interactive-element-affordance": { 418 | "score": null, 419 | "scoreDisplayMode": "manual", 420 | "rawValue": false 421 | }, 422 | "logical-tab-order": { 423 | "score": null, 424 | "scoreDisplayMode": "manual", 425 | "rawValue": false 426 | }, 427 | "managed-focus": { 428 | "score": null, 429 | "scoreDisplayMode": "manual", 430 | "rawValue": false 431 | }, 432 | "offscreen-content-hidden": { 433 | "score": null, 434 | "scoreDisplayMode": "manual", 435 | "rawValue": false 436 | }, 437 | "use-landmarks": { 438 | "score": null, 439 | "scoreDisplayMode": "manual", 440 | "rawValue": false 441 | }, 442 | "visual-order-follows-dom": { 443 | "score": null, 444 | "scoreDisplayMode": "manual", 445 | "rawValue": false 446 | }, 447 | "uses-long-cache-ttl": { 448 | "score": 0.65, 449 | "scoreDisplayMode": "numeric", 450 | "rawValue": 84090.31071927375 451 | }, 452 | "total-byte-weight": { 453 | "score": 0.89, 454 | "scoreDisplayMode": "numeric", 455 | "rawValue": 2797459 456 | }, 457 | "offscreen-images": { 458 | "score": 0.3, 459 | "scoreDisplayMode": "numeric", 460 | "rawValue": 2470, 461 | "warnings": [] 462 | }, 463 | "render-blocking-resources": { 464 | "score": 0.7, 465 | "scoreDisplayMode": "numeric", 466 | "rawValue": 384 467 | }, 468 | "unminified-css": { 469 | "score": 1, 470 | "scoreDisplayMode": "numeric", 471 | "rawValue": 0 472 | }, 473 | "unminified-javascript": { 474 | "score": 1, 475 | "scoreDisplayMode": "numeric", 476 | "rawValue": 0, 477 | "warnings": [] 478 | }, 479 | "unused-css-rules": { 480 | "score": 0.67, 481 | "scoreDisplayMode": "numeric", 482 | "rawValue": 440 483 | }, 484 | "uses-webp-images": { 485 | "score": 0.46, 486 | "scoreDisplayMode": "numeric", 487 | "rawValue": 1100, 488 | "warnings": [] 489 | }, 490 | "uses-optimized-images": { 491 | "score": 1, 492 | "scoreDisplayMode": "numeric", 493 | "rawValue": 0, 494 | "warnings": [] 495 | }, 496 | "uses-text-compression": { 497 | "score": 1, 498 | "scoreDisplayMode": "numeric", 499 | "rawValue": 0 500 | }, 501 | "uses-responsive-images": { 502 | "score": 0.82, 503 | "scoreDisplayMode": "numeric", 504 | "rawValue": 220, 505 | "warnings": [] 506 | }, 507 | "efficient-animated-content": { 508 | "score": 1, 509 | "scoreDisplayMode": "numeric", 510 | "rawValue": 0 511 | }, 512 | "appcache-manifest": { 513 | "score": 1, 514 | "scoreDisplayMode": "binary", 515 | "rawValue": true 516 | }, 517 | "doctype": { 518 | "score": 1, 519 | "scoreDisplayMode": "binary", 520 | "rawValue": true 521 | }, 522 | "dom-size": { 523 | "score": 0.84, 524 | "scoreDisplayMode": "numeric", 525 | "rawValue": 916 526 | }, 527 | "external-anchors-use-rel-noopener": { 528 | "score": 0, 529 | "scoreDisplayMode": "binary", 530 | "rawValue": false, 531 | "warnings": [ 532 | "Unable to determine the destination for anchor (). If not used as a hyperlink, consider removing target=_blank." 533 | ] 534 | }, 535 | "geolocation-on-start": { 536 | "score": 1, 537 | "scoreDisplayMode": "binary", 538 | "rawValue": true 539 | }, 540 | "no-document-write": { 541 | "score": 1, 542 | "scoreDisplayMode": "binary", 543 | "rawValue": true 544 | }, 545 | "no-vulnerable-libraries": { 546 | "score": 1, 547 | "scoreDisplayMode": "binary", 548 | "rawValue": true 549 | }, 550 | "js-libraries": { 551 | "score": 1, 552 | "scoreDisplayMode": "binary", 553 | "rawValue": true 554 | }, 555 | "no-websql": { 556 | "score": 1, 557 | "scoreDisplayMode": "binary", 558 | "rawValue": true 559 | }, 560 | "notification-on-start": { 561 | "score": 1, 562 | "scoreDisplayMode": "binary", 563 | "rawValue": true 564 | }, 565 | "password-inputs-can-be-pasted-into": { 566 | "score": 1, 567 | "scoreDisplayMode": "binary", 568 | "rawValue": true 569 | }, 570 | "uses-http2": { 571 | "score": 0, 572 | "scoreDisplayMode": "binary", 573 | "rawValue": false 574 | }, 575 | "uses-passive-event-listeners": { 576 | "score": 0, 577 | "scoreDisplayMode": "binary", 578 | "rawValue": false 579 | }, 580 | "meta-description": { 581 | "score": 0, 582 | "scoreDisplayMode": "binary", 583 | "rawValue": false 584 | }, 585 | "http-status-code": { 586 | "score": 1, 587 | "scoreDisplayMode": "binary", 588 | "rawValue": true 589 | }, 590 | "font-size": { 591 | "score": 1, 592 | "scoreDisplayMode": "binary", 593 | "rawValue": true 594 | }, 595 | "link-text": { 596 | "score": 1, 597 | "scoreDisplayMode": "binary", 598 | "rawValue": true 599 | }, 600 | "is-crawlable": { 601 | "score": 1, 602 | "scoreDisplayMode": "binary", 603 | "rawValue": true 604 | }, 605 | "robots-txt": { 606 | "score": 1, 607 | "scoreDisplayMode": "binary", 608 | "rawValue": true 609 | }, 610 | "hreflang": { 611 | "score": 1, 612 | "scoreDisplayMode": "binary", 613 | "rawValue": true 614 | }, 615 | "plugins": { 616 | "score": 1, 617 | "scoreDisplayMode": "binary", 618 | "rawValue": true 619 | }, 620 | "canonical": { 621 | "score": null, 622 | "scoreDisplayMode": "not-applicable", 623 | "rawValue": true 624 | }, 625 | "mobile-friendly": { 626 | "score": null, 627 | "scoreDisplayMode": "manual", 628 | "rawValue": false 629 | }, 630 | "structured-data": { 631 | "score": null, 632 | "scoreDisplayMode": "manual", 633 | "rawValue": false 634 | } 635 | }, 636 | "configSettings": { 637 | "output": "json", 638 | "maxWaitForLoad": 45000, 639 | "throttlingMethod": "simulate", 640 | "throttling": { 641 | "rttMs": 150, 642 | "throughputKbps": 1638.4, 643 | "requestLatencyMs": 562.5, 644 | "downloadThroughputKbps": 1474.5600000000002, 645 | "uploadThroughputKbps": 675, 646 | "cpuSlowdownMultiplier": 4 647 | }, 648 | "auditMode": false, 649 | "gatherMode": false, 650 | "disableStorageReset": false, 651 | "disableDeviceEmulation": false, 652 | "emulatedFormFactor": "mobile", 653 | "locale": "en-US", 654 | "blockedUrlPatterns": null, 655 | "additionalTraceCategories": null, 656 | "extraHeaders": {}, 657 | "onlyAudits": null, 658 | "onlyCategories": null, 659 | "skipAudits": null 660 | }, 661 | "categories": { 662 | "performance": { 663 | "title": "Performance", 664 | "id": "performance", 665 | "score": 0.29 666 | }, 667 | "pwa": { 668 | "title": "Progressive Web App", 669 | "description": "These checks validate the aspects of a Progressive Web App, as specified by the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist).", 670 | "manualDescription": "These checks are required by the baseline [PWA Checklist](https://developers.google.com/web/progressive-web-apps/checklist) but are not automatically checked by Lighthouse. They do not affect your score but it's important that you verify them manually.", 671 | "id": "pwa", 672 | "score": 0.31 673 | }, 674 | "accessibility": { 675 | "title": "Accessibility", 676 | "description": "These checks highlight opportunities to [improve the accessibility of your web app](https://developers.google.com/web/fundamentals/accessibility). Only a subset of accessibility issues can be automatically detected so manual testing is also encouraged.", 677 | "manualDescription": "These items address areas which an automated testing tool cannot cover. Learn more in our guide on [conducting an accessibility review](https://developers.google.com/web/fundamentals/accessibility/how-to-review).", 678 | "id": "accessibility", 679 | "score": 0.7 680 | }, 681 | "best-practices": { 682 | "title": "Best Practices", 683 | "id": "best-practices", 684 | "score": 0.73 685 | }, 686 | "seo": { 687 | "title": "SEO", 688 | "description": "These checks ensure that your page is optimized for search engine results ranking. There are additional factors Lighthouse does not check that may affect your search ranking. [Learn more](https://support.google.com/webmasters/answer/35769).", 689 | "manualDescription": "Run these additional validators on your site to check additional SEO best practices.", 690 | "id": "seo", 691 | "score": 0.9 692 | } 693 | }, 694 | "categoryGroups": null, 695 | "timing": { 696 | "total": 13166 697 | }, 698 | "i18n": null 699 | } --------------------------------------------------------------------------------