├── .dockerignore ├── .gitignore ├── API.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── SETUP.md ├── docker-compose.yml └── src ├── .eslintrc.json ├── app.js ├── client ├── app.js ├── components │ ├── audit-form.js │ ├── header.js │ └── tasks-panel.js ├── index.html └── style.css ├── config.performance.js ├── lighthouse-audit.js ├── package.json ├── server.js ├── test └── test-api-utils.js └── utils ├── api.js ├── bq.js ├── tasks.js └── validate.js /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | .vscode 4 | .DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | node_modules/ 3 | package-lock.json 4 | .DS_Store -------------------------------------------------------------------------------- /API.md: -------------------------------------------------------------------------------- 1 | ## Using the service API 2 | 3 | ### /audit (POST) 4 | 5 | Runs one or more audits sequentially, utilizing a shared puppeteer instance between tests. Logs results to the configured BQ dataset table. 6 | 7 | **Parameters** 8 | 9 | | Name | Type | Optional | Description 10 | | ------------- | ------------- | ------------- | ------------- | 11 | | urls | Array | | List of urls to run a lighthouse audit on | 12 | | blockedRequests | Array | Yes | List of requests to block on each audit e.g. 3rd party tag origins | 13 | 14 | **Example** 15 | 16 | ``` 17 | curl -X POST \ 18 | http://127.0.0.1:8080/audit \ 19 | -H 'content-type: application/json' \ 20 | -d '{ 21 | "urls": [ 22 | "https://www.exampleurl1.com", 23 | "https://www.exampleurl1.com", 24 | ... 25 | ], 26 | "blockedRequests": [ 27 | "https://www.someblockedrequestdomain.com" 28 | ] 29 | }' 30 | ``` 31 | 32 | ### /audit-async (POST) 33 | 34 | Schedules one or more audits to run asynchronously, utilizing [Cloud Tasks](https://cloud.google.com/tasks). Each dispatched task calls `/audit` as a target to run and log the test. 35 | 36 | | Name | Type | Optional | Description 37 | | ------------- | ------------- | ------------- | ------------- | 38 | | urls | Array | | List of urls to run a lighthouse audit on | 39 | | blockedRequests | Array | Yes | List of requests to block on each audit e.g. 3rd party tag origins | 40 | 41 | **Example** 42 | 43 | ``` 44 | curl -X POST \ 45 | http://127.0.0.1:8080/audit-async \ 46 | -H 'content-type: application/json' \ 47 | -d '{ 48 | "urls": [ 49 | "https://www.exampleurl1.com", 50 | "https://www.exampleurl1.com", 51 | ... 52 | ], 53 | "blockedRequests": [ 54 | "https://www.someblockedrequestdomain.com" 55 | ] 56 | }' 57 | ``` 58 | 59 | ### /active-tasks (GET) 60 | 61 | Lists all the active audit tasks that are in queue along with the payload and status. Applies pagination to results. 62 | 63 | | Name | Type | Optional | Description 64 | | ------------- | ------------- | ------------- | ------------- | 65 | | pageSize | Number | Yes | Number of items to return per page | 66 | | nextPageToken | String | Yes | Token to access results in the next page | 67 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement (CLA). You (or your employer) retain the copyright to your 10 | contribution; this simply gives us permission to use and redistribute your 11 | contributions as part of the project. Head over to 12 | to see your current agreements on file or 13 | to sign a new one. 14 | 15 | You generally only need to submit a CLA once, so if you've already submitted one 16 | (even if it was for a different project), you probably don't need to do it 17 | again. 18 | 19 | ## Code reviews 20 | 21 | All submissions, including submissions by project members, require review. We 22 | use GitHub pull requests for this purpose. Consult 23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 24 | information on using pull requests. 25 | 26 | ## Community Guidelines 27 | 28 | This project follows 29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM buildkite/puppeteer 2 | WORKDIR /app 3 | COPY ./src /app 4 | RUN npm install 5 | EXPOSE 8080 6 | CMD ["node", "server.js"] 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lasso - Lighthouse as a service 2 | 3 | > An API service built on top of [lighthouse](https://github.com/GoogleChrome/lighthouse#readme) to automate running Lighthouse tests on large number of URLs in parallel. Utilizes [Cloud Run](https://cloud.google.com/run) and [Cloud Tasks](https://cloud.google.com/tasks) to distribute and run multiple tests across multiple containers, outputs results to a [BigQuery](https://cloud.google.com/bigquery) dataset. 4 | 5 | ## Features 6 | 7 | ✅ Bulk test 100s of URLs with lighthouse asynchronously 8 | 9 | ✅ Writes test results to a date partitioned BigQuery table 10 | 11 | ✅ Specify which resource requests to block from tests e.g. For running tests excluding 3rd party scripts or libraries 12 | 13 | ## Getting started 14 | 15 | - [Setup and deployment guide](SETUP.md) 16 | - [API Service usage](API.md) 17 | 18 | ## Disclaimer 19 | This is not an officially supported Google product. 20 | -------------------------------------------------------------------------------- /SETUP.md: -------------------------------------------------------------------------------- 1 | ## Setup 2 | 3 | ### Deploying to Cloud Run 4 | 5 | Create a build from the Docker file and create a new tag 6 | 7 | `docker build -t lasso .` 8 | 9 | Tag the built image for GCR targeting the cloud project 10 | 11 | `docker tag lasso gcr.io/[CLOUD-PROJECT-ID]/lasso:[TAG]` 12 | 13 | Push to GCR (Make sure the GCR API is [enabled](https://console.cloud.google.com/apis/api/containerregistry.googleapis.com/overview) already) 14 | 15 | `docker push gcr.io/[CLOUD-PROJECT-ID]/lasso:[TAG]` 16 | 17 | To learn more about configuring a Cloud Run service, you can follow [this guide](https://cloud.google.com/run/docs/deploying). 18 | 19 | The following ENV variables will need to be configured on cloud run: 20 | 21 | | ENV var | Description | 22 | | ------------- | ------------- | 23 | | BQ_DATASET | The name of a BigQuery dataset containing your results table | 24 | | BQ_TABLE | The name of the BQ table to output results to | 25 | | GOOGLE_CLOUD_PROJECT | ID of your cloud project | 26 | | CLOUD_TASKS_QUEUE | Name of your cloud tasks queue | 27 | | CLOUD_TASKS_QUEUE_LOCATION | Location of the cloud task queue | 28 | | SERVICE_URL | base url and protocol of the deployed service on cloud run | 29 | 30 | 31 | ### Running Locally via docker compose 32 | 33 | #### Setting Environment Variables 34 | 35 | Set the path for the Google Cloud credentials of the project as an ENV variable in your system on `GCP_KEY_PATH`. This value will be picked up by the docker compose config mapped to a volume and set on the running container. The following ENV variables will need to be configured in docker-compose.yml: 36 | 37 | | ENV var | Description | 38 | | ------------- | ------------- | 39 | | BQ_DATASET | The name of a BigQuery dataset containing your results table | 40 | | BQ_TABLE | The name of the BQ table to output results to | 41 | | GOOGLE_CLOUD_PROJECT | ID of your cloud project | 42 | | CLOUD_TASKS_QUEUE | Name of your cloud tasks queue | 43 | | CLOUD_TASKS_QUEUE_LOCATION | Location of the cloud task queue | 44 | | SERVICE_URL | base url and protocol of the deployed service on cloud run | 45 | 46 | #### Run 47 | 48 | For local development, you can choose to run the project via [docker compose](https://cloud.google.com/community/tutorials/cloud-run-local-dev-docker-compose). Running `docker-compose up` launches the service. 49 | 50 | ### Running the image directly locally or on a server 51 | 52 | ``` 53 | PORT=8080 && docker run \ 54 | -p 8080:${PORT} \ 55 | -e PORT=${PORT} \ 56 | -e BQ_DATASET=lh_results \ 57 | -e BQ_TABLE=psmetrics \ 58 | -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/keys/lighthouse-service-09c62b8cd84e.json \ 59 | -v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/keys/lighthouse-service-09c62b8cd84e.json:ro \ 60 | gcr.io/lighthouse-service/pagespeed-metrics 61 | ``` -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | lhservice: 5 | image: buildkite/puppeteer:latest 6 | volumes: 7 | - ./src:/app 8 | - ${GCP_KEY_PATH}:/tmp/keys/gcp-keyfile.json:ro 9 | restart: always 10 | working_dir: /app 11 | command: sh -c "npm install && npm start" 12 | ports: 13 | - "8080:${PORT:-8080}" 14 | environment: 15 | PORT: ${PORT:-8080} 16 | K_SERVICE: lasso 17 | K_REVISION: 0 18 | K_CONFIGURATION: lasso 19 | BQ_DATASET: lh_results 20 | BQ_TABLE: lh_set 21 | GOOGLE_APPLICATION_CREDENTIALS: /tmp/keys/gcp-keyfile.json 22 | GOOGLE_CLOUD_PROJECT: 'lighthouse-service' 23 | CLOUD_TASKS_QUEUE: 'lhtests' 24 | CLOUD_TASKS_QUEUE_LOCATION: 'europe-west1' 25 | SERVICE_URL: ${SERVICE_URL} 26 | -------------------------------------------------------------------------------- /src/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true, 4 | "es6": true 5 | }, 6 | "parserOptions": { 7 | "ecmaVersion": 2017, 8 | "sourceType": "module" 9 | }, 10 | "extends": ["eslint:recommended", "google"] 11 | } -------------------------------------------------------------------------------- /src/app.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | 'use strict'; 18 | 19 | const path = require('path'); 20 | const express = require('express'); 21 | const perfConfig = require('./config.performance.js'); 22 | const {LighthouseAudit} = require('./lighthouse-audit'); 23 | const {CloudTasksClient} = require('@google-cloud/tasks'); 24 | const {writeResultStream} = require('./utils/bq'); 25 | const {getChunkedList, isURL} = require('./utils/api'); 26 | const {listActiveTasks, processTaskResults} = require('./utils/tasks'); 27 | const { 28 | auditRequestValidation, 29 | asyncAuditRequestValidation, 30 | activeTasksRequestValidation} = require('./utils/validate'); 31 | 32 | const app = express(); 33 | 34 | app.use(express.raw()); 35 | app.use(express.json({limit: '5mb', extended: true})); 36 | app.use(express.urlencoded({limit: '5mb', extended: true})); 37 | 38 | app.use(express.static('client')); 39 | 40 | app.get('/', function(req, res) { 41 | res.sendFile(path.join(__dirname + '/client/index.html')); 42 | }); 43 | 44 | app.post('/audit', auditRequestValidation, performAudit); 45 | app.post('/audit-async', asyncAuditRequestValidation, scheduleAudits); 46 | app.get('/active-tasks', activeTasksRequestValidation, getActiveTasks); 47 | 48 | /** 49 | * Performs a lighthouse audit on a set of URLs supplied in the payload 50 | * @param {Object} req 51 | * @param {Object} res 52 | */ 53 | async function performAudit(req, res) { 54 | const BQ_DATASET = process.env.BQ_DATASET; 55 | const BQ_TABLE = process.env.BQ_TABLE; 56 | const payload = req.body; 57 | 58 | const lhAudit = new LighthouseAudit( 59 | payload.urls, 60 | payload.blockedRequests, 61 | perfConfig.auditConfig, 62 | perfConfig.auditResultsMapping); 63 | 64 | try { 65 | await lhAudit.run(); 66 | const results = lhAudit.getBQFormatResults(); 67 | await writeResultStream(BQ_DATASET, BQ_TABLE, results); 68 | 69 | return res.json(results); 70 | } catch (e) { 71 | res.status(500); 72 | return res.json({ 73 | 'error': { 74 | 'code': 500, 75 | 'message': e.message, 76 | }, 77 | }); 78 | } 79 | } 80 | 81 | /** 82 | * Divides a set of URLs in the payload to equal sized chunks and push 83 | * them into cloud tasks to run be run as async. The cloud tasks use 84 | * the /audit endpoint to run each smaller set of audits. 85 | * @param {Object} req 86 | * @param {Object} res 87 | * @return {JSON} 88 | */ 89 | async function scheduleAudits(req, res) { 90 | const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT; 91 | const CLOUD_TASKS_QUEUE = process.env.CLOUD_TASKS_QUEUE; 92 | const CLOUD_TASKS_QUEUE_LOCATION = process.env.CLOUD_TASKS_QUEUE_LOCATION; 93 | const SERVICE_URL = process.env.SERVICE_URL; 94 | 95 | const chunks = getChunkedList(req.body.urls, 1, isURL); 96 | const tasksClient = new CloudTasksClient(); 97 | const serviceUrl = `${SERVICE_URL}/audit`; 98 | 99 | const parent = tasksClient.queuePath( 100 | GOOGLE_CLOUD_PROJECT, 101 | CLOUD_TASKS_QUEUE_LOCATION, 102 | CLOUD_TASKS_QUEUE, 103 | ); 104 | 105 | let inSeconds = 10; 106 | const createTasks = []; 107 | let blockedRequests = []; 108 | 109 | if (typeof req.body.blockedRequests != 'undefined') { 110 | blockedRequests = req.body.blockedRequests; 111 | } 112 | 113 | for (let i = 0; i < chunks.length; i++) { 114 | const payload = JSON.stringify({ 115 | 'urls': chunks[i], 116 | 'blockedRequests': blockedRequests, 117 | }); 118 | 119 | const task = { 120 | httpRequest: { 121 | httpMethod: 'POST', 122 | url: serviceUrl, 123 | headers: {'Content-Type': 'application/json'}, 124 | body: Buffer.from(payload).toString('base64'), 125 | scheduleTime: (inSeconds + (Date.now() / 1000)), 126 | }, 127 | }; 128 | 129 | inSeconds += 10; 130 | 131 | const request = {parent, task}; 132 | const [response] = await tasksClient.createTask(request); 133 | createTasks.push({ 134 | name: response.name, 135 | urls: chunks[i], 136 | }); 137 | } 138 | 139 | return res.json({'tasks': createTasks}); 140 | } 141 | 142 | /** 143 | * Lists all the tasks that are currently active 144 | * @param {Object} req 145 | * @param {Object} res 146 | * @return {JSON} 147 | */ 148 | async function getActiveTasks(req, res) { 149 | try { 150 | const tasksResults = await listActiveTasks( 151 | process.env.GOOGLE_CLOUD_PROJECT, 152 | process.env.CLOUD_TASKS_QUEUE_LOCATION, 153 | process.env.CLOUD_TASKS_QUEUE, 154 | req.query.pageSize, 155 | req.query.pageToken, 156 | ); 157 | const processedResults = processTaskResults(tasksResults[2]); 158 | return res.json(processedResults); 159 | } catch (e) { 160 | res.status(500); 161 | return res.json({ 162 | 'error': { 163 | 'code': 500, 164 | 'message': e.message, 165 | }, 166 | }); 167 | } 168 | } 169 | 170 | module.exports = app; 171 | -------------------------------------------------------------------------------- /src/client/app.js: -------------------------------------------------------------------------------- 1 | import {html, Component, render} from 'https://unpkg.com/htm/preact/standalone.module.js'; 2 | import Header from './components/header.js'; 3 | import AuditForm from './components/audit-form.js'; 4 | import TasksPanel from './components/tasks-panel.js'; 5 | 6 | /** 7 | * App 8 | */ 9 | class App extends Component { 10 | /** 11 | * Render 12 | * @return {*} 13 | */ 14 | render() { 15 | return html` 16 |
17 | <${Header} name="Lasso" /> 18 |
19 |
20 | <${AuditForm} /> 21 | <${TasksPanel} /> 22 |
23 |
24 |
`; 25 | } 26 | } 27 | 28 | render(html`<${App} page="All" />`, document.body); 29 | -------------------------------------------------------------------------------- /src/client/components/audit-form.js: -------------------------------------------------------------------------------- 1 | import {html, Component} from 'https://unpkg.com/htm/preact/standalone.module.js'; 2 | 3 | /** 4 | * AuditForm 5 | */ 6 | class AuditForm extends Component { 7 | /** 8 | * handleSubmit 9 | * @param {Object} e 10 | */ 11 | handleSubmit(e) { 12 | e.preventDefault(); 13 | 14 | const formData = new FormData(e.target); 15 | const formObject = {}; 16 | 17 | formData.forEach((value, key) => { 18 | formObject[key] = value; 19 | }); 20 | 21 | const reader = new FileReader(); 22 | 23 | reader.onload = ((e) => { 24 | formObject['urlsTxt'] = e.target.result; 25 | // TODO: Run validation on all content before calling submit request 26 | this.submitTasksRequest(formObject); 27 | }); 28 | 29 | reader.readAsText(formObject['urls']); 30 | } 31 | 32 | /** 33 | * Submits a request to the API for adding new tasks 34 | * @param {Object} formData 35 | */ 36 | submitTasksRequest(formData) { 37 | const requestObject = {}; 38 | requestObject['urls'] = formData['urlsTxt'].split('\n'); 39 | 40 | this.postData('/audit-async', requestObject) 41 | .then((data) => { 42 | console.log(data); 43 | }); 44 | } 45 | 46 | /** 47 | * postData 48 | * @param {String} url 49 | * @param {Object} data 50 | */ 51 | async postData(url, data) { 52 | const response = await fetch(url, { 53 | method: 'POST', 54 | cache: 'no-cache', 55 | credentials: 'same-origin', 56 | headers: { 57 | 'Content-Type': 'application/json', 58 | }, 59 | body: JSON.stringify(data), 60 | }); 61 | 62 | return response.json(); 63 | } 64 | 65 | /** 66 | * Render 67 | * @return {*} 68 | */ 69 | render() { 70 | return html` 71 | 105 | `; 106 | } 107 | } 108 | 109 | 110 | export default AuditForm; 111 | -------------------------------------------------------------------------------- /src/client/components/header.js: -------------------------------------------------------------------------------- 1 | import {html} from 'https://unpkg.com/htm/preact/standalone.module.js'; 2 | 3 | const Header = ({name}) => html`

${name}

`; 4 | 5 | export default Header; 6 | -------------------------------------------------------------------------------- /src/client/components/tasks-panel.js: -------------------------------------------------------------------------------- 1 | import {html, Component} from 'https://unpkg.com/htm/preact/standalone.module.js'; 2 | 3 | /** 4 | * TasksPanel 5 | */ 6 | class TasksPanel extends Component { 7 | /** 8 | * Constructor 9 | * @param {Object} props 10 | */ 11 | constructor(props) { 12 | super(props); 13 | 14 | this.state = { 15 | tasksInView: [], 16 | }; 17 | } 18 | 19 | /** 20 | * componentDidMount 21 | */ 22 | componentDidMount() { 23 | this.timer = setInterval(() => { 24 | this.getItems().then((data) => { 25 | this.setState({ 26 | tasksInView: data.tasks, 27 | }); 28 | }); 29 | }, 2000); 30 | } 31 | 32 | /** 33 | * componentWillUnmount 34 | */ 35 | componentWillUnmount() { 36 | this.timer = null; 37 | } 38 | 39 | /** 40 | * getItems 41 | */ 42 | async getItems() { 43 | const response = await fetch('/active-tasks'); 44 | return response.json(); 45 | } 46 | 47 | /** 48 | * Render 49 | * @param {Object} props 50 | * @param {Object} state 51 | * @return {*} 52 | */ 53 | render(props, state) { 54 | return html` 55 |
56 |
57 | Active Audit Tasks 58 |
59 |
60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | ${state.tasksInView.map((task) => { 72 | return html` 73 | 74 | 75 | 76 | 77 | 78 | 79 | `; 80 | })} 81 | 82 |
Task nameDispatch countResponse countFirst attemptLast attempt
${task.name}${task.dispatchCount}${task.responseCount}${task.firstAttempt}${task.lastAttempt}
83 |
84 |
85 | `; 86 | } 87 | } 88 | 89 | 90 | export default TasksPanel; 91 | -------------------------------------------------------------------------------- /src/client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Lasso 4 | 5 | 6 | 7 | Lasso 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/client/style.css: -------------------------------------------------------------------------------- 1 | html, body, div, span, applet, object, iframe, 2 | h1, h2, h3, h4, h5, h6, p, blockquote, pre, 3 | a, abbr, acronym, address, big, cite, code, 4 | del, dfn, em, img, ins, kbd, q, s, samp, 5 | small, strike, strong, sub, sup, tt, var, 6 | b, u, i, center, 7 | dl, dt, dd, ol, ul, li, 8 | fieldset, form, label, legend, 9 | table, caption, tbody, tfoot, thead, tr, th, td, 10 | article, aside, canvas, details, embed, 11 | figure, figcaption, footer, hgroup, 12 | menu, nav, output, ruby, section, summary, 13 | time, mark, audio, video { 14 | margin: 0; 15 | padding: 0; 16 | border: 0; 17 | font-size: 100%; 18 | font: inherit; 19 | vertical-align: baseline; 20 | } 21 | 22 | html,body{ 23 | height:100%; 24 | min-height:100%; 25 | line-height: 1; 26 | } 27 | 28 | body { 29 | display: flex; 30 | flex-direction: column; 31 | height: 100vh; 32 | font-family: 'Roboto', sans-serif; 33 | line-height: 1; 34 | color: rgba(0,0,0,0.87); 35 | } 36 | 37 | header { 38 | display: flex; 39 | justify-content: space-between; 40 | position: relative; 41 | display: grid; 42 | grid-gap: 1rem; 43 | color: #fff; 44 | margin: 0; 45 | padding:10px; 46 | box-sizing: border-box; 47 | display: flex; 48 | align-items: center; 49 | background-color: #fff; 50 | color: rgba(0,0,0,.54); 51 | } 52 | 53 | header h1 { 54 | font-size: 1.5rem; 55 | font-weight: bold; 56 | } 57 | 58 | .fork-btn { 59 | display: flex; 60 | align-items: center; 61 | } 62 | 63 | .fork-btn a { 64 | color: #FFF; 65 | padding:5px; 66 | text-decoration: none; 67 | } 68 | 69 | .main-wrapper { 70 | display: grid; 71 | grid-template-rows: fit-content(10%) fit-content(10%) auto; 72 | flex-grow: 1; 73 | height: 100vh; 74 | } 75 | 76 | .panels-wrapper { 77 | display: grid; 78 | grid-template-columns: 30% 70%; 79 | height: 100%; 80 | min-height: 80vh; 81 | } 82 | 83 | .panel-content { 84 | padding: 2em; 85 | } 86 | 87 | .c-input { 88 | border-right: 1px solid #bebebe; 89 | color: #000; 90 | background-color: #FFF; 91 | } 92 | 93 | .c-output { 94 | padding: 0; 95 | color: #000; 96 | background-color: #FFF; 97 | } 98 | 99 | .toolbar { 100 | color: #fff; 101 | box-sizing: border-box; 102 | padding: 10px; 103 | background: #4285f4; 104 | text-align: center; 105 | } 106 | 107 | form input[type], form select, form textarea { 108 | margin-bottom: 1em; 109 | } 110 | 111 | input[type=email], input[type=month], input[type=number], input[type=password], input[type=search], input[type=tel], input[type=text], input[type=time], input[type=url], input[type=week], input[type^=date] { 112 | outline: 0; 113 | box-sizing: border-box; 114 | height: 2em; 115 | margin: 0; 116 | padding: calc(.25em - 1px) .5em; 117 | font-family: inherit; 118 | font-size: 1em; 119 | border: 1px solid #aaa; 120 | border-radius: 2px; 121 | background: #fff; 122 | color: #000; 123 | display: block; 124 | width: 100%; 125 | line-height: calc(2em - 1px*2 - (.25em - 1px)*2); 126 | -webkit-appearance: none; 127 | -moz-appearance: none; 128 | appearance: none; 129 | } 130 | 131 | input[type=button], input[type=reset], input[type=submit] { 132 | outline: 0; 133 | box-sizing: border-box; 134 | height: 2em; 135 | margin: 0; 136 | padding: calc(.25em - 1px) .5em; 137 | font-family: inherit; 138 | font-size: 1em; 139 | border: 1px solid #aaa; 140 | border-radius: 2px; 141 | display: inline-block; 142 | width: auto; 143 | background-color: #3367d6; 144 | box-shadow: none; 145 | color: #FFF; 146 | cursor: pointer; 147 | -webkit-appearance: none; 148 | -moz-appearance: none; 149 | appearance: none; 150 | } 151 | 152 | input[type=file] { 153 | outline: 0; 154 | box-sizing: border-box; 155 | margin: 0; 156 | padding: calc(.25em - 1px) .5em; 157 | font-family: inherit; 158 | border: 1px solid #aaa; 159 | border-radius: 2px; 160 | background: #fff; 161 | background: #f2f2f2; 162 | color: #000; 163 | cursor: pointer; 164 | display: block; 165 | width: 100%; 166 | height: auto; 167 | padding: .75em .5em; 168 | font-size: 12px; 169 | line-height: 1; 170 | } 171 | 172 | table { 173 | display: inline-block; 174 | border-spacing: 0; 175 | border-collapse: collapse; 176 | overflow-x: auto; 177 | max-width: 100%; 178 | text-align: left; 179 | vertical-align: top; 180 | background: linear-gradient(rgba(0,0,0,.15) 0%,rgba(0,0,0,.15) 100%) 0 0,linear-gradient(rgba(0,0,0,.15) 0%,rgba(0,0,0,.15) 100%) 100% 0; 181 | background-attachment: scroll,scroll; 182 | background-size: 1px 100%,1px 100%; 183 | background-repeat: no-repeat,no-repeat; 184 | } 185 | 186 | table td:first-child, table th:first-child { 187 | padding-left: 0; 188 | background-image: linear-gradient(to right,#fff 50%,rgba(255,255,255,0) 100%); 189 | background-size: 2px 100%; 190 | background-repeat: no-repeat; 191 | } 192 | 193 | table td, table th { 194 | padding: .35em .75em; 195 | vertical-align: top; 196 | font-size: .9em; 197 | border: 1px solid #f2f2f2; 198 | border-top: 0; 199 | border-left: 0; 200 | } 201 | 202 | table th { 203 | line-height: 1.2; 204 | } 205 | 206 | textarea { 207 | outline: 0; 208 | box-sizing: border-box; 209 | margin: 0; 210 | padding: calc(.25em - 1px) .5em; 211 | font-family: inherit; 212 | font-size: 1em; 213 | border: 1px solid #aaa; 214 | border-radius: 2px; 215 | background: #fff; 216 | color: #000; 217 | display: block; 218 | width: 100%; 219 | line-height: calc(2em - 1px*2 - (.25em - 1px)*2); 220 | -webkit-appearance: none; 221 | -moz-appearance: none; 222 | appearance: none; 223 | height: 4.5em; 224 | resize: vertical; 225 | padding-top: .5em; 226 | padding-bottom: .5em; 227 | } 228 | 229 | button:focus, input[type=email]:focus, input[type=month]:focus, input[type=number]:focus, input[type=password]:focus, input[type=search]:focus, input[type=tel]:focus, input[type=text]:focus, input[type=time]:focus, input[type=url]:focus, input[type=week]:focus, input[type^=date]:focus, select:focus, textarea:focus { 230 | border: 1px solid #4285f4; 231 | } 232 | 233 | details, nav, p { 234 | margin: 1em 0; 235 | } 236 | 237 | form label:not(:first-child) { 238 | margin-top: 1em; 239 | } 240 | form label, form select, output { 241 | display: block; 242 | } 243 | -------------------------------------------------------------------------------- /src/config.performance.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | module.exports = { 18 | auditConfig: { 19 | extends: 'lighthouse:default', 20 | settings: { 21 | onlyAudits: [ 22 | 'first-contentful-paint', 23 | 'largest-contentful-paint', 24 | 'first-meaningful-paint', 25 | 'speed-index', 26 | 'estimated-input-latency', 27 | 'total-blocking-time', 28 | 'max-potential-fid', 29 | 'cumulative-layout-shift', 30 | 'first-cpu-idle', 31 | 'interactive', 32 | 'server-response-time', 33 | ], 34 | }, 35 | }, 36 | auditResultsMapping: { 37 | 'firstContentfulPaint': 'first-contentful-paint', 38 | 'largestContentfulPaint': 'largest-contentful-paint', 39 | 'firstMeaningfulPaint': 'first-meaningful-paint', 40 | 'speedIndex': 'speed-index', 41 | 'estimatedInputLatency': 'estimated-input-latency', 42 | 'totalBlockingTime': 'total-blocking-time', 43 | 'maxPotentialFID': 'max-potential-fid', 44 | 'cumulativeLayoutShift': 'cumulative-layout-shift', 45 | 'firstCpuIdle': 'first-cpu-idle', 46 | 'interactive': 'interactive', 47 | 'serverResponseTime': 'server-response-time', 48 | }, 49 | bqAuditTableSchema: 50 | `url:string, 51 | date:date, 52 | firstContentfulPaint:float, 53 | largestContentfulPaint:float, 54 | firstMeaningfulPaint:float, 55 | speedIndex:float, 56 | estimatedInputLatency:float, 57 | totalBlockingTime:float, 58 | maxPotentialFid:float, 59 | cumulativeLayoutShift:float, 60 | firstCpuIdle:float, 61 | interactive:float, 62 | serverResponseTime:float, 63 | blockedRequests:string`, 64 | }; 65 | -------------------------------------------------------------------------------- /src/lighthouse-audit.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | 'use strict'; 18 | 19 | const puppeteer = require('puppeteer'); 20 | const lighthouse = require('lighthouse'); 21 | const {BigQuery} = require('@google-cloud/bigquery'); 22 | 23 | /** 24 | * Lighthouse Audit 25 | */ 26 | class LighthouseAudit { 27 | /** 28 | * Constructor 29 | * @param {Array} urls 30 | * @param {Array} blockedRequestPatterns 31 | * @param {Object} auditConfig 32 | * @param {Object} auditFieldMapping 33 | */ 34 | constructor(urls, blockedRequestPatterns = [], auditConfig, auditFieldMapping) { 35 | this.urls = urls; 36 | this.blockedRequestPatterns = blockedRequestPatterns; 37 | this.auditConfig = auditConfig; 38 | this.auditResults = []; 39 | this.auditFieldMapping = auditFieldMapping; 40 | } 41 | 42 | /** 43 | * Initializes a new puppeteer instance and triggers a set of 44 | * LH audits to run sequentially 45 | * @return {Array} 46 | */ 47 | async run() { 48 | const browser = await puppeteer.launch({ 49 | headless: true, 50 | args: ['--no-sandbox'], 51 | }); 52 | 53 | for (let i = 0; i < this.urls.length; i++) { 54 | const url = this.urls[i]; 55 | 56 | try { 57 | const page = await browser.newPage(); 58 | const metrics = await this.performAudit(url, page, this.blockedRequestPatterns); 59 | this.auditResults.push(metrics); 60 | } catch (e) { 61 | throw new Error(`${e.message} on ${url}`); 62 | } 63 | } 64 | 65 | return this.auditResults; 66 | } 67 | 68 | /** 69 | * Runs a lighthouse performance audit on specific page in a chrome instance 70 | * @param {string} url 71 | * @param {Object} page 72 | * @param {Array} blockedUrlPatterns 73 | * @return {Promise} 74 | */ 75 | async performAudit(url, page, blockedUrlPatterns) { 76 | const port = page.browser().wsEndpoint().split(':')[2].split('/')[0]; 77 | const options = { 78 | blockedUrlPatterns, 79 | port, 80 | }; 81 | 82 | return await lighthouse(url, options, this.auditConfig) 83 | .then((metrics) => { 84 | const audits = metrics.lhr.audits; 85 | 86 | if (typeof(audits) != 'undefined' && audits != null) { 87 | audits['url'] = url; 88 | return audits; 89 | } 90 | }).catch((e) => { 91 | throw new Error(`LH Audit error: ${e.message}`); 92 | }); 93 | } 94 | 95 | /** 96 | * Returns the instance's audit results, properly formatted for 97 | * inserting into BigQuery. Columns selected are based on the 98 | * auditFileMapping configuration supplied in the constructor. 99 | * @return {Array} 100 | */ 101 | getBQFormatResults() { 102 | const today = new Date().toJSON().slice(0, 10); 103 | return this.auditResults.map((audit) => { 104 | if (typeof (audit) != 'undefined') { 105 | const formattedAudit = Object.entries(this.auditFieldMapping). 106 | reduce((res, keyVal) => { 107 | res[keyVal[0]] = audit[keyVal[1]].numericValue; 108 | return res; 109 | }, {}); 110 | 111 | formattedAudit['date'] = BigQuery.date(today); 112 | formattedAudit['url'] = audit.url; 113 | formattedAudit['blockedRequests'] = this.blockedRequestPatterns.join(','); 114 | 115 | return formattedAudit; 116 | } 117 | }); 118 | } 119 | 120 | /** 121 | * @return {Array} 122 | */ 123 | getRawResults() { 124 | return this.auditResults; 125 | } 126 | } 127 | 128 | module.exports = { 129 | LighthouseAudit, 130 | }; 131 | -------------------------------------------------------------------------------- /src/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Lasso", 3 | "version": "1.0.0", 4 | "description": "Automate running Lighthouse tests on large number of URLs in parallel", 5 | "main": "main.js", 6 | "scripts": { 7 | "test": "mocha", 8 | "start": "nodemon server.js" 9 | }, 10 | "author": "Google Inc", 11 | "license": "Apache-2.0", 12 | "dependencies": { 13 | "@google-cloud/bigquery": "^4.7.0", 14 | "@google-cloud/tasks": "^2.1.0", 15 | "express": "^4.17.1", 16 | "joi": "^17.2.0", 17 | "lighthouse": "^6.0.0" 18 | }, 19 | "devDependencies": { 20 | "chai": "^4.2.0", 21 | "eslint": "^6.8.0", 22 | "eslint-config-google": "^0.14.0", 23 | "mocha": "^8.0.1", 24 | "nodemon": "^2.0.2" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/server.js: -------------------------------------------------------------------------------- 1 | const app = require('./app'); 2 | 3 | const PORT = process.env.PORT || 8080; 4 | const HOST = '0.0.0.0'; 5 | 6 | app.listen(PORT, HOST); 7 | console.log(`Running on http://${HOST}:${PORT}`); 8 | -------------------------------------------------------------------------------- /src/test/test-api-utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | const assert = require('chai').assert; 18 | const suite = require('mocha').suite; 19 | const test = require('mocha').test; 20 | const {getChunkedList, validateAuditRequest} = require('../utils/api'); 21 | 22 | 23 | suite('getChunkedList tests', () => { 24 | test('Returns an empty list if the input is empty', () => { 25 | const expected = []; 26 | const result = getChunkedList([], 3); 27 | 28 | assert.deepEqual(result, expected); 29 | }); 30 | 31 | test('Return single chunk if input less or equal than input chunk size', () => { 32 | const expected = [['one', 'two', 'three']]; 33 | const result = getChunkedList(['one', 'two', 'three'], 3); 34 | assert.deepEqual(result, expected); 35 | }); 36 | 37 | test('Return multiple chunks when input greater than chunk size', () => { 38 | const expected = [['one', 'two', 'three'], ['four', 'five']]; 39 | const result = getChunkedList(['one', 'two', 'three', 'four', 'five'], 3); 40 | assert.deepEqual(result, expected); 41 | }); 42 | 43 | test('Filter out values in the input list based on a filter callback', () => { 44 | const isEvenNum = (x) => x % 2 !== 0; 45 | const expected = [[1, 3, 5], [7, 9]]; 46 | const result = getChunkedList([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3, isEvenNum); 47 | 48 | assert.deepEqual(result, expected); 49 | }); 50 | }); 51 | 52 | suite('validateAuditSchedule', () => { 53 | test('Return error when URL list is not present', () => { 54 | const expected = { 55 | valid: false, 56 | errorMessage: 'URL list not available or empty', 57 | }; 58 | const requestPayload = {'potato': ['one']}; 59 | 60 | const result = validateAuditRequest(requestPayload, 10); 61 | assert.deepEqual(result, expected); 62 | }); 63 | 64 | test('Return error when URL list is empty', () => { 65 | const expected = { 66 | valid: false, 67 | errorMessage: 'URL list not available or empty', 68 | }; 69 | const requestPayload = {'urls': []}; 70 | const result = validateAuditRequest(requestPayload, 10); 71 | assert.deepEqual(result, expected); 72 | }); 73 | 74 | test('Return error when URL list count is greater than a threshold', () => { 75 | const maxEntries = '4'; 76 | const expected = { 77 | valid: false, 78 | errorMessage: `URL list should not exceed ${maxEntries}`, 79 | }; 80 | 81 | const requestPayload = {'urls': [ 82 | 'http://one.com', 83 | 'http://two.com', 84 | 'http://three.com', 85 | 'http://four.com', 86 | 'http://five.com', 87 | ]}; 88 | 89 | const result = validateAuditRequest(requestPayload, maxEntries); 90 | assert.deepEqual(result, expected); 91 | }); 92 | 93 | test('Return success object when all required tests pass', () => { 94 | const expected = { 95 | valid: true, 96 | }; 97 | 98 | const requestPayload = {'urls': [ 99 | 'http://two.com', 100 | 'http://three.com', 101 | 'http://four.com', 102 | 'http://five.com', 103 | ]}; 104 | 105 | const result = validateAuditRequest(requestPayload, 4); 106 | assert.deepEqual(result, expected); 107 | }); 108 | }); 109 | -------------------------------------------------------------------------------- /src/utils/api.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | /** 18 | * Transforms an array into multiple chunks of a specific size 19 | * @param {Array} inputList 20 | * @param {Integer} chunkSize 21 | * @param {Function} filterCallback 22 | * @return {Array} 23 | */ 24 | function getChunkedList(inputList, chunkSize, filterCallback = null) { 25 | let currentChunk = []; 26 | const chunks = []; 27 | 28 | if (typeof filterCallback === 'function') { 29 | inputList = inputList.filter(filterCallback); 30 | } 31 | 32 | inputList.forEach((item, i) => { 33 | currentChunk.push(item); 34 | 35 | if ((currentChunk.length === chunkSize) || (i === inputList.length -1)) { 36 | chunks.push(currentChunk); 37 | currentChunk = []; 38 | } 39 | }); 40 | 41 | return chunks; 42 | } 43 | 44 | /** 45 | * Simple validation over a requests JSON payload to check for issues 46 | * such as size of request, structure etc... Determines the reply and 47 | * status code to return back to the calling function. 48 | * Note: To be moved to a proper middleware later 49 | * @param {Object} requestPayload 50 | * @param {Object} maxUrlEntries 51 | * @return {Object} 52 | */ 53 | function validateAuditRequest(requestPayload, maxUrlEntries) { 54 | if (typeof (requestPayload.urls) === 'undefined' || 55 | requestPayload.urls.length === 0) { 56 | return { 57 | valid: false, 58 | errorMessage: 'URL list not available or empty', 59 | }; 60 | } else if (requestPayload.urls.length > maxUrlEntries) { 61 | return { 62 | valid: false, 63 | errorMessage: `URL list should not exceed ${maxUrlEntries}`, 64 | }; 65 | } else { 66 | return { 67 | valid: true, 68 | }; 69 | } 70 | } 71 | 72 | /** 73 | * Converts a base 64 object buffer into a plain object 74 | * @param {Buffer} objectData 75 | * @return {String} 76 | */ 77 | function objectFromBuffer(objectData) { 78 | const buff = new Buffer(objectData, 'base64'); 79 | return JSON.parse(buff.toString('ascii')); 80 | } 81 | 82 | /** 83 | * @param {String} str 84 | * @return {Boolean} 85 | */ 86 | function isURL(str) { 87 | const pattern = new RegExp('^(https?:\\/\\/)?'+ 88 | '((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.?)+[a-z]{2,}|'+ 89 | '((\\d{1,3}\\.){3}\\d{1,3}))'+ 90 | '(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*'+ 91 | '(\\?[;&a-z\\d%_.~+=-]*)?'+ 92 | '(\\#[-a-z\\d_]*)?$', 'i'); 93 | 94 | return pattern.test(str); 95 | } 96 | 97 | module.exports = { 98 | getChunkedList, 99 | validateAuditRequest, 100 | objectFromBuffer, 101 | isURL, 102 | }; 103 | 104 | -------------------------------------------------------------------------------- /src/utils/bq.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | 'use strict'; 18 | 19 | const {BigQuery} = require('@google-cloud/bigquery'); 20 | 21 | /** 22 | * Creates a new BQ date pratitioned table for inserting results 23 | * @param {*} datasetId 24 | * @param {*} tableId 25 | * @param {*} schema 26 | */ 27 | async function createDatePartitionTable(datasetId, tableId, schema) { 28 | const options = { 29 | schema: schema, 30 | location: 'US', 31 | timePartitioning: { 32 | type: 'DAY', 33 | expirationMs: '7776000000', 34 | field: 'date', 35 | }, 36 | }; 37 | 38 | const bigquery = new BigQuery(); 39 | const [table] = await bigquery 40 | .dataset(datasetId) 41 | .createTable(tableId, options); 42 | 43 | console.log(`Table ${table.id} created.`); 44 | } 45 | 46 | /** 47 | * Writes an array of objects into BigQuery 48 | * @param {*} datasetName 49 | * @param {*} tableName 50 | * @param {*} rows 51 | */ 52 | async function writeResultStream(datasetName, tableName, rows) { 53 | const bigqueryClient = new BigQuery(); 54 | 55 | return bigqueryClient 56 | .dataset(datasetName) 57 | .table(tableName) 58 | .insert(rows); 59 | } 60 | 61 | module.exports = { 62 | createDatePartitionTable: createDatePartitionTable, 63 | writeResultStream: writeResultStream, 64 | }; 65 | -------------------------------------------------------------------------------- /src/utils/tasks.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | 'use strict'; 18 | 19 | const {CloudTasksClient} = require('@google-cloud/tasks'); 20 | const {objectFromBuffer} = require('./api'); 21 | 22 | /** 23 | * @param {string} cloudProject 24 | * @param {string} queueLocation 25 | * @param {string} tasksQueue 26 | * @param {number} pageSize 27 | * @param {string} pageToken 28 | * @return {object} 29 | */ 30 | async function listActiveTasks( 31 | cloudProject, 32 | queueLocation, 33 | tasksQueue, 34 | pageSize, 35 | pageToken) { 36 | const tasksClient = new CloudTasksClient(); 37 | const parent = tasksClient.queuePath( 38 | cloudProject, 39 | queueLocation, 40 | tasksQueue, 41 | ); 42 | 43 | return tasksClient.listTasks({ 44 | parent, 45 | responseView: 'FULL', 46 | pageSize: parseInt(pageSize) || 100, 47 | pageToken: pageToken || null, 48 | }, { 49 | autoPaginate: false, 50 | }, 51 | ); 52 | } 53 | 54 | /** 55 | * @param {Array} taskList 56 | * @return {Object} 57 | */ 58 | function processTaskResults(taskList) { 59 | const tasksResults = { 60 | tasks: [], 61 | nextPageToken: null, 62 | }; 63 | 64 | if (taskList !== undefined && taskList != null) { 65 | tasksResults.tasks = taskList['tasks'].map((taskItem) => { 66 | return { 67 | name: taskItem.name, 68 | data: objectFromBuffer(taskItem.httpRequest.body), 69 | dispatchCount: taskItem.dispatchCount, 70 | responseCount: taskItem.responseCount, 71 | firstAttempt: taskItem.firstAttempt, 72 | lastAttempt: taskItem.lastAttempt, 73 | }; 74 | }); 75 | 76 | tasksResults.nextPageToken = taskList['nextPageToken']; 77 | } 78 | 79 | return tasksResults; 80 | } 81 | 82 | module.exports = { 83 | listActiveTasks: listActiveTasks, 84 | processTaskResults: processTaskResults, 85 | }; 86 | -------------------------------------------------------------------------------- /src/utils/validate.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | **/ 16 | 17 | const Joi = require('joi'); 18 | 19 | /** 20 | * Validator for audit requests 21 | * @param {Object} req 22 | * @param {Object} res 23 | * @param {Function} next 24 | */ 25 | function auditRequestValidation(req, res, next) { 26 | const schema = Joi.object({ 27 | urls: Joi.array().required().min(1).max(5), 28 | blockedRequests: Joi.array(), 29 | }); 30 | 31 | validateRequest(req, res, next, schema); 32 | } 33 | 34 | /** 35 | * Validator for async audit requests 36 | * @param {Object} req 37 | * @param {Object} res 38 | * @param {Function} next 39 | */ 40 | function asyncAuditRequestValidation(req, res, next) { 41 | const schema = Joi.object({ 42 | urls: Joi.array().required().min(1).max(1000), 43 | blockedRequests: Joi.array(), 44 | }); 45 | 46 | validateRequest(req, res, next, schema); 47 | } 48 | 49 | /** 50 | * Validator for active tasks listing request 51 | * @param {Object} req 52 | * @param {Object} res 53 | * @param {Function} next 54 | */ 55 | function activeTasksRequestValidation(req, res, next) { 56 | const schema = Joi.object({ 57 | pageSize: Joi.number(), 58 | pageToken: Joi.string(), 59 | }); 60 | 61 | validateRequest(req, res, next, schema); 62 | } 63 | 64 | /** 65 | * Runs the request validation 66 | * @param {Object} req 67 | * @param {Object} res 68 | * @param {Function} next 69 | * @param {Object} schema 70 | * @return {*} 71 | */ 72 | function validateRequest(req, res, next, schema) { 73 | const options = { 74 | abortEarly: false, 75 | allowUnknown: true, 76 | stripUnknown: true, 77 | }; 78 | const {error, value} = schema.validate(req.body, options); 79 | if (error) { 80 | const errors = error.details.map((x) => x.message).join(', '); 81 | 82 | res.status(500); 83 | return res.json({ 84 | 'error': { 85 | 'code': 500, 86 | 'message': errors, 87 | }, 88 | }); 89 | } else { 90 | req.body = value; 91 | next(); 92 | } 93 | } 94 | 95 | module.exports = { 96 | auditRequestValidation, 97 | asyncAuditRequestValidation, 98 | activeTasksRequestValidation, 99 | }; 100 | --------------------------------------------------------------------------------