├── .github └── workflows │ └── ci-check.yml ├── .gitignore ├── .npmignore ├── LICENSE ├── README.md ├── example └── porchmark.puppeteer.conf.js ├── install.js ├── jest.config.js ├── package.json ├── src ├── bin │ ├── porchmark-compare.ts │ └── porchmark.ts ├── constants.ts ├── lib │ ├── comparison.ts │ ├── config │ │ ├── commanderArgv.ts │ │ ├── config.example.js │ │ ├── default.ts │ │ ├── index.ts │ │ ├── schema.ts │ │ └── types.ts │ ├── dataProcessor │ │ ├── __spec__ │ │ │ ├── dataProcessor.spec.ts │ │ │ └── mock.ts │ │ └── index.ts │ ├── findFreePorts │ │ ├── findFreePort.ts │ │ └── index.ts │ ├── fs.ts │ ├── helpers.ts │ ├── logger.ts │ ├── puppeteer │ │ ├── browser.ts │ │ ├── index.ts │ │ ├── metrics.ts │ │ ├── networkPresets.ts │ │ ├── page.ts │ │ ├── pageStructureSizes.ts │ │ ├── runCheck.ts │ │ └── types.ts │ ├── report │ │ ├── __spec__ │ │ │ ├── mock.ts │ │ │ └── report.spec.ts │ │ ├── humanReport.ts │ │ ├── index.ts │ │ └── jsonReport.ts │ ├── stats.ts │ ├── view.ts │ ├── webdriverio.ts │ ├── workerFarm.ts │ └── wpr │ │ ├── WprAbstract.ts │ │ ├── WprRecord.ts │ │ ├── WprReplay.ts │ │ ├── index.ts │ │ ├── select.ts │ │ └── types.ts ├── packages │ └── porchmark-pretty-reporter │ │ ├── README.md │ │ ├── __spec__ │ │ ├── config.mock.ts │ │ ├── index.spec.ts │ │ └── jsonRawReport.mock.ts │ │ ├── chartReport.ts │ │ ├── index.ts │ │ └── lib │ │ ├── aggregationBarChart.ts │ │ ├── lineChart.ts │ │ ├── template │ │ ├── html.ts │ │ ├── index.ts │ │ ├── svg.ts │ │ └── template.ts │ │ └── utils.ts └── types.ts ├── tsconfig.json ├── tslint.json └── types ├── console.table └── index.d.ts └── jStat └── index.d.ts /.github/workflows/ci-check.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: npm install 16 | run: npm install 17 | - name: npm run ci:check 18 | run: npm run ci:check 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | package-lock.json* 8 | /porchmark.conf.js 9 | 10 | /dist 11 | *.iml 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # Optional npm cache directory 48 | .npm 49 | 50 | # Optional eslint cache 51 | .eslintcache 52 | 53 | # Optional REPL history 54 | .node_repl_history 55 | 56 | # Output of 'npm pack' 57 | *.tgz 58 | 59 | # Yarn Integrity file 60 | .yarn-integrity 61 | 62 | # dotenv environment variables file 63 | .env 64 | 65 | # next.js build output 66 | .next 67 | 68 | # Jetbrains 69 | .idea 70 | 71 | # visual studio code 72 | .vscode 73 | 74 | # wpr downloaded 75 | /wpr 76 | 77 | # Result of comparision 78 | /compare 79 | 80 | /example 81 | .DS_Store 82 | /html-report-report.html 83 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | /porchmark.conf.js 3 | /wpr 4 | .idea 5 | /example 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Cornholio 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # porchmark 2 | Simple tool to compare browser performance of several pages. 3 | It opens given pages at the same time, and capture browser performance metrics. A lot of times. 4 | Main purpose is to help testing hypotheses about frontend performance in development environment. 5 | 6 | ![screencast.gif](http://mcornholio-s3.s3.amazonaws.com/porchmark-screencast-3.gif) 7 | 8 | ### Installation: 9 | What do you think? 10 | ``` 11 | npm install -g porchmark 12 | ``` 13 | 14 | ### Example: 15 | 16 | ``` 17 | npm i 18 | cd example 19 | node ../dist/bin/porchmark.js compare -c ./porchmark.puppeteer.conf.js 20 | ``` 21 | 22 | ### Usage: 23 | #### Puppeteer mode 24 | porchmark launches several headless chromium browsers on your desktop. Easy start, but there's never enough CPU to get that data fast. It's possible to run porchmark in puppeteer mode on server, but that'll require X. 25 | 26 | #### Webdriver mode 27 | Pretty much the same, but does the work on remote webdriver browsers. If you have a large Selenium Grid, you'll be able to get that data in no time. 28 | To run porchmark in webdriver mode You'll have to make a config file (just copy that one ↓). 29 | 30 | CLI args: 31 | ``` 32 | Usage: porchmark compare ... 33 | 34 | Options: 35 | -V, --version output the version number 36 | -i, --iterations stop after n iterations; defaults to 300 37 | -P, --parallel run checks in n workers; defaults to 1 38 | -m, --mobile chrome mobile UA, iphone 6-like screen, touch events, etc. 39 | -k, --insecure ignore HTTPS errors 40 | -t, --timeout timeout in seconds for each check; defaults to 20s 41 | -c --config [configfile.js] path to config; default is `porchmark.conf.js` in current dir 42 | -v, --verbose verbose logging, -v (debug), -vv (trace) 43 | -h, --help output usage information 44 | ``` 45 | 46 | Config file: 47 | ```js 48 | module.exports = { 49 | // log level - 'trace', 'debug', 'info', 'warn', 'error', 'fatal' 50 | logLevel: 'info', 51 | // workdir for comparison files (screenshots, WPR-archives, etc) 52 | // by default is current work dir (cwd) 53 | workDir: `${__dirname}/some-workdir`, 54 | // mode - 'puppeteer' or 'webdriver' 55 | mode: 'puppeteer', 56 | // how many iterations run for every comparison (and every WPR for puppeteer mode) 57 | iterations: 70, 58 | // how many parallel workers 59 | workers: 1, 60 | // page open timeout 61 | pageTimeout: 90, 62 | // disable terminal table UI 63 | withoutUi: false, 64 | 65 | // options for puppeteer mode 66 | puppeteerOptions: { 67 | // run browser headless or not 68 | headless: true, 69 | 70 | // ignore https errors - useful for custom ssl certificates 71 | ignoreHTTPSErrors: false, 72 | 73 | // use Web Page Replay (WPR) archives - https://bit.ly/2JgTUbt 74 | useWpr: true, 75 | 76 | // how many WPR archives record 77 | recordWprCount: 50, 78 | 79 | // how many WPR archive pairs select for every comparison 80 | selectWprCount: 10, 81 | 82 | // method for WPR archive selection 83 | // - simple - select WPRs in recorded order 84 | // - closestByWprSize - select WPR pairs by WPR archive size, with minimal diff by size 85 | // - closestByHtmlSize - select WPRs by html size from server, this is default 86 | // - closestByScriptSize - select WPRs by script size from server 87 | selectWprMethod: 'closestByHtmlSize', 88 | 89 | // enable/disable browser cache on page open 90 | cacheEnabled: false, 91 | 92 | // cpu throttling 93 | cpuThrottling: { 94 | rate: 4, 95 | }, 96 | 97 | // network throttling 98 | // 'GPRS', 'Regular2G', 'Good2G', 'Regular3G', 'Good3G', 'Regular4G', 'DSL', 'WiFi' 99 | networkThrottling: 'Regular2G', 100 | 101 | // enable/disable load images 102 | imagesEnabled: true, 103 | 104 | // enable/disable javascript on page 105 | javascriptEnabled: true, 106 | 107 | // enable/disable css files 108 | // this work with puppeteer request interceptions and may slow down comparison 109 | cssFilesEnabled: true, 110 | 111 | // puppeteer page navigation timeout 112 | pageNavigationTimeout: 60000, 113 | 114 | // puppeteer waitUntil page open 115 | // 'load', 'domcontentloaded', 'networkidle0', 'networkidle2' 116 | waitUntil: 'load', 117 | 118 | // retry count if WPR record fails 119 | retryCount: 10, 120 | }, 121 | 122 | // webdriver options 123 | webdriverOptions: { 124 | host: 'your-grid-address.sh', 125 | port: 4444, 126 | user : '', 127 | key: '', 128 | desiredCapabilities: { 129 | 'browserName': 'chrome', 130 | 'version': '65.0', 131 | }, 132 | }, 133 | 134 | // browser profile 135 | browserProfile: { 136 | // emulate mobile useragent and viewport 137 | mobile: false, 138 | 139 | // set useragent 140 | userAgent: 'your-user-agent', 141 | 142 | // viewport height 143 | height: 600, 144 | 145 | // viewport width 146 | width: 800, 147 | }, 148 | 149 | // setup comparisons with array 150 | comparisons: [ 151 | { 152 | // uniq comparison name 153 | name: 'main', 154 | 155 | sites: [ 156 | { 157 | // uniq site name 158 | name: 'production', 159 | // url 160 | url: 'https://host1.ru' 161 | }, 162 | { 163 | name: 'prestable', 164 | url: 'https://host2.ru' 165 | }, 166 | ], 167 | }, 168 | // ... 169 | ], 170 | stages: { 171 | // run record WPR stage 172 | recordWpr: true, 173 | 174 | // run comparison metrics 175 | compareMetrics: true, 176 | }, 177 | 178 | // metrics for collect 179 | metrics: [ 180 | {name: 'requestStart'}, 181 | {name: 'responseStart', title: 'TTFB'}, 182 | {name: 'responseEnd', title: 'TTLB'}, 183 | {name: 'first-paint'}, 184 | {name: 'first-contentful-paint', title: 'FCP'}, 185 | {name: 'domContentLoadedEventEnd', title: 'DCL'}, 186 | {name: 'loadEventEnd', title: 'loaded'}, 187 | {name: 'domInteractive'}, 188 | {name: 'domComplete'}, 189 | {name: 'transferSize'}, 190 | {name: 'encodedBodySize'}, 191 | {name: 'decodedBodySize'}, 192 | ], 193 | 194 | // metric aggregations 195 | metricAggregations: [ 196 | { 197 | name: 'count', 198 | includeMetrics: ['requestStart'] 199 | }, // apply aggregation only for requestStart metric 200 | {name: 'q50'}, 201 | {name: 'q80'}, 202 | {name: 'q95'}, 203 | { 204 | name: 'stdev', 205 | excludeMetrics: ['transferSize'] 206 | }, 207 | ], 208 | 209 | // 210 | hooks: { 211 | 212 | // verify page on WPR record - throw error if not valid 213 | async onVerifyWpr({logger, page, comparison, site}) { 214 | const hasJquery = await page.evaluate( 215 | () => !!window.jQuery 216 | ); 217 | 218 | if (!hasJquery) { 219 | throw new Error( 220 | 'no jQuery on page, page incorrect' 221 | ); 222 | } 223 | }, 224 | 225 | // collect and return custom metrics from page 226 | async onCollectMetrics({logger, page, comparison, site}) { 227 | const nodesCount = await page.evaluate( 228 | () => document.querySelectorAll('*').length 229 | ); 230 | 231 | return { 232 | nodesCount, 233 | }; 234 | }, 235 | }, 236 | }; 237 | 238 | ``` 239 | -------------------------------------------------------------------------------- /example/porchmark.puppeteer.conf.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | logLevel: 'debug', 3 | mode: 'puppeteer', 4 | iterations: 10, 5 | workers: 1, 6 | puppeteerOptions: { 7 | headless: true, 8 | useWpr: true, 9 | recordWprCount: 3, 10 | selectWprCount: 1, 11 | warmIterations: 2, 12 | }, 13 | comparisons: [ 14 | { 15 | name: 'main', 16 | sites: [ 17 | { 18 | name: 'first', 19 | url: 'http://example.com/', 20 | }, 21 | { 22 | name: 'second', 23 | url: 'http://example.com/', 24 | } 25 | ] 26 | } 27 | ] 28 | }; 29 | -------------------------------------------------------------------------------- /install.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const os = require('os'); 4 | const https = require('https'); 5 | const {execSync} = require('child_process'); 6 | const fs = require('fs-extra'); 7 | const tracer = require('tracer'); 8 | const logger = tracer.console({format: "{{timestamp}} <{{title}}> {{message}}"}); 9 | 10 | const baseUrl = 'https://github.com/alekzonder/catapult/releases/download'; 11 | const release = 'wpr-build%2F2019.12.17-5'; 12 | const file = `wpr-${os.platform()}.tgz`; 13 | 14 | const downloadUrl = `${baseUrl}/${release}/${file}`; 15 | 16 | logger.info(`download wpr binaries: ${downloadUrl}`); 17 | 18 | const targetDir = `${__dirname}/wpr`; 19 | const archiveFilepath = `${__dirname}/wpr.tgz`; 20 | const archiveStream = fs.createWriteStream(archiveFilepath); 21 | 22 | if (fs.existsSync(targetDir)) { 23 | logger.info(`remove ${targetDir}`); 24 | fs.removeSync(targetDir); 25 | } 26 | 27 | https.get(downloadUrl, (firstRes) => { 28 | // github redirect to amazonaws 29 | if (firstRes.statusCode === 302) { 30 | logger.info(`got redirect to: ${firstRes.headers.location}`); 31 | 32 | https.get(firstRes.headers.location, (redirectedRes) => { 33 | logger.info(`download ${redirectedRes.headers['content-length']} bytes`); 34 | redirectedRes.pipe(archiveStream); 35 | }); 36 | 37 | } else { 38 | throw new Error(`cant download WPR binaries, download and unpack manually: ${downloadUrl}`); 39 | } 40 | }); 41 | 42 | archiveStream.on('finish', () => { 43 | logger.info(`downloaded, untar to ${targetDir}`); 44 | const cmd = `tar xzf ${archiveFilepath}`; 45 | execSync(cmd); 46 | 47 | logger.info(`cleanup, remove ${archiveFilepath}`); 48 | fs.removeSync(archiveFilepath); 49 | 50 | logger.info('done'); 51 | }); 52 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | moduleNameMapper: { 5 | '^@/(.*)$': ['/src/$1'] 6 | }, 7 | testPathIgnorePatterns: [ 8 | '/node_modules/', 9 | '/dist/' 10 | ] 11 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "porchmark", 3 | "version": "2.0.0-beta.9", 4 | "description": "Simple tool to compare browser performance of several sites simultaneously", 5 | "scripts": { 6 | "build": "tsc && npm run updateTscPaths", 7 | "watch": "tsc-watch --onSuccess 'npm run updateTscPaths' --onFailure 'npm run updateTscPaths'", 8 | "updateTscPaths": "tscpaths -p tsconfig.json -s ./src -o ./dist", 9 | "prepare": "npm run build", 10 | "install": "node install.js", 11 | "lint": "tslint --project .", 12 | "ci:check": "npm run build && npm run lint && npm run test", 13 | "test": "jest --no-cache", 14 | "test:watch": "jest --watch --no-cache", 15 | "clean": "rm -rf dist" 16 | }, 17 | "bin": "dist/bin/porchmark.js", 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/mutantcornholio/porchmark" 21 | }, 22 | "author": "cornholio <0@mcornholio.ru>", 23 | "license": "ISC", 24 | "dependencies": { 25 | "@hapi/joi": "^16.1.8", 26 | "blessed": "^0.1.81", 27 | "cli-table2": "^0.2.0", 28 | "colors": "^1.3.1", 29 | "commander": "^4.0.1", 30 | "console.table": "^0.10.0", 31 | "cookie": "^0.3.1", 32 | "d3": "^5.16.0", 33 | "fs-extra": "^8.1.0", 34 | "jsdom": "^16.2.2", 35 | "jstat": "^1.9.2", 36 | "lighthouse": "^5.6.0", 37 | "lodash": "^4.17.11", 38 | "porchmark-pretty-reporter": "^0.2.0", 39 | "puppeteer": "^2.0.0", 40 | "source-map-support": "^0.5.16", 41 | "tracer": "^1.0.1", 42 | "webdriverio": "^4.14.0" 43 | }, 44 | "devDependencies": { 45 | "@types/blessed": "^0.1.10", 46 | "@types/cli-table2": "^0.2.2", 47 | "@types/d3": "^5.7.2", 48 | "@types/fs-extra": "^8.0.1", 49 | "@types/hapi__joi": "^16.0.3", 50 | "@types/jest": "^25.2.1", 51 | "@types/jsdom": "^16.2.1", 52 | "@types/lodash": "^4.14.117", 53 | "@types/node": "^12.12.6", 54 | "@types/puppeteer": "^2.0.0", 55 | "@types/traverse": "^0.6.32", 56 | "@types/webdriverio": "^4.13.0", 57 | "husky": "^3.1.0", 58 | "jest": "^25.4.0", 59 | "ts-jest": "^25.4.0", 60 | "tsc-watch": "^4.0.0", 61 | "tscpaths": "0.0.9", 62 | "tslint": "^5.20.1", 63 | "typescript": "3.8.3" 64 | }, 65 | "husky": { 66 | "hooks": { 67 | "pre-commit": "npm run build && npm run lint && npm run test" 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/bin/porchmark-compare.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | 4 | import path from 'path'; 5 | 6 | import program, {Command} from 'commander'; 7 | 8 | import {createLogger, setLogfilePath, setLogger} from '@/lib/logger'; 9 | 10 | // setLogger should be before resolveConfig import 11 | const logger = createLogger(); 12 | setLogger(logger); 13 | 14 | import {startComparison} from '@/lib/comparison'; 15 | import {resolveConfig, saveConfig} from '@/lib/config'; 16 | import {getView} from '@/lib/view'; 17 | 18 | const view = getView(); 19 | 20 | process.on('unhandledRejection', (e) => { 21 | logger.error(e); 22 | process.exit(1); 23 | }); 24 | process.on('SIGINT', () => view.shutdown(false)); 25 | process.on('SIGTERM', () => view.shutdown(false)); 26 | 27 | program 28 | .description('realtime compare websites') 29 | .option('-i, --iterations ', 'stop after n iterations; defaults to 300', parseInt) 30 | .option('-P, --parallel ', 'run checks in n workers; defaults to 1', parseInt) 31 | .option('-m, --mobile', 'chrome mobile UA, iphone 6-like screen, touch events, etc.') 32 | .option('-k, --insecure', 'ignore HTTPS errors') 33 | .option('-t, --timeout ', 'timeout in seconds for each check; defaults to 20s', parseInt) 34 | .option('-c --config [configfile.js]', 'path to config; default is `porchmark.conf.js` in current dir') 35 | .option( 36 | '-v, --verbose', 37 | 'verbose logging, -v (debug), -vv (trace)', 38 | function increaseVerbosity(_: number, previous: number) { 39 | return previous + 1; 40 | }, 41 | 0, 42 | ) 43 | .action(async function(cmd: Command) { 44 | const config = await resolveConfig(cmd); 45 | 46 | view.config = config; 47 | 48 | view.init(); 49 | 50 | const logfilePath = path.resolve(config.workDir, 'porchmark.log'); 51 | 52 | setLogfilePath(logfilePath); 53 | 54 | await saveConfig(logger, config); 55 | 56 | logger.info('config', config); 57 | 58 | for (const comparison of config.comparisons) { 59 | await startComparison(config, comparison); 60 | } 61 | 62 | view.shutdown(false); 63 | }) 64 | .parse(process.argv); 65 | -------------------------------------------------------------------------------- /src/bin/porchmark.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | 4 | import program from 'commander'; 5 | 6 | import {getView} from '@/lib/view'; 7 | 8 | import {createLogger, setLogger} from '@/lib/logger'; 9 | 10 | const view = getView(); 11 | 12 | const logger = createLogger(); 13 | setLogger(logger); 14 | 15 | // if import, tsc ignores rootDir and transpile package.json and src to dist 16 | // tslint:disable-next-line no-var-requires 17 | const pkg = require('@/../package.json'); 18 | 19 | const version = pkg.version; 20 | 21 | process.on('unhandledRejection', (e) => { 22 | logger.error(e); 23 | process.exit(1); 24 | }); 25 | process.on('SIGINT', () => view.shutdown(false)); 26 | process.on('SIGTERM', () => view.shutdown(false)); 27 | 28 | program 29 | .version(version) 30 | .description('Compare websites speed!') 31 | .command('compare ', 'realtime compare websites') 32 | .parse(process.argv); 33 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | // tslint:disable-next-line no-var-requires 2 | const pkg = require('@/../package.json'); 3 | 4 | export const PORCHMARK_VERSION = pkg.version; 5 | 6 | export const PORCHMARK_REPORT_VERSION = 1.0; 7 | -------------------------------------------------------------------------------- /src/lib/comparison.ts: -------------------------------------------------------------------------------- 1 | import {ChartReport} from '@/packages/porchmark-pretty-reporter'; 2 | import fs from 'fs-extra'; 3 | 4 | import {IComparison, IConfig} from '@/lib/config'; 5 | import {DataProcessor} from '@/lib/dataProcessor'; 6 | import {getLogger} from '@/lib/logger'; 7 | import {HumanReport, JsonReport, saveReports} from '@/lib/report'; 8 | 9 | import {getComparisonDir} from '@/lib/fs'; 10 | import {isoDate} from '@/lib/helpers'; 11 | import {getView} from '@/lib/view'; 12 | import startWorking from '@/lib/workerFarm'; 13 | import {recordWprArchives} from '@/lib/wpr'; 14 | import {getWprArchives, selectWprArchives} from '@/lib/wpr/select'; 15 | import {ISelectedWprArchives} from '@/lib/wpr/types'; 16 | 17 | const logger = getLogger(); 18 | const view = getView(); 19 | 20 | export async function startComparison(config: IConfig, comparison: IComparison) { 21 | const startedAt = isoDate(); 22 | 23 | logger.info(`pid=${process.pid}`); 24 | 25 | const dataProcessor = new DataProcessor(config, comparison); 26 | 27 | const renderTableInterval = setInterval(() => { 28 | view.renderTable(dataProcessor.calculateResults()); 29 | }, 200); 30 | 31 | if ( 32 | config.mode === 'puppeteer' && 33 | config.puppeteerOptions.useWpr && 34 | config.stages.recordWpr 35 | ) { 36 | await recordWprArchives(comparison, config); 37 | } 38 | 39 | if (config.stages.compareMetrics) { 40 | let selectedWprArchives: ISelectedWprArchives[] = []; 41 | let cycleCount = 1; 42 | 43 | const withWpr = config.mode === 'puppeteer' && config.puppeteerOptions.useWpr; 44 | 45 | const comparisonDir = getComparisonDir(config.workDir, comparison); 46 | 47 | await fs.ensureDir(comparisonDir); 48 | 49 | if (withWpr) { 50 | const wprArchives = await getWprArchives(comparisonDir, comparison.sites); 51 | 52 | selectedWprArchives = await selectWprArchives( 53 | config, 54 | wprArchives, 55 | comparison.sites, 56 | ); 57 | 58 | cycleCount = selectedWprArchives.length; 59 | } 60 | 61 | for (let compareId = 0; compareId < cycleCount; compareId++) { 62 | logger.info(`start comparison name=${comparison.name}, id=${compareId}`); 63 | 64 | if (withWpr) { 65 | comparison.wprArchives = selectedWprArchives[compareId].wprArchives; 66 | logger.info(`start comparison with wpr archives: ${JSON.stringify(selectedWprArchives[compareId])}`); 67 | } 68 | 69 | try { 70 | await startWorking(compareId, comparison, dataProcessor, config).catch(view.emergencyShutdown); 71 | } catch (error) { 72 | logger.error(error); 73 | } 74 | } 75 | 76 | clearInterval(renderTableInterval); 77 | 78 | logger.info('save reports'); 79 | 80 | const jsonRawReport = await dataProcessor.calcReport(comparison.sites); 81 | 82 | const completedAt = isoDate(); 83 | 84 | // TODO process status and status message 85 | const status = 'not_implemented'; 86 | const statusMessage = 'not implemented yet'; 87 | 88 | await saveReports({ 89 | startedAt, 90 | completedAt, 91 | status, 92 | statusMessage, 93 | jsonRawReport, 94 | config, 95 | id: 'total', 96 | workDir: comparisonDir, 97 | reporters: [ 98 | HumanReport, 99 | JsonReport, 100 | ChartReport, 101 | ], 102 | }); 103 | 104 | logger.info('complete'); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/lib/config/commanderArgv.ts: -------------------------------------------------------------------------------- 1 | import {Command} from 'commander'; 2 | import path from 'path'; 3 | 4 | import {IComparison, IConfig, IPartialConfig, mergeWithDefaults, validateConfig} from '@/lib/config'; 5 | 6 | import {isInteractive} from '@/lib/helpers'; 7 | import {getLogger, setLevel} from '@/lib/logger'; 8 | import joi from '@hapi/joi'; 9 | 10 | const logger = getLogger(); 11 | 12 | export interface ICompareMetricsArgv { 13 | iterations?: number; 14 | parallel?: number; 15 | mobile?: boolean; 16 | insecure?: boolean; 17 | timeout?: number; 18 | config?: string; 19 | verbose?: number; 20 | } 21 | 22 | export const defaultDesktopProfile = { 23 | height: 768, 24 | width: 1366, 25 | }; 26 | 27 | export const defaultMobileProfile = { 28 | userAgent: 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 ' + 29 | '(KHTML, like Gecko) Chrome/76.0.3809.100 Mobile Safari/537.36', 30 | height: 667, 31 | width: 375, 32 | }; 33 | 34 | export function readConfig(configPath: string): IPartialConfig { 35 | try { 36 | const config = require(configPath); 37 | return config as IPartialConfig; 38 | } catch (e) { 39 | if (e.code !== 'MODULE_NOT_FOUND') { 40 | logger.fatal(`invalid config at path ${configPath}: ${e.stack}`); 41 | return process.exit(1); 42 | } 43 | 44 | return {}; 45 | } 46 | } 47 | 48 | export async function resolveConfig(commanderArgv: Command): Promise { 49 | const porchmarkConfPath = path.resolve(process.cwd(), 'porchmark.conf.js'); 50 | 51 | let configPath = ''; 52 | 53 | let rawConfig: IPartialConfig = {}; 54 | 55 | if (typeof commanderArgv.config === 'string') { 56 | // config option 57 | configPath = path.isAbsolute(commanderArgv.config) 58 | ? commanderArgv.config 59 | : path.resolve(process.cwd(), commanderArgv.config); 60 | rawConfig = readConfig(configPath); 61 | } else { 62 | // porchmark.conf.js exists 63 | configPath = porchmarkConfPath; 64 | rawConfig = readConfig(configPath); 65 | } 66 | 67 | logger.debug('raw config', porchmarkConfPath, rawConfig); 68 | 69 | const config = mergeWithDefaults(rawConfig as IConfig); 70 | 71 | if (!config.workDir) { 72 | config.workDir = process.cwd(); 73 | } 74 | 75 | if (typeof commanderArgv.mobile === 'boolean') { 76 | config.browserProfile.mobile = commanderArgv.mobile; 77 | } 78 | 79 | addOptsFromArgv(config, commanderArgv as ICompareMetricsArgv); 80 | addSitesFromArgv(config, commanderArgv); 81 | 82 | // convert seconds to ms 83 | config.pageTimeout = config.pageTimeout * 1000; 84 | 85 | initBrowserProfile(config); 86 | 87 | if (!isInteractive()) { 88 | config.withoutUi = true; 89 | } 90 | 91 | normalizeMetrics(config); 92 | 93 | logger.debug('config', config); 94 | 95 | try { 96 | await validateConfig(config); 97 | 98 | setLevel(config.logLevel); 99 | } catch (error) { 100 | // @ts-ignore 101 | if (error instanceof joi.ValidationError) { 102 | logger.fatal(`invalid config ${configPath ? `file=${configPath}` : ''}`); 103 | 104 | error.details.forEach((e: any) => { 105 | logger.fatal(`path=${e.path.join('.')}, ${e.message}`); 106 | }); 107 | } else { 108 | logger.fatal(error); 109 | } 110 | 111 | process.exit(1); 112 | } 113 | 114 | return config; 115 | } 116 | 117 | function addOptsFromArgv(config: IConfig, commanderArgv: ICompareMetricsArgv) { 118 | if (typeof commanderArgv.iterations === 'number') { 119 | config.iterations = commanderArgv.iterations; 120 | } 121 | 122 | if (typeof commanderArgv.parallel === 'number') { 123 | config.workers = commanderArgv.parallel; 124 | } 125 | 126 | if (typeof commanderArgv.insecure === 'boolean') { 127 | config.puppeteerOptions.ignoreHTTPSErrors = commanderArgv.insecure; 128 | } 129 | 130 | if (typeof commanderArgv.timeout === 'number') { 131 | config.pageTimeout = commanderArgv.timeout; 132 | } 133 | 134 | if (typeof commanderArgv.verbose === 'number') { 135 | if (commanderArgv.verbose === 1) { 136 | config.logLevel = 'debug'; 137 | } else if (commanderArgv.verbose > 1) { 138 | config.logLevel = 'trace'; 139 | } 140 | } 141 | } 142 | 143 | function addSitesFromArgv(config: IConfig, cmd: Command) { 144 | const sites: string[] = cmd.args; 145 | 146 | if (sites && sites.length) { 147 | const comparison: IComparison = { 148 | name: 'compare', 149 | sites: sites.map((url, index) => ({ 150 | name: `site${index}`, 151 | url, 152 | })), 153 | }; 154 | 155 | config.comparisons = [ 156 | comparison, 157 | ]; 158 | } 159 | } 160 | 161 | function initBrowserProfile(config: IConfig) { 162 | let browserProfile = config.browserProfile; 163 | if (browserProfile.mobile) { 164 | browserProfile = { 165 | ...browserProfile, 166 | ...defaultMobileProfile, 167 | }; 168 | } 169 | 170 | if (!browserProfile.width || !browserProfile.height) { 171 | browserProfile = { 172 | ...defaultDesktopProfile, 173 | ...browserProfile, 174 | }; 175 | } 176 | 177 | config.browserProfile = browserProfile; 178 | } 179 | 180 | function normalizeMetrics(config: IConfig) { 181 | config.metrics = config.metrics.map((metric) => { 182 | if (typeof metric.showInTable === 'undefined') { 183 | metric.showInTable = true; 184 | } 185 | 186 | return metric; 187 | }); 188 | } 189 | -------------------------------------------------------------------------------- /src/lib/config/config.example.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @type {IConfig} 4 | */ 5 | const exampleConfig = { 6 | logLevel: 'info', 7 | workDir: `${__dirname}/yandex`, 8 | mode: 'puppeteer', 9 | iterations: 70, 10 | workers: 1, 11 | pageTimeout: 90, 12 | puppeteerOptions: { 13 | headless: true, 14 | ignoreHTTPSErrors: false, 15 | useWpr: true, 16 | recordWprCount: 50, 17 | selectWprCount: 10, 18 | selectWprMethod: 'closestByHtmlSize', 19 | cacheEnabled: false, 20 | cpuThrottling: { 21 | rate: 4, 22 | }, 23 | networkThrottling: 'Regular2G', 24 | singleProcess: false, 25 | imagesEnabled: true, 26 | javascriptEnabled: true, 27 | cssFilesEnabled: true, 28 | }, 29 | webdriverOptions: { 30 | host: 'localhost', 31 | port: 4444, 32 | user : '', 33 | key: '', 34 | desiredCapabilities: { 35 | 'browserName': 'chrome', 36 | 'version': '65.0', 37 | }, 38 | }, 39 | browserProfile: { 40 | mobile: false, 41 | userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)' + 42 | ' Chrome/60.0.3112.113 Safari/537.36', 43 | height: 600, 44 | width: 800, 45 | }, 46 | comparisons: [ 47 | { 48 | name: 'main', 49 | sites: [ 50 | {name: 'production', url: 'https://yandex.ru'}, 51 | {name: 'prestable', url: 'https://yandex.ru'}, 52 | ], 53 | }, 54 | { 55 | name: 'company', 56 | sites: [ 57 | {name: 'production', url: 'https://yandex.ru/company/'}, 58 | {name: 'prestable', url: 'https://yandex.ru/company/'}, 59 | ], 60 | }, 61 | ], 62 | stages: { 63 | recordWpr: true, 64 | compareMetrics: true, 65 | compareLighthouse: true, 66 | }, 67 | metrics: [ 68 | {name: 'requestStart'}, 69 | {name: 'responseStart', title: 'TTFB'}, 70 | {name: 'responseEnd', title: 'TTLB'}, 71 | {name: 'first-paint'}, 72 | {name: 'first-contentful-paint', title: 'FCP'}, 73 | {name: 'domContentLoadedEventEnd', title: 'DCL'}, 74 | {name: 'loadEventEnd', title: 'loaded'}, 75 | {name: 'domInteractive'}, 76 | {name: 'domComplete'}, 77 | {name: 'transferSize'}, 78 | {name: 'encodedBodySize'}, 79 | {name: 'decodedBodySize'}, 80 | ], 81 | metricAggregations: [ 82 | {name: 'count', includeMetrics: ['requestStart']}, // apply aggregation only for requestStart metric 83 | {name: 'q50'}, 84 | {name: 'q80'}, 85 | {name: 'q95'}, 86 | {name: 'stdev', excludeMetrics: ['transferSize']}, 87 | ], 88 | hooks: { 89 | async onVerifyWpr({logger, page, comparison, site}) { 90 | const hasJquery = await page.evaluate(() => !!window.jQuery); 91 | 92 | if (!hasJquery) { 93 | throw new Error('no jQuery on page, page incorrect'); 94 | } 95 | }, 96 | async onCollectMetrics({logger, page, comparison, site}) { 97 | const nodesCount = await page.evaluate(() => document.querySelectorAll('*').length); 98 | 99 | return { 100 | nodesCount, 101 | }; 102 | }, 103 | }, 104 | }; 105 | 106 | module.exports = exampleConfig; 107 | -------------------------------------------------------------------------------- /src/lib/config/default.ts: -------------------------------------------------------------------------------- 1 | import {IConfig, SelectWprMethods} from '@/lib/config/types'; 2 | 3 | export default (): IConfig => ({ 4 | logLevel: 'info', 5 | workDir: '', 6 | mode: 'puppeteer', 7 | iterations: 70, 8 | workers: 1, 9 | pageTimeout: 20, // in seconds 10 | withoutUi: false, 11 | puppeteerOptions: { 12 | headless: true, 13 | ignoreHTTPSErrors: false, 14 | useWpr: true, 15 | recordWprCount: 50, 16 | selectWprCount: 10, 17 | selectWprMethod: SelectWprMethods.closestByHtmlSize, 18 | cacheEnabled: true, 19 | imagesEnabled: true, 20 | javascriptEnabled: true, 21 | cssFilesEnabled: true, 22 | cpuThrottling: null, 23 | networkThrottling: null, 24 | pageNavigationTimeout: 60000, 25 | waitUntil: 'load', 26 | retryCount: 10, 27 | warmIterations: 1, 28 | }, 29 | webdriverOptions: { 30 | host: 'localhost', 31 | port: 4444, 32 | user : '', 33 | key: '', 34 | desiredCapabilities: { 35 | browserName: 'chrome', 36 | version: '65.0', 37 | }, 38 | }, 39 | browserProfile: { 40 | mobile: false, 41 | userAgent: null, 42 | height: 0, 43 | width: 0, 44 | }, 45 | comparisons: [], 46 | stages: { 47 | recordWpr: true, 48 | compareMetrics: true, 49 | compareLighthouse: false, 50 | }, 51 | metrics: [ 52 | {name: 'requestStart', showInTable: true}, 53 | {name: 'responseStart', title: 'TTFB', showInTable: true}, 54 | {name: 'responseEnd', title: 'TTLB', showInTable: true}, 55 | {name: 'first-paint', showInTable: false}, 56 | {name: 'first-contentful-paint', title: 'FCP', showInTable: true}, 57 | {name: 'domContentLoadedEventEnd', title: 'DCL', showInTable: true}, 58 | {name: 'loadEventEnd', title: 'loaded', showInTable: true}, 59 | {name: 'domInteractive', showInTable: false}, 60 | {name: 'domComplete', showInTable: false}, 61 | {name: 'transferSize', showInTable: false}, 62 | {name: 'encodedBodySize', showInTable: false}, 63 | {name: 'decodedBodySize', showInTable: false}, 64 | ], 65 | metricAggregations: [ 66 | {name: 'count', includeMetrics: ['requestStart']}, 67 | {name: 'q50'}, 68 | {name: 'q80'}, 69 | {name: 'q95'}, 70 | ], 71 | hooks: {}, 72 | }); 73 | -------------------------------------------------------------------------------- /src/lib/config/index.ts: -------------------------------------------------------------------------------- 1 | export * from './types'; 2 | export * from './commanderArgv'; 3 | 4 | import joi from '@hapi/joi'; 5 | import fs from 'fs-extra'; 6 | import {defaultsDeep} from 'lodash'; 7 | 8 | import {IConfig} from '@/lib/config/types'; 9 | import {Logger} from '@/lib/logger'; 10 | 11 | import getDefaultConfig from './default'; 12 | import schema from './schema'; 13 | 14 | const mergeWithDefaults = (rawConfig: IConfig): IConfig => { 15 | const merged: IConfig = defaultsDeep({}, rawConfig, getDefaultConfig()); 16 | 17 | if (typeof rawConfig.metrics !== 'undefined') { 18 | merged.metrics = rawConfig.metrics; 19 | } 20 | 21 | if (typeof rawConfig.metricAggregations !== 'undefined') { 22 | merged.metricAggregations = rawConfig.metricAggregations; 23 | } 24 | 25 | return merged; 26 | }; 27 | 28 | const validateConfig = (rawConfig: any): Promise => { 29 | const options: joi.ValidationOptions = { 30 | abortEarly: false, 31 | convert: true, 32 | }; 33 | 34 | return schema.validateAsync(rawConfig, options); 35 | }; 36 | 37 | const getConfigFilepath = (workdir: string): string => `${workdir}/porchmark.config.json`; 38 | 39 | const saveConfig = async (logger: Logger, config: IConfig): Promise => { 40 | const data = JSON.stringify(config, undefined, 2); 41 | 42 | let configFilepath = getConfigFilepath(config.workDir); 43 | 44 | const configExists = await fs.pathExists(configFilepath); 45 | 46 | if (configExists) { 47 | const newConfigFilepath = `${configFilepath}-${Date.now()}`; 48 | logger.warn(`config ${configFilepath} already exists, save config to ${newConfigFilepath}`); 49 | configFilepath = newConfigFilepath; 50 | } 51 | 52 | await fs.writeFile(configFilepath, data); 53 | }; 54 | 55 | export { 56 | getDefaultConfig, 57 | schema, 58 | mergeWithDefaults, 59 | validateConfig, 60 | getConfigFilepath, 61 | saveConfig, 62 | }; 63 | -------------------------------------------------------------------------------- /src/lib/config/schema.ts: -------------------------------------------------------------------------------- 1 | import joi = require('@hapi/joi'); 2 | 3 | // Network throttling presets 4 | const NETWORK_PRESETS = ['GPRS', 'Regular2G', 'Good2G', 'Regular3G', 'Good3G', 'Regular4G', 'DSL', 'WiFi']; 5 | 6 | // select wpr pairs methods 7 | // select wpr pairs after recordWpr stage 8 | const SELECT_WPR_METHODS = [ 9 | 'simple', // select pairs as recorded by wprArchiveId 10 | 'closestByWprSize', // select more closer WPRs by size diff in absolute value 11 | 'closestByHtmlSize', // select WPR pairs with most closer html size from backend in absolute value 12 | 'closestByScriptSize', // select WPR pairs with most closer script size in backend html in absolute value 13 | ]; 14 | 15 | const AGGREGATIONS = [ 16 | 'count', // count metric values 17 | 'q50', // percentiles 18 | 'q80', 19 | 'q95', 20 | 'stdev', // metric standard deviation 21 | ]; 22 | 23 | const schema = joi.object().required().keys({ 24 | logLevel: joi.string().default('info') 25 | .valid('trace', 'debug', 'info', 'warn', 'error', 'fatal'), 26 | workDir: joi.string().required(), // ---------------------------------- workDir for WPRs, logs, screenshots, reports 27 | mode: joi.string().required().valid('puppeteer', 'webdriver'), 28 | iterations: joi.number().integer().min(1), // --------- how many iterations on compare 29 | workers: joi.number().integer().min(1), 30 | pageTimeout: joi.number().integer().min(0), 31 | withoutUi: joi.boolean().default(false), 32 | puppeteerOptions: joi.object().required().keys({ // ---------------------------- 33 | headless: joi.boolean().default(true), // ------------------------- start headless chromium 34 | ignoreHTTPSErrors: joi.boolean().default(false), 35 | useWpr: joi.boolean().default(true), // --------------------------- use WPR or realtime compare 36 | recordWprCount: joi.number().integer().min(1).default(10), // -------- how many WPR archives collect 37 | selectWprCount: joi.number().integer().min(1).default(1), // ---------- how many WPR pairs select from recorded 38 | // using options.selectWprMethod 39 | selectWprMethod: joi.string().valid(...SELECT_WPR_METHODS) // ----- how select WPR pairs, see SELECT_WPR_METHODS 40 | .default('HtmlSizeCloser'), // for details 41 | cacheEnabled: joi.boolean().default(true), // --------------------- browser cache enabled on comparison 42 | cpuThrottling: joi.object().allow(null).keys({ // ----------------------------- CPU throttling options 43 | rate: joi.number().integer().min(0), // ----------------------- CPU throttling rate 44 | }), 45 | 46 | // Network throttling, see NETWORK_PRESETS 47 | networkThrottling: joi.string().valid(...NETWORK_PRESETS).allow(null), 48 | 49 | singleProcess: joi.boolean().default(false), // ------------------- compare with single browser 50 | imagesEnabled: joi.boolean().default(true), // -------------------- images enabled 51 | javascriptEnabled: joi.boolean().default(true), // ---------------- javascript enabled 52 | cssFilesEnabled: joi.boolean().default(true), // ------------------ css files enabled, 53 | // ! slow down comparison speed 54 | // because use puppeteer request interception 55 | pageNavigationTimeout: joi.number().integer().min(0).default(60000), 56 | 57 | // puppeteer waitUntil option for page.open 58 | waitUntil: joi.string().required().valid('load', 'domcontentloaded', 'networkidle0', 'networkidle2'), 59 | retryCount: joi.number().integer().min(0).default(10), 60 | 61 | warmIterations: joi.number().integer().min(0).default(1), 62 | }), 63 | webdriverOptions: joi.object().keys({ 64 | host: joi.string().required(), 65 | port: joi.number().integer().min(0).required(), 66 | user: joi.string().allow(''), 67 | key: joi.string().allow(''), 68 | desiredCapabilities: joi.object().keys({ 69 | browserName: joi.string(), 70 | version: joi.string(), 71 | }), 72 | }), 73 | browserProfile: joi.object().keys({ 74 | mobile: joi.boolean().default(false), // -------------------------- use default mobile userAgent and viewport 75 | userAgent: joi.string().allow(null), 76 | height: joi.number().integer().min(0).allow(null), 77 | width: joi.number().integer().min(0).allow(null), 78 | }), 79 | comparisons: joi.array().required().min(1).items( // ----------------- named comparisons with site urls 80 | joi.object().required().keys({ // see config.example.js 81 | name: joi.string().required(), 82 | sites: joi.array().required().min(1) 83 | .items(joi.object().required().keys({ 84 | name: joi.string().required(), 85 | url: joi.string().required(), 86 | })), 87 | }), 88 | ), 89 | stages: joi.object().required().keys({ // ---------------------------- enable/disable compare stages 90 | recordWpr: joi.boolean().default(true), // 91 | compareMetrics: joi.boolean().default(true), // ------------------ compare performance metrics 92 | compareLighthouse: joi.boolean().default(false), // -------------- compare lighthouse scores and metrics 93 | }), 94 | metrics: joi.array().min(1).items( // -------------------------------- page performance and custom metrics 95 | // see https://developer.mozilla.org/en-US/docs/Web/API/PerformanceTiming, 96 | // https://developer.mozilla.org/en-US/docs/Web/API/PerformancePaintTiming 97 | // and hooks onCollectMetrics 98 | joi.object().keys({ 99 | name: joi.string().required(), // metric name, domContentLoadedEventEnd for example 100 | title: joi.string(), // metric title for table view, DCL for example 101 | showInTable: joi.boolean(), 102 | }), 103 | ), 104 | metricAggregations: joi.array().min(1).items( // metric aggregations, applied for every metric 105 | // you can include or exclude metrics from aggregation 106 | joi.object().keys({ 107 | name: joi.string().required() // see AGGREGATIONS 108 | .valid(...AGGREGATIONS), 109 | 110 | includeMetrics: joi.array().items(joi.string()), 111 | excludeMetrics: joi.array().items(joi.string()), 112 | }), 113 | ), 114 | hooks: joi.object().keys({ // hooks 115 | 116 | onVerifyWpr: joi.func(), // onVerifyWpr: ({ 117 | // logger: Logger, 118 | // page: Puppeteer.Page, 119 | // comparison: IComparison, 120 | // site: ISite 121 | // }) => Promise 122 | // called after WPR was collected 123 | // here you can check page loaded correctly or not, 124 | // throw error if page incorrect -- WPR record will be retried once 125 | // see config.example.js 126 | 127 | onCollectMetrics: joi.func(), // onCollectMetrics: ({ 128 | // logger: Logger, 129 | // page: Puppeteer.Page, 130 | // comparison: IComparison, 131 | // site: ISite 132 | // ) => Promise 133 | // called after page loaded in comparison 134 | // return object with custom metrics 135 | // see config.example.js 136 | }), 137 | }); 138 | 139 | export default schema; 140 | -------------------------------------------------------------------------------- /src/lib/config/types.ts: -------------------------------------------------------------------------------- 1 | import puppeteer from 'puppeteer'; 2 | 3 | import {Logger} from '@/lib/logger'; 4 | import {IPageStructureSizes} from '@/lib/puppeteer/types'; 5 | import {IWprArchive} from '@/lib/wpr/types'; 6 | import {ISite, RecursivePartial} from '@/types'; 7 | 8 | export enum SelectWprMethods { 9 | simple = 'simple', 10 | closestByWprSize = 'closestByWprSize', 11 | // medianByWprSize = 'medianByWprSize', 12 | closestByHtmlSize = 'closestByHtmlSize', 13 | closestByScriptSize = 'closestByScriptSize', 14 | } 15 | 16 | export type NetworkProfiles = 'GPRS' | 'Regular2G' | 'Good2G' | 'Regular3G' | 'Good3G' | 'Regular4G' | 'DSL' | 'WiFi'; 17 | 18 | export interface IPuppeteerOptions { 19 | headless: boolean; 20 | ignoreHTTPSErrors: boolean; 21 | useWpr: boolean; 22 | recordWprCount: number; 23 | selectWprCount: number; 24 | selectWprMethod: SelectWprMethods; 25 | cacheEnabled: boolean; 26 | cpuThrottling: null | { 27 | rate: number; 28 | }; 29 | networkThrottling: NetworkProfiles | null; 30 | imagesEnabled: boolean; 31 | javascriptEnabled: boolean; 32 | cssFilesEnabled: boolean; 33 | pageNavigationTimeout: number; 34 | waitUntil: 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2'; 35 | retryCount: number; 36 | warmIterations: number; 37 | } 38 | 39 | export interface IWebdriverOptions { 40 | host: string; 41 | port: number; 42 | user: string; 43 | key: string; 44 | desiredCapabilities: { 45 | browserName: string; 46 | version: string; 47 | }; 48 | } 49 | 50 | export interface IBrowserProfile { 51 | mobile: boolean; 52 | userAgent: string | null; 53 | width: number; 54 | height: number; 55 | } 56 | 57 | export interface IComparison { 58 | name: string; 59 | sites: ISite[]; 60 | wprArchives?: IWprArchive[]; 61 | } 62 | 63 | export interface IConfigMetric { 64 | name: string; 65 | title?: string; 66 | showInTable?: boolean; 67 | } 68 | 69 | export interface IConfigMetricsAggregation { 70 | name: 'q50' | 'q80' | 'q95' | 'stdev' | 'count'; 71 | includeMetrics?: string[]; 72 | excludeMetrics?: string[]; 73 | } 74 | 75 | export interface IHookObject { 76 | logger: Logger; 77 | page: puppeteer.Page; 78 | comparison: IComparison; 79 | site: ISite; 80 | } 81 | 82 | export type VerifyWprHook = (hook: IHookObject) => Promise; 83 | export type CollectMetricsHook = (hook: IHookObject) => Promise<{[index: string]: number}>; 84 | 85 | // TODO node type 86 | export type PageStructureSizesNodeHook = (sizes: IPageStructureSizes, node: any) => void; 87 | 88 | export type PageStructureSizesCompleteHook = ( 89 | sizes: IPageStructureSizes, 90 | html: string, 91 | getSizeInBytes: (html: string, start: number, end: number) => number, 92 | ) => void; 93 | 94 | export interface IConfigHooks { 95 | onVerifyWpr?: VerifyWprHook; 96 | onCollectMetrics?: CollectMetricsHook; 97 | onPageStructureSizesNode?: PageStructureSizesNodeHook; 98 | onPageStructureSizesComplete?: PageStructureSizesCompleteHook; 99 | } 100 | 101 | export interface IConfig { 102 | logLevel: 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal'; 103 | workDir: string; 104 | mode: 'puppeteer' | 'webdriver'; 105 | iterations: number; 106 | workers: number; 107 | pageTimeout: number; 108 | withoutUi: boolean; 109 | puppeteerOptions: IPuppeteerOptions; 110 | webdriverOptions: IWebdriverOptions; 111 | browserProfile: IBrowserProfile; 112 | comparisons: IComparison[]; 113 | stages: { 114 | recordWpr: boolean; 115 | compareMetrics: boolean; 116 | compareLighthouse: boolean; 117 | }; 118 | metrics: IConfigMetric[]; 119 | metricAggregations: IConfigMetricsAggregation[]; 120 | hooks: IConfigHooks; 121 | } 122 | 123 | export type IPartialConfig = RecursivePartial; 124 | 125 | export interface IPuppeteerConfig extends IConfig { 126 | puppeteerOptions: IPuppeteerOptions; 127 | } 128 | -------------------------------------------------------------------------------- /src/lib/dataProcessor/__spec__/dataProcessor.spec.ts: -------------------------------------------------------------------------------- 1 | import { IComparison, IConfig } from '@/lib/config'; 2 | import getDefaultConfig from '@/lib/config/default'; 3 | import { DataProcessor } from '..'; 4 | import { jsonReportResult, rawMetrics, sites } from './mock'; 5 | 6 | jest.mock('@/lib/logger', () => ({ 7 | getLogger: jest.fn().mockImplementation(() => ({ 8 | log: () => undefined, 9 | trace: () => undefined, 10 | debug: () => undefined, 11 | info: () => undefined, 12 | warn: () => undefined, 13 | error: () => undefined, 14 | fatal: () => undefined, 15 | })), 16 | })); 17 | 18 | describe('DataProcessor:', () => { 19 | let config: IConfig; 20 | let comparision: IComparison; 21 | let dataProccessor: DataProcessor; 22 | 23 | beforeAll(() => { 24 | [config, comparision] = prepareConfigAndComparision(); 25 | dataProccessor = new DataProcessor(config, comparision); 26 | prepareDataInDataProcessor(dataProccessor); 27 | }); 28 | 29 | describe('calcReport', () => { 30 | it ('prepare raw json report', async () => { 31 | const report = await dataProccessor.calcReport(sites); 32 | 33 | expect(report).toEqual(jsonReportResult); 34 | }); 35 | }); 36 | }); 37 | 38 | const prepareConfigAndComparision = (): [IConfig, IComparison] => { 39 | const config: IConfig = getDefaultConfig(); 40 | const comparision: IComparison = { 41 | name: 'compare', 42 | sites, 43 | }; 44 | 45 | config.comparisons.push(comparision); 46 | 47 | return [config, comparision]; 48 | }; 49 | 50 | const prepareDataInDataProcessor = (dataProccessor: DataProcessor) => { 51 | for (const [siteIndex, metrics] of rawMetrics.entries()) { 52 | 53 | // Transposing back raw data of dataProcessor to emulate working of workerFarm 54 | const transposed: number[][] = []; 55 | for (const metricValues of metrics) { 56 | for (const [metricValueIndex, metricValue] of metricValues.entries()) { 57 | if (!transposed[metricValueIndex]) { 58 | transposed.push([]); 59 | } 60 | transposed[metricValueIndex].push(metricValue); 61 | } 62 | } 63 | 64 | for (const measurement of transposed) { 65 | dataProccessor.registerMetrics(siteIndex, measurement); 66 | } 67 | } 68 | 69 | dataProccessor.calculateResults(); 70 | }; 71 | -------------------------------------------------------------------------------- /src/lib/dataProcessor/__spec__/mock.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | 3 | import { IJsonRawReport } from '@/types'; 4 | 5 | export const sites = [ 6 | { 7 | "name": "site0", 8 | "url": "https://goodOldRockNRoll.dead" 9 | }, 10 | { 11 | "name": "site1", 12 | "url": "https://iDontCare.wohoho" 13 | } 14 | ]; 15 | 16 | /** Raw Metrics */ 17 | export const rawMetrics = [ 18 | [ 19 | [ 20 | 9.789999981876463, 21 | 0.6899999571032822, 22 | 10.089999996125698, 23 | 0.7000000332482159, 24 | ], 25 | [ 26 | 35.435000027064234, 27 | 22.594999987632036, 28 | 34.63999996893108, 29 | 22.015000053215772, 30 | ], 31 | [68.91500001074746, 51.95499997353181, 65.7449999707751, 50.81500002415851], 32 | [ 33 | 457.43499998934567, 34 | 171.13499995321035, 35 | 316.1149999941699, 36 | 177.55500000203028, 37 | ], 38 | [ 39 | 457.43499998934567, 40 | 171.13499995321035, 41 | 316.1149999941699, 42 | 177.55500000203028, 43 | ], 44 | [964.1500000143424, 874.7599999769591, 1054.00499998359, 894.875000056345], 45 | [ 46 | 1463.3599999942817, 47 | 1359.8499999498017, 48 | 1509.2799999983981, 49 | 1152.4800000479445, 50 | ], 51 | [ 52 | 456.8100000033155, 53 | 413.46999997040257, 54 | 576.5999999712221, 55 | 425.7950000464916, 56 | ], 57 | [ 58 | 1457.0850000018254, 59 | 1353.464999992866, 60 | 1503.1299999682233, 61 | 1146.4750000159256, 62 | ], 63 | [135062, 135062, 134560, 134560], 64 | [131372, 131372, 130807, 130807], 65 | [957284, 957284, 953257, 953257], 66 | ], 67 | [ 68 | [ 69 | 10.565000004135072, 70 | 0.754999986384064, 71 | 11.140000016894192, 72 | 0.7200000109151006, 73 | ], 74 | [ 75 | 35.66500003216788, 76 | 24.42999999038875, 77 | 37.1199999935925, 78 | 22.409999975934625, 79 | ], 80 | [ 81 | 67.19500001054257, 82 | 53.88000002130866, 83 | 66.53000001097098, 84 | 51.98499996913597, 85 | ], 86 | [ 87 | 375.47000002814457, 88 | 151.86000004177913, 89 | 389.54499998362735, 90 | 174.5750000118278, 91 | ], 92 | [ 93 | 375.47000002814457, 94 | 151.86000004177913, 95 | 389.54499998362735, 96 | 174.5750000118278, 97 | ], 98 | [ 99 | 996.2400000076741, 100 | 900.4950000089593, 101 | 934.8899999749847, 102 | 872.6049999822862, 103 | ], 104 | [ 105 | 1491.7150000110269, 106 | 1182.4900000356138, 107 | 1438.704999978654, 108 | 1169.2799999727868, 109 | ], 110 | [ 111 | 507.4449999956414, 112 | 448.1699999887496, 113 | 430.3749999962747, 114 | 408.99999998509884, 115 | ], 116 | [ 117 | 1485.6150000123307, 118 | 1176.8100000335835, 119 | 1432.5650000246242, 120 | 1162.9800000227988, 121 | ], 122 | [135408, 135408, 135485, 135485], 123 | [131662, 131662, 131807, 131807], 124 | [957683, 957683, 953658, 953658], 125 | ], 126 | ]; 127 | 128 | 129 | 130 | export const jsonReportResult: IJsonRawReport = { 131 | "sites": [ 132 | { 133 | "name": "site0", 134 | "url": "https://goodOldRockNRoll.dead", 135 | }, 136 | { 137 | "name": "site1", 138 | "url": "https://iDontCare.wohoho", 139 | } 140 | ], 141 | "metrics": [ 142 | { 143 | "name": "requestStart" 144 | }, 145 | { 146 | "name": "responseStart", 147 | "title": "TTFB" 148 | }, 149 | { 150 | "name": "responseEnd", 151 | "title": "TTLB" 152 | }, 153 | { 154 | "name": "first-paint" 155 | }, 156 | { 157 | "name": "first-contentful-paint", 158 | "title": "FCP" 159 | }, 160 | { 161 | "name": "domContentLoadedEventEnd", 162 | "title": "DCL" 163 | }, 164 | { 165 | "name": "loadEventEnd", 166 | "title": "loaded" 167 | }, 168 | { 169 | "name": "domInteractive" 170 | }, 171 | { 172 | "name": "domComplete" 173 | }, 174 | { 175 | "name": "transferSize" 176 | }, 177 | { 178 | "name": "encodedBodySize" 179 | }, 180 | { 181 | "name": "decodedBodySize" 182 | } 183 | ], 184 | "metricAggregations": [ 185 | { 186 | "name": "count", 187 | "includeMetrics": [ 188 | "requestStart" 189 | ] 190 | }, 191 | { 192 | "name": "q50" 193 | }, 194 | { 195 | "name": "q80" 196 | }, 197 | { 198 | "name": "q95" 199 | } 200 | ], 201 | "data": { 202 | "allMetrics": { 203 | "decodedBodySize": { 204 | "site0": [ 205 | 957284, 206 | 957284, 207 | 953257, 208 | 953257, 209 | ], 210 | "site1": [ 211 | 957683, 212 | 957683, 213 | 953658, 214 | 953658, 215 | ], 216 | }, 217 | "domComplete": { 218 | "site0": [ 219 | 1457.0850000018254, 220 | 1353.464999992866, 221 | 1503.1299999682233, 222 | 1146.4750000159256, 223 | ], 224 | "site1": [ 225 | 1485.6150000123307, 226 | 1176.8100000335835, 227 | 1432.5650000246242, 228 | 1162.9800000227988, 229 | ], 230 | }, 231 | "domContentLoadedEventEnd": { 232 | "site0": [ 233 | 964.1500000143424, 234 | 874.7599999769591, 235 | 1054.00499998359, 236 | 894.875000056345, 237 | ], 238 | "site1": [ 239 | 996.2400000076741, 240 | 900.4950000089593, 241 | 934.8899999749847, 242 | 872.6049999822862, 243 | ], 244 | }, 245 | "domInteractive": { 246 | "site0": [ 247 | 456.8100000033155, 248 | 413.46999997040257, 249 | 576.5999999712221, 250 | 425.7950000464916, 251 | ], 252 | "site1": [ 253 | 507.4449999956414, 254 | 448.1699999887496, 255 | 430.3749999962747, 256 | 408.99999998509884, 257 | ], 258 | }, 259 | "encodedBodySize": { 260 | "site0": [ 261 | 131372, 262 | 131372, 263 | 130807, 264 | 130807, 265 | ], 266 | "site1": [ 267 | 131662, 268 | 131662, 269 | 131807, 270 | 131807, 271 | ], 272 | }, 273 | "first-contentful-paint": { 274 | "site0": [ 275 | 457.43499998934567, 276 | 171.13499995321035, 277 | 316.1149999941699, 278 | 177.55500000203028, 279 | ], 280 | "site1": [ 281 | 375.47000002814457, 282 | 151.86000004177913, 283 | 389.54499998362735, 284 | 174.5750000118278, 285 | ], 286 | }, 287 | "first-paint": { 288 | "site0": [ 289 | 457.43499998934567, 290 | 171.13499995321035, 291 | 316.1149999941699, 292 | 177.55500000203028, 293 | ], 294 | "site1": [ 295 | 375.47000002814457, 296 | 151.86000004177913, 297 | 389.54499998362735, 298 | 174.5750000118278, 299 | ], 300 | }, 301 | "loadEventEnd": { 302 | "site0": [ 303 | 1463.3599999942817, 304 | 1359.8499999498017, 305 | 1509.2799999983981, 306 | 1152.4800000479445, 307 | ], 308 | "site1": [ 309 | 1491.7150000110269, 310 | 1182.4900000356138, 311 | 1438.704999978654, 312 | 1169.2799999727868, 313 | ], 314 | }, 315 | "requestStart": { 316 | "site0": [ 317 | 9.789999981876463, 318 | 0.6899999571032822, 319 | 10.089999996125698, 320 | 0.7000000332482159, 321 | ], 322 | "site1": [ 323 | 10.565000004135072, 324 | 0.754999986384064, 325 | 11.140000016894192, 326 | 0.7200000109151006, 327 | ], 328 | }, 329 | "responseEnd": { 330 | "site0": [ 331 | 68.91500001074746, 332 | 51.95499997353181, 333 | 65.7449999707751, 334 | 50.81500002415851, 335 | ], 336 | "site1": [ 337 | 67.19500001054257, 338 | 53.88000002130866, 339 | 66.53000001097098, 340 | 51.98499996913597, 341 | ], 342 | }, 343 | "responseStart": { 344 | "site0": [ 345 | 35.435000027064234, 346 | 22.594999987632036, 347 | 34.63999996893108, 348 | 22.015000053215772, 349 | ], 350 | "site1": [ 351 | 35.66500003216788, 352 | 24.42999999038875, 353 | 37.1199999935925, 354 | 22.409999975934625, 355 | ], 356 | }, 357 | "transferSize": { 358 | "site0": [ 359 | 135062, 360 | 135062, 361 | 134560, 362 | 134560, 363 | ], 364 | "site1": [ 365 | 135408, 366 | 135408, 367 | 135485, 368 | 135485, 369 | ], 370 | }, 371 | }, 372 | "metrics": { 373 | "requestStart": { 374 | "count": { 375 | "site0": 4, 376 | "site1": 4 377 | }, 378 | "q50": { 379 | "site0": 5.245000007562339, 380 | "site1": 5.659999995259568 381 | }, 382 | "q80": { 383 | "site0": 9.789999981876463, 384 | "site1": 10.565000004135072 385 | }, 386 | "q95": { 387 | "site0": 9.789999981876463, 388 | "site1": 10.565000004135072 389 | } 390 | }, 391 | "responseStart": { 392 | "q50": { 393 | "site0": 28.617499978281558, 394 | "site1": 30.047500011278316 395 | }, 396 | "q80": { 397 | "site0": 34.63999996893108, 398 | "site1": 35.66500003216788 399 | }, 400 | "q95": { 401 | "site0": 34.63999996893108, 402 | "site1": 35.66500003216788 403 | } 404 | }, 405 | "responseEnd": { 406 | "q50": { 407 | "site0": 58.849999972153455, 408 | "site1": 60.20500001613982 409 | }, 410 | "q80": { 411 | "site0": 65.7449999707751, 412 | "site1": 66.53000001097098 413 | }, 414 | "q95": { 415 | "site0": 65.7449999707751, 416 | "site1": 66.53000001097098 417 | } 418 | }, 419 | "first-paint": { 420 | "q50": { 421 | "site0": 246.8349999981001, 422 | "site1": 275.0225000199862 423 | }, 424 | "q80": { 425 | "site0": 316.1149999941699, 426 | "site1": 375.47000002814457 427 | }, 428 | "q95": { 429 | "site0": 316.1149999941699, 430 | "site1": 375.47000002814457 431 | } 432 | }, 433 | "first-contentful-paint": { 434 | "q50": { 435 | "site0": 246.8349999981001, 436 | "site1": 275.0225000199862 437 | }, 438 | "q80": { 439 | "site0": 316.1149999941699, 440 | "site1": 375.47000002814457 441 | }, 442 | "q95": { 443 | "site0": 316.1149999941699, 444 | "site1": 375.47000002814457 445 | } 446 | }, 447 | "domContentLoadedEventEnd": { 448 | "q50": { 449 | "site0": 929.5125000353437, 450 | "site1": 917.692499991972 451 | }, 452 | "q80": { 453 | "site0": 964.1500000143424, 454 | "site1": 934.8899999749847 455 | }, 456 | "q95": { 457 | "site0": 964.1500000143424, 458 | "site1": 934.8899999749847 459 | } 460 | }, 461 | "loadEventEnd": { 462 | "q50": { 463 | "site0": 1411.6049999720417, 464 | "site1": 1310.597500007134 465 | }, 466 | "q80": { 467 | "site0": 1463.3599999942817, 468 | "site1": 1438.704999978654 469 | }, 470 | "q95": { 471 | "site0": 1463.3599999942817, 472 | "site1": 1438.704999978654 473 | } 474 | }, 475 | "domInteractive": { 476 | "q50": { 477 | "site0": 441.30250002490357, 478 | "site1": 439.27249999251217 479 | }, 480 | "q80": { 481 | "site0": 456.8100000033155, 482 | "site1": 448.1699999887496 483 | }, 484 | "q95": { 485 | "site0": 456.8100000033155, 486 | "site1": 448.1699999887496 487 | } 488 | }, 489 | "domComplete": { 490 | "q50": { 491 | "site0": 1405.2749999973457, 492 | "site1": 1304.6875000291038 493 | }, 494 | "q80": { 495 | "site0": 1457.0850000018254, 496 | "site1": 1432.5650000246242 497 | }, 498 | "q95": { 499 | "site0": 1457.0850000018254, 500 | "site1": 1432.5650000246242 501 | } 502 | }, 503 | "transferSize": { 504 | "q50": { 505 | "site0": 134811, 506 | "site1": 135446.5 507 | }, 508 | "q80": { 509 | "site0": 135062, 510 | "site1": 135485 511 | }, 512 | "q95": { 513 | "site0": 135062, 514 | "site1": 135485 515 | } 516 | }, 517 | "encodedBodySize": { 518 | "q50": { 519 | "site0": 131089.5, 520 | "site1": 131734.5 521 | }, 522 | "q80": { 523 | "site0": 131372, 524 | "site1": 131807 525 | }, 526 | "q95": { 527 | "site0": 131372, 528 | "site1": 131807 529 | } 530 | }, 531 | "decodedBodySize": { 532 | "q50": { 533 | "site0": 955270.5, 534 | "site1": 955670.5 535 | }, 536 | "q80": { 537 | "site0": 957284, 538 | "site1": 957683 539 | }, 540 | "q95": { 541 | "site0": 957284, 542 | "site1": 957683 543 | } 544 | } 545 | }, 546 | "diffs": { 547 | "requestStart": { 548 | "count": { 549 | "site1": 0 550 | }, 551 | "q50": { 552 | "site1": 0.4149999876972288 553 | }, 554 | "q80": { 555 | "site1": 0.7750000222586095 556 | }, 557 | "q95": { 558 | "site1": 0.7750000222586095 559 | } 560 | }, 561 | "responseStart": { 562 | "q50": { 563 | "site1": 1.4300000329967588 564 | }, 565 | "q80": { 566 | "site1": 1.0250000632368028 567 | }, 568 | "q95": { 569 | "site1": 1.0250000632368028 570 | } 571 | }, 572 | "responseEnd": { 573 | "q50": { 574 | "site1": 1.3550000439863652 575 | }, 576 | "q80": { 577 | "site1": 0.7850000401958823 578 | }, 579 | "q95": { 580 | "site1": 0.7850000401958823 581 | } 582 | }, 583 | "first-paint": { 584 | "q50": { 585 | "site1": 28.18750002188608 586 | }, 587 | "q80": { 588 | "site1": 59.35500003397465 589 | }, 590 | "q95": { 591 | "site1": 59.35500003397465 592 | } 593 | }, 594 | "first-contentful-paint": { 595 | "q50": { 596 | "site1": 28.18750002188608 597 | }, 598 | "q80": { 599 | "site1": 59.35500003397465 600 | }, 601 | "q95": { 602 | "site1": 59.35500003397465 603 | } 604 | }, 605 | "domContentLoadedEventEnd": { 606 | "q50": { 607 | "site1": -11.820000043371692 608 | }, 609 | "q80": { 610 | "site1": -29.260000039357692 611 | }, 612 | "q95": { 613 | "site1": -29.260000039357692 614 | } 615 | }, 616 | "loadEventEnd": { 617 | "q50": { 618 | "site1": -101.00749996490777 619 | }, 620 | "q80": { 621 | "site1": -24.655000015627593 622 | }, 623 | "q95": { 624 | "site1": -24.655000015627593 625 | } 626 | }, 627 | "domInteractive": { 628 | "q50": { 629 | "site1": -2.030000032391399 630 | }, 631 | "q80": { 632 | "site1": -8.640000014565885 633 | }, 634 | "q95": { 635 | "site1": -8.640000014565885 636 | } 637 | }, 638 | "domComplete": { 639 | "q50": { 640 | "site1": -100.5874999682419 641 | }, 642 | "q80": { 643 | "site1": -24.519999977201223 644 | }, 645 | "q95": { 646 | "site1": -24.519999977201223 647 | } 648 | }, 649 | "transferSize": { 650 | "q50": { 651 | "site1": 635.5 652 | }, 653 | "q80": { 654 | "site1": 423 655 | }, 656 | "q95": { 657 | "site1": 423 658 | } 659 | }, 660 | "encodedBodySize": { 661 | "q50": { 662 | "site1": 645 663 | }, 664 | "q80": { 665 | "site1": 435 666 | }, 667 | "q95": { 668 | "site1": 435 669 | } 670 | }, 671 | "decodedBodySize": { 672 | "q50": { 673 | "site1": 400 674 | }, 675 | "q80": { 676 | "site1": 399 677 | }, 678 | "q95": { 679 | "site1": 399 680 | } 681 | } 682 | } 683 | } 684 | } -------------------------------------------------------------------------------- /src/lib/dataProcessor/index.ts: -------------------------------------------------------------------------------- 1 | import {IComparison, IConfig, IConfigMetricsAggregation} from '@/lib/config'; 2 | import {roundToNDigits} from '@/lib/helpers'; 3 | import {calculatingStats} from '@/lib/stats'; 4 | import {ISite} from '@/types'; 5 | import colors from 'colors/safe'; 6 | import jstat from 'jstat'; 7 | 8 | import {getLogger} from '@/lib/logger'; 9 | import {IJsonRawReport, IJsonRawReportData, IMetric } from '@/types'; 10 | 11 | const logger = getLogger(); 12 | 13 | type Sites = string[]; 14 | type RawMetrics = (number)[][][]; 15 | type Stats = Array[][]; 16 | type Diffs = Array[][]; 17 | type Highlights = Array<-1|0|1|null>[][]; 18 | type PaintedMetrics = Array[][]; 19 | type PaintedDiffs = Array[][]; 20 | type Iterations = number[]; 21 | type ActiveTests = number[]; 22 | type StatArrays = Array[][]; 23 | 24 | export {IJsonRawReportData as IJsonReportData, IJsonRawReport as IJsonReport, IMetric}; 25 | 26 | export interface IMetrics { 27 | [index: string]: number[]; 28 | } 29 | 30 | export class DataProcessor { 31 | public sites: Sites; 32 | public comparision: IComparison; 33 | public config: IConfig; 34 | 35 | public rawMetrics: RawMetrics; 36 | public stats: Stats; 37 | public diffs: Diffs; 38 | public highlights: Highlights; 39 | public paintedMetrics: PaintedMetrics; 40 | public paintedDiffs: PaintedDiffs; 41 | public iterations: Iterations; 42 | public activeTests: ActiveTests; 43 | public calculationCache: { 44 | statArrays: StatArrays, 45 | }; 46 | protected _metrics: { 47 | [index: string]: IMetrics, // [index: SiteName] 48 | }; 49 | 50 | constructor(config: IConfig, comparision: IComparison) { 51 | this.sites = comparision.sites.map((site) => site.url); 52 | this.comparision = comparision; 53 | this.config = config; 54 | this._metrics = {}; 55 | 56 | // These are growing with each sample 57 | // this.rawMetrics[siteIndex][metricIndex] is array of all metric samples 58 | // it's length will grow up to maxIterations 59 | this.rawMetrics = []; 60 | 61 | // These are recalculated with each sample 62 | // this.stats[siteIndex][metricIndex][statIndex] stores value for each stat calculated 63 | this.stats = []; 64 | 65 | // Diffs from reference site (the first supplied). Recalculated with each sample 66 | // this.diffs[siteIndex][metricIndex][statIndex] 67 | this.diffs = []; 68 | 69 | // Highlits for each stat -1 will paint it red, 0: gray, 1: green 70 | // this.highlights[siteIndex][metricIndex][statIndex] 71 | this.highlights = []; 72 | 73 | // Metric strings after transformation and coloring. These will be shown in terminal 74 | // this.paintedMetrics[siteIndex][metricIndex][statIndex] 75 | this.paintedMetrics = []; 76 | 77 | // Diff strings after transformation and coloring. These will be shown in terminal 78 | // this.paintedDiffs[siteIndex][metricIndex][statIndex] 79 | this.paintedDiffs = []; 80 | 81 | // this.iterations[siteIndex] stores count of successful iterations for each site 82 | this.iterations = []; 83 | 84 | // this.activeTests[siteIndex] stores count of currently running tests for each site 85 | this.activeTests = []; 86 | 87 | for (let siteIndex = 0; siteIndex < this.sites.length; siteIndex++) { 88 | this.rawMetrics[siteIndex] = []; 89 | this.stats[siteIndex] = []; 90 | this.diffs[siteIndex] = []; 91 | this.highlights[siteIndex] = []; 92 | this.paintedMetrics[siteIndex] = []; 93 | this.paintedDiffs[siteIndex] = []; 94 | this.iterations[siteIndex] = 0; 95 | this.activeTests[siteIndex] = 0; 96 | 97 | for (let metricIndex = 0; metricIndex < this.config.metrics.length; metricIndex++) { 98 | this.rawMetrics[siteIndex][metricIndex] = []; 99 | this.stats[siteIndex][metricIndex] = []; 100 | this.diffs[siteIndex][metricIndex] = []; 101 | this.highlights[siteIndex][metricIndex] = []; 102 | this.paintedMetrics[siteIndex][metricIndex] = []; 103 | this.paintedDiffs[siteIndex][metricIndex] = []; 104 | } 105 | } 106 | 107 | this.calculationCache = {statArrays: []}; 108 | } 109 | 110 | // Takes metrics for site 111 | public registerMetrics(siteIndex: number, metricValues: number[]): void { 112 | const site = this.comparision.sites[siteIndex]; 113 | 114 | logger.trace(`dataProcessor.registerMetrics ${siteIndex} ${metricValues}`); 115 | 116 | for (const [metricIndex, metricValue] of metricValues.entries()) { 117 | logger.trace(`dataProcessor.registerMetrics: ${metricIndex} ${metricValue}`); 118 | this.rawMetrics[siteIndex][metricIndex].push(metricValue); 119 | 120 | const {name: metricName} = this.config.metrics[metricIndex]; 121 | 122 | logger.trace(`dataProcessor.registerMetrics: ${site.name} ${metricName}, ${metricValue}`); 123 | 124 | const metric = this._getSiteMetric(site.name, metricName); 125 | metric.push(metricValue); 126 | } 127 | 128 | this.iterations[siteIndex]++; 129 | } 130 | 131 | // Takes metrics for site 132 | public reportTestStart(siteIndex: number, job: Promise): void { 133 | this.activeTests[siteIndex]++; 134 | 135 | const jobCallback = () => this.activeTests[siteIndex]--; 136 | 137 | job.then(jobCallback, jobCallback); 138 | } 139 | 140 | // Does ALL calculations, returns ALL data 141 | public calculateResults(): { 142 | sites: Sites, 143 | stats: Stats, 144 | diffs: Diffs, 145 | highlights: Highlights, 146 | paintedMetrics: PaintedMetrics, 147 | paintedDiffs: PaintedDiffs, 148 | iterations: Iterations, 149 | activeTests: ActiveTests, 150 | } { 151 | 152 | this.resetCache(); 153 | 154 | this.calculateStats(); 155 | this.calculateDiffs(); 156 | this.calculateHighlits(); 157 | this.calculatePaintedMetrics(); 158 | this.calculatePaintedDiffs(); 159 | 160 | return { 161 | sites: this.sites, 162 | stats: this.stats, 163 | diffs: this.diffs, 164 | highlights: this.highlights, 165 | paintedMetrics: this.paintedMetrics, 166 | paintedDiffs: this.paintedDiffs, 167 | iterations: this.iterations, 168 | activeTests: this.activeTests, 169 | }; 170 | } 171 | 172 | public calculateStats() { 173 | // Calculating stats using only minIterations metric slice: every site gets equal chances 174 | const minIterations = Math.min(...this.iterations); 175 | 176 | for (let siteIndex = 0; siteIndex < this.sites.length; siteIndex++) { 177 | for (let metricIndex = 0; metricIndex < this.config.metrics.length; metricIndex++) { 178 | const values = this.rawMetrics[siteIndex][metricIndex].slice(0, minIterations); 179 | const referenceValues = this.rawMetrics[0][metricIndex].slice(0, minIterations); 180 | 181 | for (let statIndex = 0; statIndex < calculatingStats.length; statIndex++) { 182 | let res; 183 | if (values.length === 0) { 184 | res = null; 185 | } else if (siteIndex === 0 && !calculatingStats[statIndex].applicableToReference) { 186 | res = null; 187 | } else { 188 | const stat = calculatingStats[statIndex]; 189 | res = roundToNDigits(stat.calc(values, referenceValues), stat.roundDigits); 190 | } 191 | 192 | this.stats[siteIndex][metricIndex][statIndex] = res; 193 | } 194 | } 195 | } 196 | } 197 | 198 | public calculateDiffs() { 199 | for (let siteIndex = 0; siteIndex < this.sites.length; siteIndex++) { 200 | for (let metricIndex = 0; metricIndex < this.config.metrics.length; metricIndex++) { 201 | for (let statIndex = 0; statIndex < calculatingStats.length; statIndex++) { 202 | let res; 203 | 204 | const value = this.stats[siteIndex][metricIndex][statIndex]; 205 | const referenceValue = this.stats[0][metricIndex][statIndex]; 206 | const stat = calculatingStats[statIndex]; 207 | 208 | if (value === null || referenceValue === null) { 209 | res = null; 210 | } else if (!stat.diffAplicable) { 211 | res = null; 212 | } else if (siteIndex === 0 && !stat.applicableToReference) { 213 | res = null; 214 | } else { 215 | res = roundToNDigits(value - referenceValue, stat.roundDigits); 216 | } 217 | 218 | this.diffs[siteIndex][metricIndex][statIndex] = res; 219 | } 220 | } 221 | } 222 | } 223 | 224 | public calculateHighlits() { 225 | for (let metricIndex = 0; metricIndex < this.config.metrics.length; metricIndex++) { 226 | for (let statIndex = 0; statIndex < calculatingStats.length; statIndex++) { 227 | const statArray = this.getStatArray(statIndex, metricIndex); 228 | 229 | let paintArray = null; 230 | const stat = calculatingStats[statIndex]; 231 | paintArray = stat.paint(statArray); 232 | 233 | for (let siteIndex = 0; siteIndex < this.sites.length; siteIndex++) { 234 | this.highlights[siteIndex][metricIndex][statIndex] = 235 | paintArray === null ? 0 : paintArray[siteIndex]; 236 | } 237 | } 238 | } 239 | } 240 | 241 | public calculatePaintedMetrics() { 242 | for (let siteIndex = 0; siteIndex < this.sites.length; siteIndex++) { 243 | for (let metricIndex = 0; metricIndex < this.config.metrics.length; metricIndex++) { 244 | for (let statIndex = 0; statIndex < calculatingStats.length; statIndex++) { 245 | const itemPaint = this.highlights[siteIndex][metricIndex][statIndex]; 246 | const itemValue = this.stats[siteIndex][metricIndex][statIndex]; 247 | 248 | let res; 249 | 250 | if (itemValue === null) { 251 | res = ''; 252 | } else if (itemPaint === 1) { 253 | res = colors.green(itemValue.toString()); 254 | } else if (itemPaint === -1) { 255 | res = colors.red(itemValue.toString()); 256 | } else { 257 | res = itemValue.toString(); 258 | } 259 | 260 | this.paintedMetrics[siteIndex][metricIndex][statIndex] = res; 261 | } 262 | } 263 | } 264 | } 265 | 266 | public calculatePaintedDiffs() { 267 | for (let siteIndex = 0; siteIndex < this.sites.length; siteIndex++) { 268 | for (let metricIndex = 0; metricIndex < this.config.metrics.length; metricIndex++) { 269 | for (let statIndex = 0; statIndex < calculatingStats.length; statIndex++) { 270 | const diff = this.diffs[siteIndex][metricIndex][statIndex]; 271 | 272 | let res; 273 | 274 | if (siteIndex === 0 || diff === null) { 275 | res = ''; 276 | } else if (diff < 0) { 277 | res = diff.toString(); 278 | } else { 279 | res = '+' + diff; 280 | } 281 | 282 | this.paintedDiffs[siteIndex][metricIndex][statIndex] = res; 283 | } 284 | } 285 | } 286 | } 287 | 288 | public getStatArray(statIndex: number, metricIndex: number): Array { 289 | if (this.calculationCache.statArrays[statIndex][metricIndex]) { 290 | return this.calculationCache.statArrays[statIndex][metricIndex]; 291 | } 292 | 293 | const statArray: Array = []; 294 | 295 | for (let siteIndex = 0; siteIndex < this.sites.length; siteIndex++) { 296 | statArray[siteIndex] = this.stats[siteIndex][metricIndex][statIndex]; 297 | } 298 | 299 | this.calculationCache.statArrays[statIndex][metricIndex] = statArray; 300 | 301 | return statArray; 302 | } 303 | 304 | public resetCache() { 305 | this.calculationCache.statArrays = []; 306 | for (let statIndex = 0; statIndex < calculatingStats.length; statIndex++) { 307 | this.calculationCache.statArrays[statIndex] = []; 308 | } 309 | } 310 | 311 | // returns iteration count of least successful site 312 | public getLeastIterations(): number { 313 | return Math.min(...this.iterations); 314 | } 315 | 316 | public async calcReport(sites: ISite[]) { 317 | const siteNames = sites.map((site) => site.name); 318 | const jsonReportData: IJsonRawReportData = { 319 | metrics: {}, 320 | diffs: {}, 321 | allMetrics: {}, 322 | }; 323 | 324 | for (const metric of this.config.metrics) { 325 | const metricName = metric.name; 326 | 327 | jsonReportData.metrics[metricName] = {}; 328 | jsonReportData.diffs[metricName] = {}; 329 | jsonReportData.allMetrics[metricName] = {}; 330 | 331 | for (const aggregation of this.config.metricAggregations) { 332 | if (aggregation.includeMetrics && !aggregation.includeMetrics.includes(metricName)) { 333 | logger.trace(`includeMetrics: skip aggregation=${aggregation.name} for metric=${metricName}`); 334 | continue; 335 | } 336 | 337 | if (aggregation.excludeMetrics && aggregation.excludeMetrics.includes(metricName)) { 338 | logger.trace(`excludeMetrics: skip aggregation=${aggregation.name} for metric=${metricName}`); 339 | continue; 340 | } 341 | 342 | jsonReportData.metrics[metricName][aggregation.name] = {}; 343 | jsonReportData.diffs[metricName][aggregation.name] = {}; 344 | 345 | const values: number[] = []; 346 | 347 | for (const siteName of siteNames) { 348 | const metricValues = this._getSiteMetric(siteName, metricName); 349 | jsonReportData.allMetrics[metricName][siteName] = metricValues; 350 | 351 | logger.trace(`metricValues: ${metricName}, ${metricValues}`); 352 | 353 | const aggregated = this._calcAggregation(aggregation, metricName, metricValues); 354 | jsonReportData.metrics[metricName][aggregation.name][siteName] = aggregated; 355 | 356 | values.push(aggregated); 357 | } 358 | 359 | // add diff 360 | values.forEach((value, index) => { 361 | if (index === 0) { 362 | return; 363 | } 364 | 365 | const diff = value - values[0]; 366 | 367 | jsonReportData.diffs[metricName][aggregation.name][sites[index].name] = diff; 368 | }); 369 | } 370 | } 371 | 372 | const jsonReport: IJsonRawReport = { 373 | sites, 374 | metrics: this.config.metrics.map((m) => ({title: m.title, name: m.name})), 375 | metricAggregations: this.config.metricAggregations, 376 | data: jsonReportData, 377 | }; 378 | 379 | return jsonReport; 380 | } 381 | 382 | protected _getSiteMetric(siteName: string, metricName: string): number[] { 383 | if (!this._metrics[siteName]) { 384 | this._metrics[siteName] = {}; 385 | } 386 | 387 | if (!this._metrics[siteName][metricName]) { 388 | this._metrics[siteName][metricName] = []; 389 | } 390 | 391 | return this._metrics[siteName][metricName]; 392 | } 393 | 394 | protected _calcAggregation(aggregation: IConfigMetricsAggregation, metricName: string, metrics: number[]): number { 395 | logger.debug(`metric=${metricName}: calc aggregation=${aggregation}`); 396 | 397 | switch (aggregation.name) { 398 | case 'q50': 399 | return jstat.percentile(metrics, 0.5); 400 | case 'q80': 401 | return jstat.percentile(metrics, 0.8); 402 | case 'q95': 403 | return jstat.percentile(metrics, 0.95); 404 | case 'stdev': 405 | return jstat.stdev(metrics, true); 406 | case 'count': 407 | return metrics.length; 408 | default: 409 | throw new Error(`unknown aggregation ${aggregation}`); 410 | } 411 | } 412 | } 413 | -------------------------------------------------------------------------------- /src/lib/findFreePorts/findFreePort.ts: -------------------------------------------------------------------------------- 1 | // @see https://github.com/mhzed/find-free-port 2 | 3 | import net = require('net'); 4 | 5 | // call method 1: (port, cb(err, freePort)) 6 | // call method 2: (portBeg, portEnd, cb(err, freePort)) 7 | // call method 3: (portBeg, host, cb(err, freePort)) 8 | // call method 4: (portBeg, portEnd, host, cb(err, freePort)) 9 | // call method 5: (portBeg, portEnd, host, howmany, cb(err, freePort1, freePort2, ...)) 10 | 11 | function findFreePortCb(beg: number, ...rest: (string | number)[]) { 12 | const p = rest.slice(0, rest.length - 1); 13 | const cb = rest[rest.length - 1] as any as (err: any, port?: number) => void; 14 | 15 | const arr = Array.from(p); 16 | 17 | let end: number = Number(arr[0]); 18 | let ip: string = arr[1] as any as string; 19 | let cnt: number = Number(arr[2]); 20 | 21 | // let [end, ip, cnt] = Array.from(p); 22 | 23 | if (!ip && end && !/^\d+$/.test(String(end))) { // deal with method 3 24 | ip = String(end); 25 | end = 65534; 26 | } else { 27 | if (end == null) { end = 65534; } 28 | } 29 | if (cnt == null) { cnt = 1; } 30 | 31 | const retcb = cb; 32 | const res: number[] = []; 33 | 34 | const probe = function( 35 | probeIp: string, 36 | probePort: number, 37 | probeCb: (port: number | null, nextPort?: number, 38 | ) => void) { 39 | const s = net.createConnection({port: probePort, host: probeIp}); 40 | s.on('connect', function() { s.end(); probeCb(null, probePort + 1); }); 41 | s.on('error', (/* err */) => { probeCb(probePort); }); // can't connect, port is available 42 | }; 43 | const onprobe = function(port: number | null, nextPort?: number) { 44 | if (port) { 45 | res.push(port); 46 | if (res.length >= cnt) { 47 | retcb(null, ...res); 48 | } else { 49 | setImmediate(() => probe(ip, port + 1, onprobe)); 50 | } 51 | } else { 52 | if (Number(nextPort) >= end) { 53 | retcb(new Error('No available ports')); 54 | } else { 55 | setImmediate(() => probe(ip, Number(nextPort), onprobe)); 56 | } 57 | } 58 | }; 59 | return probe(ip, beg, onprobe); 60 | } 61 | 62 | // @ts-ignore 63 | function findFreePort(beg: number, ...rest: (string | number)[]) { 64 | const last = rest[rest.length - 1]; 65 | 66 | if (typeof last === 'function') { 67 | findFreePortCb(beg, ...rest); 68 | } else { 69 | return new Promise((resolve, reject) => { 70 | // @ts-ignore 71 | findFreePortCb(beg, ...rest, (err, ...ports) => { 72 | if (err) { 73 | reject(err); 74 | } else { 75 | resolve(ports); 76 | } 77 | }); 78 | }); 79 | } 80 | } 81 | export default findFreePort as any as ((begin: number, end: number, host: string, count: number) => Promise); 82 | -------------------------------------------------------------------------------- /src/lib/findFreePorts/index.ts: -------------------------------------------------------------------------------- 1 | import findFreePort from '@/lib/findFreePorts/findFreePort'; 2 | import {getLogger} from '@/lib/logger'; 3 | 4 | const FIND_PORT_BEGIN_PORT_DEFAULT = 10000; 5 | const FIND_PORT_END_PORT_DEFAULT = 12000; 6 | const FIND_PORT_STEP = 2; 7 | 8 | const config = { 9 | beginPort: FIND_PORT_BEGIN_PORT_DEFAULT, 10 | endPort: FIND_PORT_END_PORT_DEFAULT, 11 | host: 'localhost', 12 | count: 2, 13 | }; 14 | 15 | const logger = getLogger(); 16 | 17 | export async function findTwoFreePorts(): Promise { 18 | logger.debug(`search free ports`, config); 19 | 20 | config.beginPort += FIND_PORT_STEP; 21 | 22 | logger.debug( 23 | `moving config.findFreePort.beginPort + ${FIND_PORT_STEP}`, 24 | config.beginPort, 25 | ); 26 | 27 | if (config.beginPort > config.endPort) { 28 | logger.debug('reset config.findFreePort.beginPort to default', FIND_PORT_BEGIN_PORT_DEFAULT); 29 | config.beginPort = FIND_PORT_BEGIN_PORT_DEFAULT; 30 | } 31 | 32 | const beginPort = config.beginPort; 33 | const endPort = config.endPort; 34 | 35 | const ports = await findFreePort( 36 | beginPort, 37 | endPort, 38 | config.host, 39 | config.count, 40 | ); 41 | 42 | logger.debug(`found free ports: ${ports}`); 43 | 44 | return ports; 45 | } 46 | -------------------------------------------------------------------------------- /src/lib/fs.ts: -------------------------------------------------------------------------------- 1 | import {IComparison} from '@/lib/config'; 2 | import {ISite} from '@/types'; 3 | import * as path from 'path'; 4 | 5 | export const getConfigFilepath = (workDir: string, name: string) => { 6 | return path.resolve(workDir, `${name}.config.json`); 7 | }; 8 | 9 | export const getComparisonDir = (workDir: string, comparison: IComparison) => { 10 | return path.resolve(workDir, comparison.name); 11 | }; 12 | 13 | export const getWprRecordStdoutFilepath = (workDir: string, site: ISite, id: number) => { 14 | return path.resolve(workDir, `${site.name}-${id}.wpr_record.stdout.log`); 15 | }; 16 | 17 | export const getWprRecordStderrFilepath = (workDir: string, site: ISite, id: number) => { 18 | return path.resolve(workDir, `${site.name}-${id}.wpr_record.stderr.log`); 19 | }; 20 | 21 | export const getWprRecordScreenshotFilepath = (workerDir: string, site: ISite, id: number) => { 22 | return path.resolve(workerDir, `${site.name}-${id}-wpr-record.png`); 23 | }; 24 | 25 | export const getWprReplayStdoutFilepath = (workDir: string, site: ISite, id: number, wprArchiveId: number) => { 26 | return path.resolve(workDir, `${site.name}-${id}-${wprArchiveId}.wpr_replay.stdout.log`); 27 | }; 28 | 29 | export const getWprReplayStderrFilepath = (workDir: string, site: ISite, id: number, wprArchiveId: number) => { 30 | return path.resolve(workDir, `${site.name}-${id}-${wprArchiveId}.wpr_replay.stderr.log`); 31 | }; 32 | 33 | export const getWprArchiveFilepath = (workDir: string, site: ISite, id: number) => { 34 | return path.resolve(workDir, `${site.name}-${id}.wprgo`); 35 | }; 36 | 37 | export const getPageStructureSizesFilepath = (workDir: string, site: ISite, wprArchiveId: number) => { 38 | return path.resolve(workDir, `${site.name}-${wprArchiveId}.page-structure-sizes.json`); 39 | }; 40 | 41 | export const getPageStructureSizesAfterLoadedFilepath = (workDir: string, site: ISite, wprArchiveId: number) => { 42 | return path.resolve(workDir, `${site.name}-${wprArchiveId}.page-structure-sizes-after-loaded.json`); 43 | }; 44 | -------------------------------------------------------------------------------- /src/lib/helpers.ts: -------------------------------------------------------------------------------- 1 | import assert from 'assert'; 2 | 3 | // https://stackoverflow.com/a/11301464/1958334 4 | export function indexOfMin(arr: number[]) { 5 | if (arr.length === 0) { 6 | return -1; 7 | } 8 | 9 | let min = arr[0]; 10 | let minIndex = 0; 11 | 12 | for (let i = 1; i < arr.length; i++) { 13 | if (arr[i] < min) { 14 | minIndex = i; 15 | min = arr[i]; 16 | } 17 | } 18 | 19 | return minIndex; 20 | } 21 | 22 | export function roundToNDigits(value: number, digits: number) { 23 | const rounder = Math.pow(10, digits); 24 | return Math.round(value * rounder) / rounder; 25 | } 26 | 27 | export function sleep(time: number): Promise { 28 | return new Promise((resolve) => { 29 | setTimeout(resolve, time); 30 | }); 31 | } 32 | 33 | export function hasOnlyNumbers(input: Array): input is number[] { 34 | return !input.some((el) => typeof el !== 'number'); 35 | } 36 | 37 | export function stdoutRect(): [number, number] { 38 | if (typeof process.stdout.columns !== 'number') { 39 | throw new Error('process.stdout.columns is not a number'); 40 | } 41 | 42 | if (typeof process.stdout.rows !== 'number') { 43 | throw new Error('process.stdout.rows is not a number'); 44 | } 45 | return [process.stdout.rows, process.stdout.columns]; 46 | } 47 | 48 | export function isInteractive(): boolean { 49 | return Boolean(process.stdout && process.stdout.isTTY && process.env.TERM !== 'dumb'); 50 | } 51 | 52 | export function assertNonNull(obj: T): asserts obj is NonNullable { 53 | assert.notEqual(obj, null); 54 | } 55 | 56 | export const isoDate = () => (new Date()).toISOString(); 57 | -------------------------------------------------------------------------------- /src/lib/logger.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | 3 | import {isInteractive as getIsInteractive} from '@/lib/helpers'; 4 | import * as fs from 'fs'; 5 | import * as tracer from 'tracer'; 6 | 7 | import {getViewConsole} from '@/lib/view'; 8 | 9 | const viewConsole = getViewConsole(); 10 | 11 | export type Logger = tracer.Tracer.Logger; 12 | 13 | let loggerInstance: Logger; 14 | 15 | export let logfilePath: string = path.resolve(process.cwd(), 'porchmark.log'); 16 | 17 | let logfileDescriptor: number | null = null; 18 | 19 | function closeFileDescriptor(descriptor: number | null): void { 20 | if (descriptor) { 21 | fs.closeSync(descriptor); 22 | } 23 | } 24 | 25 | process.on('beforeExit', function handleBeforeExit() { 26 | loggerInstance.info('exitHandler call'); 27 | closeFileDescriptor(logfileDescriptor); 28 | }); 29 | 30 | export const createLogger = (level: string = 'trace') => { 31 | const isInteractive = getIsInteractive(); 32 | const loggerCreator = isInteractive ? tracer.colorConsole : tracer.console; 33 | 34 | return loggerCreator({ 35 | level, 36 | format: [ 37 | '{{timestamp}} <{{title}}> {{message}}', 38 | { 39 | error: '{{timestamp}} <{{title}}> {{message}} (in {{file}}:{{line}})\nCall Stack:\n{{stack}}', 40 | }, 41 | ], 42 | dateformat: 'HH:MM:ss.L', 43 | transport(data) { 44 | if (!isInteractive) { 45 | process.stderr.write(data.rawoutput + '\n'); 46 | } 47 | 48 | viewConsole.info(data.output); 49 | 50 | if (logfilePath) { 51 | if (!logfileDescriptor) { 52 | logfileDescriptor = fs.openSync(logfilePath, 'a'); 53 | } 54 | 55 | fs.write(logfileDescriptor, data.rawoutput + '\n', (err) => { 56 | if (err) { throw err; } 57 | }); 58 | } 59 | }, 60 | }); 61 | }; 62 | 63 | export function setLogfilePath(filepath: string) { 64 | closeFileDescriptor(logfileDescriptor); 65 | logfilePath = filepath; 66 | logfileDescriptor = fs.openSync(logfilePath, 'a'); 67 | } 68 | 69 | export function setLogger(logger: Logger) { 70 | loggerInstance = logger; 71 | } 72 | 73 | export function getLogger() { 74 | if (!loggerInstance) { 75 | throw new Error('no global logger'); 76 | } 77 | return loggerInstance; 78 | } 79 | 80 | export function setLevel(level: string) { 81 | tracer.setLevel(level); 82 | } 83 | -------------------------------------------------------------------------------- /src/lib/puppeteer/browser.ts: -------------------------------------------------------------------------------- 1 | import puppeteer, {Browser} from 'puppeteer'; 2 | 3 | import {IPuppeteerConfig} from '@/lib/config/types'; 4 | import {IBrowserLaunchOptions} from '@/lib/puppeteer/types'; 5 | 6 | export const prepareBrowserLaunchOptions = (config: IPuppeteerConfig): IBrowserLaunchOptions => { 7 | const {headless, ignoreHTTPSErrors, imagesEnabled} = config.puppeteerOptions; 8 | return { 9 | headless, 10 | ignoreHTTPSErrors, 11 | imagesEnabled, 12 | wpr: null, 13 | }; 14 | }; 15 | 16 | export const launchBrowser = (options: IBrowserLaunchOptions): Promise => { 17 | const args: string[] = []; 18 | 19 | if (options.wpr) { 20 | args.push( 21 | '--ignore-certificate-errors-spki-list=PhrPvGIaAMmd29hj8BCZOq096yj7uMpRNHpn5PDxI6I=', 22 | // resolve all domains to 127.0.0.1 with WPR record or replay port 23 | `--host-resolver-rules="MAP *:80 127.0.0.1:${options.wpr.httpPort},` + 24 | `MAP *:443 127.0.0.1:${options.wpr.httpsPort},EXCLUDE localhost"`, 25 | ); 26 | } 27 | 28 | if (options.imagesEnabled === false) { 29 | args.push('--blink-settings=imagesEnabled=false'); 30 | } 31 | 32 | const launchOptions = { 33 | headless: options.headless, 34 | ignoreHTTPSErrors: options.ignoreHTTPSErrors, 35 | args, 36 | }; 37 | 38 | return puppeteer.launch(launchOptions); 39 | }; 40 | -------------------------------------------------------------------------------- /src/lib/puppeteer/index.ts: -------------------------------------------------------------------------------- 1 | export * from './browser'; 2 | export * from './page'; 3 | export * from './types'; 4 | export * from './metrics'; 5 | export * from './runCheck'; 6 | -------------------------------------------------------------------------------- /src/lib/puppeteer/metrics.ts: -------------------------------------------------------------------------------- 1 | import {IOriginalMetrics} from '@/types'; 2 | import puppeteer from 'puppeteer'; 3 | 4 | export const getPageMetrics = (page: puppeteer.Page): Promise => { 5 | return page.evaluate(() => { 6 | const timings = performance.getEntriesByType('navigation')[0].toJSON(); 7 | const paintEntries = performance.getEntriesByType('paint'); 8 | for (const entry of paintEntries) { 9 | timings[entry.name] = entry.startTime; 10 | } 11 | return timings; 12 | }); 13 | }; 14 | -------------------------------------------------------------------------------- /src/lib/puppeteer/networkPresets.ts: -------------------------------------------------------------------------------- 1 | const NETWORK_PRESETS = { 2 | GPRS: { 3 | offline: false, 4 | downloadThroughput: 50 * 1024 / 8, 5 | uploadThroughput: 20 * 1024 / 8, 6 | latency: 500, 7 | }, 8 | Regular2G: { 9 | offline: false, 10 | downloadThroughput: 250 * 1024 / 8, 11 | uploadThroughput: 50 * 1024 / 8, 12 | latency: 300, 13 | }, 14 | Good2G: { 15 | offline: false, 16 | downloadThroughput: 450 * 1024 / 8, 17 | uploadThroughput: 150 * 1024 / 8, 18 | latency: 150, 19 | }, 20 | Regular3G: { 21 | offline: false, 22 | downloadThroughput: 750 * 1024 / 8, 23 | uploadThroughput: 250 * 1024 / 8, 24 | latency: 100, 25 | }, 26 | Good3G: { 27 | offline: false, 28 | downloadThroughput: 1.5 * 1024 * 1024 / 8, 29 | uploadThroughput: 750 * 1024 / 8, 30 | latency: 40, 31 | }, 32 | Regular4G: { 33 | offline: false, 34 | downloadThroughput: 4 * 1024 * 1024 / 8, 35 | uploadThroughput: 3 * 1024 * 1024 / 8, 36 | latency: 20, 37 | }, 38 | DSL: { 39 | offline: false, 40 | downloadThroughput: 2 * 1024 * 1024 / 8, 41 | uploadThroughput: 1 * 1024 * 1024 / 8, 42 | latency: 5, 43 | }, 44 | WiFi: { 45 | offline: false, 46 | downloadThroughput: 30 * 1024 * 1024 / 8, 47 | uploadThroughput: 15 * 1024 * 1024 / 8, 48 | latency: 2, 49 | }, 50 | }; 51 | 52 | export default NETWORK_PRESETS; 53 | -------------------------------------------------------------------------------- /src/lib/puppeteer/page.ts: -------------------------------------------------------------------------------- 1 | import puppeteer from 'puppeteer'; 2 | 3 | import {IPuppeteerConfig} from '@/lib/config/types'; 4 | import {IPageProfile} from '@/lib/puppeteer/types'; 5 | import NETWORK_PRESETS from './networkPresets'; 6 | 7 | export const preparePageProfile = ( 8 | config: IPuppeteerConfig, 9 | ): IPageProfile => { 10 | const {browserProfile} = config; 11 | const options = config.puppeteerOptions; 12 | 13 | return { 14 | userAgent: browserProfile.userAgent, 15 | width: browserProfile.width, 16 | height: browserProfile.height, 17 | cacheEnabled: options.cacheEnabled, 18 | javascriptEnabled: options.javascriptEnabled, 19 | cssFilesEnabled: options.cssFilesEnabled, 20 | cpuThrottling: options.cpuThrottling, 21 | networkThrottling: options.networkThrottling, 22 | pageNavigationTimeout: options.pageNavigationTimeout, 23 | }; 24 | }; 25 | 26 | export const createPage = async (browser: puppeteer.Browser, profile: IPageProfile): Promise => { 27 | const page = await browser.newPage(); 28 | 29 | if (profile.javascriptEnabled === false) { 30 | await page.setJavaScriptEnabled(false); 31 | } 32 | 33 | if (profile.cssFilesEnabled === false) { 34 | await page.setRequestInterception(true); 35 | 36 | page.on('request', (req) => { 37 | if (req.resourceType() === 'stylesheet') { 38 | req.abort(); 39 | } else { 40 | req.continue(); 41 | } 42 | }); 43 | } 44 | 45 | page.setDefaultTimeout(profile.pageNavigationTimeout); 46 | 47 | const client = await page.target().createCDPSession(); 48 | 49 | if (profile.networkThrottling) { 50 | await client.send('Network.enable'); 51 | await client.send('Network.emulateNetworkConditions', NETWORK_PRESETS[profile.networkThrottling]); 52 | } 53 | 54 | if (profile.cpuThrottling) { 55 | await client.send('Emulation.setCPUThrottlingRate', { rate: profile.cpuThrottling.rate }); 56 | } 57 | 58 | if (profile.cacheEnabled != null) { 59 | await page.setCacheEnabled(profile.cacheEnabled); 60 | } 61 | 62 | if (profile.userAgent) { 63 | await page.setUserAgent(profile.userAgent); 64 | } 65 | 66 | if (profile.height && profile.width) { 67 | await page.setViewport({ 68 | width: profile.width, 69 | height: profile.height, 70 | }); 71 | } 72 | 73 | return page; 74 | }; 75 | -------------------------------------------------------------------------------- /src/lib/puppeteer/pageStructureSizes.ts: -------------------------------------------------------------------------------- 1 | import {IPageStructureSizes} from '@/lib/puppeteer/types'; 2 | import puppeteer from 'puppeteer'; 3 | 4 | export const getPageStructureSizes = (page: puppeteer.Page): Promise => { 5 | return page.evaluate(() => { 6 | const scripts = Array.from(document.querySelectorAll('script')); 7 | 8 | const scriptsByType = scripts.reduce((acc, element) => { 9 | (acc[element.type] = acc[element.type] || []).push(element); 10 | return acc; 11 | }, {} as {[index: string]: HTMLScriptElement[]}); 12 | 13 | return { 14 | bytes: new Blob(Array.from(document.documentElement.outerHTML)).size, 15 | script: new Blob( 16 | scripts.map((e) => e.outerHTML), 17 | ).size, 18 | style: new Blob( 19 | Array.from(document.querySelectorAll('style')).map((e) => e.outerHTML), 20 | ).size, 21 | scripts: Object.entries(scriptsByType).reduce((acc, [scriptType, _elements]) => { 22 | acc[scriptType] = new Blob( 23 | _elements.map((element) => element.outerHTML), 24 | ).size; 25 | return acc; 26 | }, {} as {[index: string]: number}), 27 | }; 28 | }); 29 | }; 30 | -------------------------------------------------------------------------------- /src/lib/puppeteer/runCheck.ts: -------------------------------------------------------------------------------- 1 | import puppeteer from 'puppeteer'; 2 | 3 | import {getLogger} from '@/lib/logger'; 4 | import {ICheckOptions, IOriginalMetrics, ISite} from '@/types'; 5 | 6 | const logger = getLogger(); 7 | 8 | import {findTwoFreePorts} from '@/lib/findFreePorts'; 9 | import { 10 | getComparisonDir, 11 | getWprArchiveFilepath, 12 | getWprReplayStderrFilepath, 13 | getWprReplayStdoutFilepath, 14 | } from '@/lib/fs'; 15 | import {createWprReplayProcess} from '@/lib/wpr'; 16 | import WprReplay from '@/lib/wpr/WprReplay'; 17 | import {launchBrowser, prepareBrowserLaunchOptions} from './browser'; 18 | import {getPageMetrics} from './metrics'; 19 | import {createPage, preparePageProfile} from './page'; 20 | 21 | const bros: puppeteer.Browser[] = []; 22 | const wprReplays: WprReplay[] = []; 23 | 24 | export async function runPuppeteerCheck( 25 | site: ISite, 26 | siteIndex: number, 27 | options: ICheckOptions, 28 | ): Promise<(IOriginalMetrics|null)> { 29 | const {compareId, comparison, config} = options; 30 | 31 | // Different browsers for different sites can avoid cache and connection reuse between them 32 | if (!bros[siteIndex]) { 33 | if (config.puppeteerOptions.useWpr) { 34 | if (!comparison.wprArchives || !comparison.wprArchives[siteIndex]) { 35 | throw new Error(`no wprArchives for comparison: ${comparison.name} for siteIndex: ${siteIndex}`); 36 | } 37 | 38 | const wprArchiveId = comparison.wprArchives[siteIndex].wprArchiveId; 39 | 40 | const [httpPort, httpsPort] = await findTwoFreePorts(); 41 | const comparisonDir = getComparisonDir(config.workDir, comparison); 42 | 43 | const wprReplay = await createWprReplayProcess({ 44 | httpPort, 45 | httpsPort, 46 | stdoutFilepath: getWprReplayStdoutFilepath(comparisonDir, site, compareId, wprArchiveId), 47 | stderrFilepath: getWprReplayStderrFilepath(comparisonDir, site, compareId, wprArchiveId), 48 | wprArchiveFilepath: getWprArchiveFilepath(comparisonDir, site, wprArchiveId), 49 | }); 50 | wprReplays[siteIndex] = wprReplay; 51 | 52 | const launchOptions = { 53 | ...prepareBrowserLaunchOptions(config), 54 | wpr: {httpPort, httpsPort}, 55 | }; 56 | 57 | const [browser] = await Promise.all([launchBrowser(launchOptions), wprReplay.start()]); 58 | bros[siteIndex] = browser; 59 | 60 | // warmIterations 61 | if (options.warmIterations) { 62 | logger.trace(`warm page before compare: iterations=${options.warmIterations}`); 63 | const pageProfile = preparePageProfile(config); 64 | 65 | for (let i = 0; i < options.warmIterations; i++) { 66 | const page = await createPage(browser, pageProfile); 67 | await page.close(); 68 | } 69 | } 70 | 71 | } else { 72 | bros[siteIndex] = await launchBrowser(prepareBrowserLaunchOptions(config)); 73 | } 74 | } 75 | 76 | const bro = bros[siteIndex]; 77 | 78 | try { 79 | const pageProfile = preparePageProfile(config); 80 | const page = await createPage(bro, pageProfile); 81 | 82 | await page.goto(site.url, {waitUntil: config.puppeteerOptions.waitUntil}); 83 | 84 | const pageMetrics = await getPageMetrics(page); 85 | 86 | let customMetrics = {}; 87 | 88 | if (config.hooks && config.hooks.onCollectMetrics) { 89 | logger.trace(`[onCollectMetrics hook] collect custom metrics for site ${site.name} (${site.url})`); 90 | customMetrics = await config.hooks.onCollectMetrics({logger, page, comparison, site}); 91 | } 92 | 93 | await page.close(); 94 | 95 | return { 96 | ...pageMetrics, 97 | ...customMetrics, 98 | }; 99 | } catch (e) { 100 | // This error appears when wpr replay not ready, but browser already open page 101 | if (/WebSocket is not open/.exec(e.message)) { 102 | logger.debug(e); 103 | } else { 104 | logger.error(e); 105 | } 106 | 107 | await bros[siteIndex].close(); 108 | delete bros[siteIndex]; 109 | return null; 110 | } 111 | } 112 | 113 | function closeBrowsers() { 114 | return Promise.all(bros.map((bro) => bro.close())); 115 | } 116 | 117 | function closeWprReplays() { 118 | return Promise.all(wprReplays.map((wpr) => wpr.kill())); 119 | } 120 | 121 | export function close() { 122 | return Promise.all([closeBrowsers(), closeWprReplays()]); 123 | } 124 | -------------------------------------------------------------------------------- /src/lib/puppeteer/types.ts: -------------------------------------------------------------------------------- 1 | import {NetworkProfiles} from '@/lib/config/types'; 2 | 3 | export interface IBrowserLaunchOptions { 4 | headless: boolean; 5 | ignoreHTTPSErrors: boolean; 6 | wpr: null | { 7 | httpPort: number; 8 | httpsPort: number; 9 | }; 10 | imagesEnabled: boolean; 11 | } 12 | 13 | export interface IPageProfile { 14 | userAgent: string | null; 15 | width: number; 16 | height: number; 17 | 18 | cacheEnabled: boolean; 19 | javascriptEnabled: boolean; 20 | cssFilesEnabled: boolean; 21 | 22 | cpuThrottling: null | { 23 | rate: number; 24 | }; 25 | networkThrottling: null | NetworkProfiles; 26 | pageNavigationTimeout: number; 27 | } 28 | 29 | export interface IPageStructureSizes { 30 | bytes: number; 31 | script: number; 32 | style: number; 33 | scripts: { 34 | [index: string]: number; 35 | }; 36 | } 37 | 38 | export interface IPageStructureSizesHooks { 39 | onPageStructureSizesNode?: (sizes: IPageStructureSizes, node: any) => void; 40 | onPageStructureSizesComplete?: ( 41 | sizes: IPageStructureSizes, 42 | html: string, 43 | getSizeInBytes: (html: string, start: number, end: number) => number, 44 | ) => void; 45 | } 46 | -------------------------------------------------------------------------------- /src/lib/report/__spec__/mock.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | 3 | import {PORCHMARK_REPORT_VERSION, PORCHMARK_VERSION} from '@/constants'; 4 | 5 | import { sites, jsonReportResult as jsonRawReportResult} from '@/lib/dataProcessor/__spec__/mock'; 6 | 7 | export {sites, jsonRawReportResult}; 8 | 9 | export const isoDate = '2021-05-04T11:58:48.552Z'; 10 | 11 | export const humanReportResult = { 12 | "headers": [ 13 | "metric", 14 | "func", 15 | "site0", 16 | "site1", 17 | "diff0-1", 18 | "p-value" 19 | ], 20 | "data": [ 21 | [ 22 | "requestStart", 23 | "count", 24 | "4.00", 25 | "4.00", 26 | "0.00", 27 | "0.91" 28 | ], 29 | [ 30 | "requestStart", 31 | "q50", 32 | "5.25", 33 | "5.66", 34 | "+0.41", 35 | "0.91" 36 | ], 37 | [ 38 | "requestStart", 39 | "q80", 40 | "9.79", 41 | "10.57", 42 | "+0.78", 43 | "0.91" 44 | ], 45 | [ 46 | "requestStart", 47 | "q95", 48 | "9.79", 49 | "10.57", 50 | "+0.78", 51 | "0.91" 52 | ], 53 | [ 54 | "TTFB", 55 | "q50", 56 | "28.62", 57 | "30.05", 58 | "+1.43", 59 | "0.82" 60 | ], 61 | [ 62 | "TTFB", 63 | "q80", 64 | "34.64", 65 | "35.67", 66 | "+1.03", 67 | "0.82" 68 | ], 69 | [ 70 | "TTFB", 71 | "q95", 72 | "34.64", 73 | "35.67", 74 | "+1.03", 75 | "0.82" 76 | ], 77 | [ 78 | "TTLB", 79 | "q50", 80 | "58.85", 81 | "60.21", 82 | "+1.36", 83 | "0.93" 84 | ], 85 | [ 86 | "TTLB", 87 | "q80", 88 | "65.74", 89 | "66.53", 90 | "+0.79", 91 | "0.93" 92 | ], 93 | [ 94 | "TTLB", 95 | "q95", 96 | "65.74", 97 | "66.53", 98 | "+0.79", 99 | "0.93" 100 | ], 101 | [ 102 | "first-paint", 103 | "q50", 104 | "246.83", 105 | "275.02", 106 | "+28.19", 107 | "0.94" 108 | ], 109 | [ 110 | "first-paint", 111 | "q80", 112 | "316.11", 113 | "375.47", 114 | "+59.36", 115 | "0.94" 116 | ], 117 | [ 118 | "first-paint", 119 | "q95", 120 | "316.11", 121 | "375.47", 122 | "+59.36", 123 | "0.94" 124 | ], 125 | [ 126 | "FCP", 127 | "q50", 128 | "246.83", 129 | "275.02", 130 | "+28.19", 131 | "0.94" 132 | ], 133 | [ 134 | "FCP", 135 | "q80", 136 | "316.11", 137 | "375.47", 138 | "+59.36", 139 | "0.94" 140 | ], 141 | [ 142 | "FCP", 143 | "q95", 144 | "316.11", 145 | "375.47", 146 | "+59.36", 147 | "0.94" 148 | ], 149 | [ 150 | "DCL", 151 | "q50", 152 | "929.51", 153 | "917.69", 154 | "-11.82", 155 | "0.68" 156 | ], 157 | [ 158 | "DCL", 159 | "q80", 160 | "964.15", 161 | "934.89", 162 | "-29.26", 163 | "0.68" 164 | ], 165 | [ 166 | "DCL", 167 | "q95", 168 | "964.15", 169 | "934.89", 170 | "-29.26", 171 | "0.68" 172 | ], 173 | [ 174 | "loaded", 175 | "q50", 176 | "1411.60", 177 | "1310.60", 178 | "-101.01", 179 | "0.68" 180 | ], 181 | [ 182 | "loaded", 183 | "q80", 184 | "1463.36", 185 | "1438.70", 186 | "-24.66", 187 | "0.68" 188 | ], 189 | [ 190 | "loaded", 191 | "q95", 192 | "1463.36", 193 | "1438.70", 194 | "-24.66", 195 | "0.68" 196 | ], 197 | [ 198 | "domInteractive", 199 | "q50", 200 | "441.30", 201 | "439.27", 202 | "-2.03", 203 | "0.67" 204 | ], 205 | [ 206 | "domInteractive", 207 | "q80", 208 | "456.81", 209 | "448.17", 210 | "-8.64", 211 | "0.67" 212 | ], 213 | [ 214 | "domInteractive", 215 | "q95", 216 | "456.81", 217 | "448.17", 218 | "-8.64", 219 | "0.67" 220 | ], 221 | [ 222 | "domComplete", 223 | "q50", 224 | "1405.27", 225 | "1304.69", 226 | "-100.59", 227 | "0.68" 228 | ], 229 | [ 230 | "domComplete", 231 | "q80", 232 | "1457.09", 233 | "1432.57", 234 | "-24.52", 235 | "0.68" 236 | ], 237 | [ 238 | "domComplete", 239 | "q95", 240 | "1457.09", 241 | "1432.57", 242 | "-24.52", 243 | "0.68" 244 | ], 245 | [ 246 | "transferSize", 247 | "q50", 248 | "134811.00", 249 | "135446.50", 250 | "+635.50", 251 | "0.00" 252 | ], 253 | [ 254 | "transferSize", 255 | "q80", 256 | "135062.00", 257 | "135485.00", 258 | "+423.00", 259 | "0.00" 260 | ], 261 | [ 262 | "transferSize", 263 | "q95", 264 | "135062.00", 265 | "135485.00", 266 | "+423.00", 267 | "0.00" 268 | ], 269 | [ 270 | "encodedBodySize", 271 | "q50", 272 | "131089.50", 273 | "131734.50", 274 | "+645.00", 275 | "0.01" 276 | ], 277 | [ 278 | "encodedBodySize", 279 | "q80", 280 | "131372.00", 281 | "131807.00", 282 | "+435.00", 283 | "0.01" 284 | ], 285 | [ 286 | "encodedBodySize", 287 | "q95", 288 | "131372.00", 289 | "131807.00", 290 | "+435.00", 291 | "0.01" 292 | ], 293 | [ 294 | "decodedBodySize", 295 | "q50", 296 | "955270.50", 297 | "955670.50", 298 | "+400.00", 299 | "0.82" 300 | ], 301 | [ 302 | "decodedBodySize", 303 | "q80", 304 | "957284.00", 305 | "957683.00", 306 | "+399.00", 307 | "0.82" 308 | ], 309 | [ 310 | "decodedBodySize", 311 | "q95", 312 | "957284.00", 313 | "957683.00", 314 | "+399.00", 315 | "0.82" 316 | ] 317 | ], 318 | "rawData": [ 319 | [ 320 | "requestStart", 321 | "count", 322 | 4, 323 | 4, 324 | 0, 325 | 0.9079146270622539 326 | ], 327 | [ 328 | "requestStart", 329 | "q50", 330 | 5.245000007562339, 331 | 5.659999995259568, 332 | 0.4149999876972288, 333 | 0.9079146270622539 334 | ], 335 | [ 336 | "requestStart", 337 | "q80", 338 | 9.789999981876463, 339 | 10.565000004135072, 340 | 0.7750000222586095, 341 | 0.9079146270622539 342 | ], 343 | [ 344 | "requestStart", 345 | "q95", 346 | 9.789999981876463, 347 | 10.565000004135072, 348 | 0.7750000222586095, 349 | 0.9079146270622539 350 | ], 351 | [ 352 | "TTFB", 353 | "q50", 354 | 28.617499978281558, 355 | 30.047500011278316, 356 | 1.4300000329967588, 357 | 0.8226937608036163 358 | ], 359 | [ 360 | "TTFB", 361 | "q80", 362 | 34.63999996893108, 363 | 35.66500003216788, 364 | 1.0250000632368028, 365 | 0.8226937608036163 366 | ], 367 | [ 368 | "TTFB", 369 | "q95", 370 | 34.63999996893108, 371 | 35.66500003216788, 372 | 1.0250000632368028, 373 | 0.8226937608036163 374 | ], 375 | [ 376 | "TTLB", 377 | "q50", 378 | 58.849999972153455, 379 | 60.20500001613982, 380 | 1.3550000439863652, 381 | 0.9330438764574205 382 | ], 383 | [ 384 | "TTLB", 385 | "q80", 386 | 65.7449999707751, 387 | 66.53000001097098, 388 | 0.7850000401958823, 389 | 0.9330438764574205 390 | ], 391 | [ 392 | "TTLB", 393 | "q95", 394 | 65.7449999707751, 395 | 66.53000001097098, 396 | 0.7850000401958823, 397 | 0.9330438764574205 398 | ], 399 | [ 400 | "first-paint", 401 | "q50", 402 | 246.8349999981001, 403 | 275.0225000199862, 404 | 28.18750002188608, 405 | 0.9366636283969927 406 | ], 407 | [ 408 | "first-paint", 409 | "q80", 410 | 316.1149999941699, 411 | 375.47000002814457, 412 | 59.35500003397465, 413 | 0.9366636283969927 414 | ], 415 | [ 416 | "first-paint", 417 | "q95", 418 | 316.1149999941699, 419 | 375.47000002814457, 420 | 59.35500003397465, 421 | 0.9366636283969927 422 | ], 423 | [ 424 | "FCP", 425 | "q50", 426 | 246.8349999981001, 427 | 275.0225000199862, 428 | 28.18750002188608, 429 | 0.9366636283969927 430 | ], 431 | [ 432 | "FCP", 433 | "q80", 434 | 316.1149999941699, 435 | 375.47000002814457, 436 | 59.35500003397465, 437 | 0.9366636283969927 438 | ], 439 | [ 440 | "FCP", 441 | "q95", 442 | 316.1149999941699, 443 | 375.47000002814457, 444 | 59.35500003397465, 445 | 0.9366636283969927 446 | ], 447 | [ 448 | "DCL", 449 | "q50", 450 | 929.5125000353437, 451 | 917.692499991972, 452 | -11.820000043371692, 453 | 0.6815405772192378 454 | ], 455 | [ 456 | "DCL", 457 | "q80", 458 | 964.1500000143424, 459 | 934.8899999749847, 460 | -29.260000039357692, 461 | 0.6815405772192378 462 | ], 463 | [ 464 | "DCL", 465 | "q95", 466 | 964.1500000143424, 467 | 934.8899999749847, 468 | -29.260000039357692, 469 | 0.6815405772192378 470 | ], 471 | [ 472 | "loaded", 473 | "q50", 474 | 1411.6049999720417, 475 | 1310.597500007134, 476 | -101.00749996490777, 477 | 0.6766899245670923 478 | ], 479 | [ 480 | "loaded", 481 | "q80", 482 | 1463.3599999942817, 483 | 1438.704999978654, 484 | -24.655000015627593, 485 | 0.6766899245670923 486 | ], 487 | [ 488 | "loaded", 489 | "q95", 490 | 1463.3599999942817, 491 | 1438.704999978654, 492 | -24.655000015627593, 493 | 0.6766899245670923 494 | ], 495 | [ 496 | "domInteractive", 497 | "q50", 498 | 441.30250002490357, 499 | 439.27249999251217, 500 | -2.030000032391399, 501 | 0.6663315015269538 502 | ], 503 | [ 504 | "domInteractive", 505 | "q80", 506 | 456.8100000033155, 507 | 448.1699999887496, 508 | -8.640000014565885, 509 | 0.6663315015269538 510 | ], 511 | [ 512 | "domInteractive", 513 | "q95", 514 | 456.8100000033155, 515 | 448.1699999887496, 516 | -8.640000014565885, 517 | 0.6663315015269538 518 | ], 519 | [ 520 | "domComplete", 521 | "q50", 522 | 1405.2749999973457, 523 | 1304.6875000291038, 524 | -100.5874999682419, 525 | 0.6774354834562795 526 | ], 527 | [ 528 | "domComplete", 529 | "q80", 530 | 1457.0850000018254, 531 | 1432.5650000246242, 532 | -24.519999977201223, 533 | 0.6774354834562795 534 | ], 535 | [ 536 | "domComplete", 537 | "q95", 538 | 1457.0850000018254, 539 | 1432.5650000246242, 540 | -24.519999977201223, 541 | 0.6774354834562795 542 | ], 543 | [ 544 | "transferSize", 545 | "q50", 546 | 134811, 547 | 135446.5, 548 | 635.5, 549 | 0.004903850220164152 550 | ], 551 | [ 552 | "transferSize", 553 | "q80", 554 | 135062, 555 | 135485, 556 | 423, 557 | 0.004903850220164152 558 | ], 559 | [ 560 | "transferSize", 561 | "q95", 562 | 135062, 563 | 135485, 564 | 423, 565 | 0.004903850220164152 566 | ], 567 | [ 568 | "encodedBodySize", 569 | "q50", 570 | 131089.5, 571 | 131734.5, 572 | 645, 573 | 0.008654490265109449 574 | ], 575 | [ 576 | "encodedBodySize", 577 | "q80", 578 | 131372, 579 | 131807, 580 | 435, 581 | 0.008654490265109449 582 | ], 583 | [ 584 | "encodedBodySize", 585 | "q95", 586 | 131372, 587 | 131807, 588 | 435, 589 | 0.008654490265109449 590 | ], 591 | [ 592 | "decodedBodySize", 593 | "q50", 594 | 955270.5, 595 | 955670.5, 596 | 400, 597 | 0.8158280522654268 598 | ], 599 | [ 600 | "decodedBodySize", 601 | "q80", 602 | 957284, 603 | 957683, 604 | 399, 605 | 0.8158280522654268 606 | ], 607 | [ 608 | "decodedBodySize", 609 | "q95", 610 | 957284, 611 | 957683, 612 | 399, 613 | 0.8158280522654268 614 | ] 615 | ] 616 | } 617 | 618 | 619 | export const jsonReportResult = { 620 | version: PORCHMARK_VERSION, 621 | reportVersion: PORCHMARK_REPORT_VERSION, 622 | startedAt: isoDate, 623 | completedAt: isoDate, 624 | status: 'success', 625 | statusMessage: 'okay', 626 | ...jsonRawReportResult, 627 | data: { 628 | ...jsonRawReportResult.data, 629 | allMetrics: undefined, 630 | } 631 | } 632 | -------------------------------------------------------------------------------- /src/lib/report/__spec__/report.spec.ts: -------------------------------------------------------------------------------- 1 | import getDefaultConfig from '@/lib/config/default'; 2 | import {isoDate} from '@/lib/helpers'; 3 | import { HumanReport } from '../humanReport'; 4 | import { JsonReport } from '../jsonReport'; 5 | import { humanReportResult, isoDate as isoDateMock, jsonRawReportResult, jsonReportResult} from './mock'; 6 | 7 | jest.mock('@/lib/helpers', () => ({ 8 | isoDate: jest.fn(() => isoDateMock), 9 | })); 10 | 11 | describe('Reports:', () => { 12 | 13 | describe('humanReport', () => { 14 | it('provides table-view of raw json report', () => { 15 | const reporter = new HumanReport(); 16 | reporter.prepareData({ 17 | startedAt: isoDate(), 18 | completedAt: isoDate(), 19 | status: 'success', 20 | statusMessage: 'okay', 21 | config: getDefaultConfig(), 22 | report: jsonRawReportResult, 23 | }); 24 | 25 | expect(reporter.exposeInternalView()).toEqual(humanReportResult); 26 | }); 27 | }); 28 | 29 | describe('jsonReport', () => { 30 | it('provides json report', () => { 31 | const reporter = new JsonReport(); 32 | reporter.prepareData({ 33 | startedAt: isoDate(), 34 | completedAt: isoDate(), 35 | status: 'success', 36 | statusMessage: 'okay', 37 | config: getDefaultConfig(), 38 | report: jsonRawReportResult, 39 | }); 40 | 41 | expect(reporter.exposeInternalView()).toEqual(jsonReportResult); 42 | }); 43 | }); 44 | }); 45 | -------------------------------------------------------------------------------- /src/lib/report/humanReport.ts: -------------------------------------------------------------------------------- 1 | import { IPrepareDataParams, IReport } from '@/types'; 2 | import cTable = require('console.table'); 3 | import * as fs from 'fs-extra'; 4 | import jstat from 'jstat'; 5 | import * as path from 'path'; 6 | 7 | export interface IHumanReport { 8 | headers: string[]; 9 | data: (string)[][]; 10 | rawData: (number | string)[][]; 11 | } 12 | 13 | export class HumanReport implements IReport { 14 | public headers: string[]; 15 | public rawData: (string | number)[][]; 16 | public data: (string)[][]; 17 | 18 | public constructor() { 19 | this.headers = []; 20 | this.rawData = []; 21 | this.data = []; 22 | } 23 | 24 | public exposeInternalView(): IHumanReport { 25 | return { 26 | data: this.data, 27 | headers: this.headers, 28 | rawData: this.rawData, 29 | }; 30 | } 31 | 32 | public prepareData({config, report}: IPrepareDataParams) { 33 | const {sites, data} = report; 34 | const siteNames = sites.map((site) => site.name); 35 | const diffSiteNames = siteNames.filter((_, index) => index > 0); 36 | this.headers = [ 37 | 'metric', 38 | 'func', 39 | ...siteNames, 40 | // diff headers (diff0-${siteIndex}): diff0-1, diff0-2 41 | ...diffSiteNames.map((_, index) => `diff0-${index + 1}`), 42 | 'p-value', 43 | ]; 44 | 45 | for (const metric of config.metrics) { 46 | const metricName = metric.name; 47 | const metricTitle = metric.title ? metric.title : metric.name; 48 | 49 | for (const aggregation of config.metricAggregations) { 50 | const rawRow: (number | string)[] = [metricTitle, aggregation.name]; 51 | const row: string[] = [metricTitle, aggregation.name]; 52 | 53 | if (!data.metrics[metricName][aggregation.name]) { 54 | continue; 55 | } 56 | 57 | for (const siteName of siteNames) { 58 | const aggregated = data.metrics[metricName][aggregation.name][siteName]; 59 | rawRow.push(aggregated); 60 | 61 | const fixedNumber = this._toFixedNumber(aggregated); 62 | row.push(fixedNumber); 63 | } 64 | 65 | for (const siteName of diffSiteNames) { 66 | const diff = data.diffs[metricName][aggregation.name][siteName]; 67 | rawRow.push(diff); 68 | row.push(`${this._getSign(diff)}${this._toFixedNumber(diff)}`); 69 | } 70 | 71 | const allSitesMetrics = Object.values(data.allMetrics[metricName]); 72 | const pval = jstat.anovaftest(...allSitesMetrics); 73 | rawRow.push(pval); 74 | row.push(this._toFixedNumber(pval)); 75 | 76 | this.rawData.push(rawRow); 77 | this.data.push(row); 78 | } 79 | } 80 | } 81 | 82 | public async saveToFs(workDir: string, id: string) { 83 | await this.saveHumanReport(workDir, this.exposeInternalView(), id); 84 | } 85 | 86 | protected _toFixedNumber(i: number): string { 87 | return typeof i === 'number' ? i.toFixed(2) : '-'; 88 | } 89 | 90 | protected _getSign(i: number): string { 91 | return i > 0 ? '+' : ''; 92 | } 93 | 94 | protected async saveHumanReport(workDir: string, report: IHumanReport, id: string) { 95 | const filepath = this.getHumanReportFilepath(workDir, id); 96 | const table = cTable.getTable(report.headers, report.data); 97 | await fs.writeFile(filepath, table); 98 | } 99 | 100 | protected getHumanReportFilepath(workDir: string, id: string) { 101 | return path.resolve(workDir, `human-report-${id}.txt`); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/lib/report/index.ts: -------------------------------------------------------------------------------- 1 | import { IConfig } from '@/lib/config'; 2 | import { IJsonRawReport, IReport } from '@/types'; 3 | 4 | type Class = new () => T; 5 | 6 | export { HumanReport } from './humanReport'; 7 | export { JsonReport } from './jsonReport'; 8 | 9 | export async function saveReports({ 10 | startedAt, 11 | completedAt, 12 | status, 13 | statusMessage, 14 | id, 15 | workDir, 16 | config, 17 | jsonRawReport, 18 | reporters, 19 | }: { 20 | startedAt: string, 21 | completedAt: string, 22 | status: string, 23 | statusMessage: string, 24 | config: IConfig, 25 | jsonRawReport: IJsonRawReport, 26 | reporters: Class[], 27 | id: string, 28 | workDir: string, 29 | }) { 30 | const reports = reporters.map((reporter) => { 31 | const reporterInstance = new reporter(); 32 | reporterInstance.prepareData({ 33 | startedAt, 34 | completedAt, 35 | status, 36 | statusMessage, 37 | config, 38 | report: jsonRawReport, 39 | }); 40 | 41 | return reporterInstance.saveToFs(workDir, id); 42 | }); 43 | 44 | return Promise.all(reports); 45 | } 46 | -------------------------------------------------------------------------------- /src/lib/report/jsonReport.ts: -------------------------------------------------------------------------------- 1 | import {PORCHMARK_REPORT_VERSION, PORCHMARK_VERSION} from '@/constants'; 2 | import { IConfigMetricsAggregation } from '@/lib/config'; 3 | import { IMetric, IPrepareDataParams, IReport, ISite } from '@/types'; 4 | import * as fs from 'fs-extra'; 5 | import * as path from 'path'; 6 | 7 | interface IJsonReportData { 8 | metrics: { 9 | [index: string]: { // key=metric, {DCL: {"q80": {"test1": 99, "test2": 100, "test3": 200}} 10 | [index: string]: { // key=aggregation 11 | [index: string]: number; // key=site name, value = metric value 12 | }; 13 | }; 14 | }; 15 | diffs: { 16 | [index: string]: { // key=metric, example {DCL: {"q80": {"test2": -12, "test3": 10}} 17 | [index: string]: { // key=aggregation 18 | [index: string]: number; // key=site name, value: diff with first site metric 19 | }, 20 | }, 21 | }; 22 | } 23 | 24 | interface IJsonReport { 25 | version: string; 26 | reportVersion: number; 27 | startedAt: string; 28 | completedAt: string; 29 | status: string; 30 | statusMessage: string; 31 | sites: ISite[]; 32 | metrics: IMetric[]; 33 | metricAggregations: IConfigMetricsAggregation[]; 34 | data: IJsonReportData; 35 | } 36 | 37 | export class JsonReport implements IReport { 38 | private startedAt: string; 39 | private completedAt: string; 40 | private status: string; 41 | private statusMessage: string; 42 | private sites: ISite[]; 43 | private metrics: IMetric[]; 44 | private metricAggregations: IConfigMetricsAggregation[]; 45 | private data: IJsonReportData; 46 | 47 | public constructor() { 48 | this.startedAt = ''; 49 | this.completedAt = ''; 50 | this.status = ''; 51 | this.statusMessage = ''; 52 | this.sites = []; 53 | this.metrics = []; 54 | this.metricAggregations = []; 55 | this.data = { 56 | metrics: {}, 57 | diffs: {}, 58 | }; 59 | } 60 | 61 | /* Obtain and convert JsonReport to internal view */ 62 | public prepareData(params: IPrepareDataParams) { 63 | const {startedAt, completedAt, status, statusMessage, report} = params; 64 | this.startedAt = startedAt; 65 | this.completedAt = completedAt; 66 | this.status = status; 67 | this.statusMessage = statusMessage; 68 | this.sites = report.sites; 69 | this.metrics = report.metrics; 70 | this.metricAggregations = report.metricAggregations; 71 | this.data = { 72 | diffs: report.data.diffs, 73 | metrics: report.data.metrics, 74 | }; 75 | } 76 | 77 | /* Flush internal data to file system */ 78 | public async saveToFs(workDir: string, id: string) { 79 | await this.saveJsonReport( 80 | workDir, 81 | this.exposeInternalView(), 82 | id, 83 | ); 84 | } 85 | 86 | /* For testing purposes only */ 87 | public exposeInternalView() { 88 | return { 89 | version: PORCHMARK_VERSION, 90 | reportVersion: PORCHMARK_REPORT_VERSION, 91 | startedAt: this.startedAt, 92 | completedAt: this.completedAt, 93 | status: this.status, 94 | statusMessage: this.statusMessage, 95 | sites: this.sites, 96 | metrics: this.metrics, 97 | metricAggregations: this.metricAggregations, 98 | data: this.data, 99 | }; 100 | } 101 | 102 | private getReportFilepath(workDir: string, id: string) { 103 | return path.resolve(workDir, `report-${id}.json`); 104 | } 105 | 106 | private async saveJsonReport(workDir: string, report: IJsonReport, id: string) { 107 | const filepath = this.getReportFilepath(workDir, id); 108 | await fs.writeJson(filepath, report); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/lib/stats.ts: -------------------------------------------------------------------------------- 1 | import {jStat} from 'jstat'; 2 | 3 | export interface IStat { 4 | name: string; 5 | calc: (values: number[], referenceValues?: number[]) => number; 6 | roundDigits: number; 7 | diffAplicable: boolean; 8 | applicableToReference: boolean; 9 | paint: (values: Array) => Array<0 | -1 | 1 | null>; 10 | } 11 | 12 | export const calculatingStats: IStat[] = [ 13 | { 14 | name: 'q50', 15 | calc: (values) => jStat.percentile(values, 0.5), 16 | roundDigits: 1, 17 | diffAplicable: true, 18 | applicableToReference: true, 19 | paint: defaultPaint, 20 | }, { 21 | name: 'q80', 22 | calc: (values) => jStat.percentile(values, 0.8), 23 | roundDigits: 1, 24 | diffAplicable: true, 25 | applicableToReference: true, 26 | paint: defaultPaint, 27 | }, { 28 | name: 'q95', 29 | calc: (values) => jStat.percentile(values, 0.95), 30 | roundDigits: 1, 31 | diffAplicable: true, 32 | applicableToReference: true, 33 | paint: defaultPaint, 34 | }, { 35 | name: 'p-val', 36 | calc: (values, referenceValues) => { 37 | if (referenceValues === values) { 38 | return 0; 39 | } 40 | const res = jStat.anovaftest(referenceValues, values); 41 | 42 | if (isNaN(res)) { 43 | return 0; // This happens when F-Score goes to Infinity; 44 | } 45 | 46 | return res; 47 | }, 48 | roundDigits: 3, 49 | diffAplicable: false, 50 | applicableToReference: false, 51 | paint: (arr) => arr.map((pVal) => { 52 | if (pVal === null) { 53 | return null; 54 | } else if (pVal < 0.05) { 55 | return 1; 56 | } else if (pVal > 0.4) { 57 | return -1; 58 | } 59 | 60 | return 0; 61 | }), 62 | }, 63 | ] 64 | ; 65 | 66 | function defaultPaint(arr: Array) { 67 | const mean = jStat.mean(arr.filter((item) => typeof item === 'number')); 68 | 69 | return arr.map((item) => { 70 | if (item === null) { 71 | return null; 72 | } else if (item < mean * 0.8) { 73 | return 1; 74 | } else if (item > mean * 1.2) { 75 | return -1; 76 | } 77 | 78 | return 0; 79 | }); 80 | } 81 | -------------------------------------------------------------------------------- /src/lib/view.ts: -------------------------------------------------------------------------------- 1 | import blessed, {Widgets} from 'blessed'; 2 | import Table, {Cell} from 'cli-table2'; 3 | import colors from 'colors/safe'; 4 | import {Console} from 'console'; 5 | import {Writable} from 'stream'; 6 | 7 | import {IConfig} from '@/lib/config'; 8 | import {stdoutRect} from '@/lib/helpers'; 9 | import {calculatingStats} from '@/lib/stats'; 10 | 11 | const splitByColsRegex = new RegExp('.{1,' + process.stdout.columns + '}', 'g'); 12 | 13 | class ConsoleStream extends Writable { 14 | public logs: string[] = []; 15 | public renderFn: () => void = () => { throw new Error('renderFn not set'); }; 16 | 17 | public _write(chunk: string | Buffer, _: any, callback: (error?: Error | null) => void): void { 18 | const strings = chunk.toString().split('\n'); 19 | 20 | for (const str of strings) { 21 | const splittedStrings = str.match(splitByColsRegex); 22 | 23 | if (splittedStrings === null) { 24 | this.logs.push(str); 25 | } else { 26 | this.logs.push(...splittedStrings); 27 | } 28 | } 29 | 30 | callback(); 31 | this.renderFn(); 32 | } 33 | } 34 | 35 | // tslint:disable-next-line max-classes-per-file 36 | class TableView { 37 | public viewConsole: Console; 38 | 39 | protected _config?: IConfig; 40 | protected _screen?: Widgets.Screen; 41 | protected _box?: Widgets.BoxElement; 42 | protected tableText: string = ''; 43 | protected logs: string[] = []; 44 | protected logStream: ConsoleStream; 45 | 46 | protected metrCount: number = 0; 47 | protected columns: number = 0; 48 | protected statNameWidth: number = 0; 49 | protected metrColumnWidth: number = 0; 50 | protected maxSitenameWidth: number = 0; 51 | 52 | protected paddedSitenames: string[] = []; 53 | protected maxLength: number = 0; 54 | protected spaceInsufficiency: number = 0; 55 | 56 | constructor() { 57 | this.logStream = new ConsoleStream(); 58 | this.logStream.logs = this.logs; 59 | this.logStream.renderFn = this.render; 60 | 61 | this.viewConsole = new Console(this.logStream, this.logStream); 62 | 63 | } 64 | 65 | get config() { 66 | if (!this._config) { 67 | throw new Error('TableView not inited, no config'); 68 | } 69 | 70 | return this._config; 71 | } 72 | 73 | set config(config: IConfig) { 74 | this._config = config; 75 | } 76 | 77 | get screen(): Widgets.Screen { 78 | if (!this._screen) { 79 | throw new Error('Screen not inited'); 80 | } 81 | 82 | return this._screen; 83 | } 84 | 85 | set screen(screen: Widgets.Screen) { 86 | this._screen = screen; 87 | } 88 | 89 | get box(): Widgets.BoxElement { 90 | if (!this._box) { 91 | throw new Error('Box not inited'); 92 | } 93 | 94 | return this._box; 95 | } 96 | 97 | set box(box: Widgets.BoxElement) { 98 | this._box = box; 99 | } 100 | 101 | public init = () => { 102 | if (this.config.withoutUi) { 103 | return; 104 | } 105 | 106 | this.metrCount = this.config.metrics.filter((metric) => metric.showInTable).length * 2; 107 | 108 | this.columns = stdoutRect()[1] - 1; 109 | this.statNameWidth = Math.max.apply(null, calculatingStats.map((stat) => stat.name.length)) + 2; 110 | this.metrColumnWidth = Math.floor( 111 | (this.columns - this.statNameWidth - (this.metrCount + 2)) / (this.metrCount + 2), 112 | ); 113 | this.maxSitenameWidth = this.metrColumnWidth * 2 - 2; 114 | 115 | this.screen = blessed.screen({ 116 | smartCSR: true, 117 | }); 118 | this.box = blessed.box({}); 119 | // @ts-ignore TODO no append method in Widgets.Screen 120 | this.screen.append(this.box); 121 | } 122 | 123 | public renderTable = ({sites, paintedMetrics, paintedDiffs, iterations, activeTests}: { 124 | sites: string[], 125 | paintedMetrics: Array[][], 126 | paintedDiffs: Array[][], 127 | iterations: number[], 128 | activeTests: number[], 129 | }) => { 130 | if (this.config.withoutUi) { 131 | return; 132 | } 133 | 134 | const table = new Table({ 135 | head: [ 136 | '', 137 | '', 138 | ...this.config.metrics.filter((metric) => metric.showInTable) 139 | .map((metric) => ({content: metric.title || metric.name, colSpan: 2})), 140 | ], 141 | colAligns: ['left', 'right', ...Array(this.metrCount).fill('right')], 142 | colWidths: [ 143 | this.metrColumnWidth * 2, 144 | this.statNameWidth, 145 | ...Array(this.metrCount).fill(this.metrColumnWidth), 146 | ], 147 | wordWrap: true, 148 | }) as Table.HorizontalTable; 149 | 150 | const trimmedSitenames = this.trimSitenames(sites); 151 | 152 | for (let siteIndex = 0; siteIndex < sites.length; siteIndex++) { 153 | const header = trimmedSitenames[siteIndex] + 154 | `\niterations: ${iterations[siteIndex]}` + 155 | `\nactive tests: ${activeTests[siteIndex]}`; 156 | 157 | const statsToDisplay = siteIndex === 0 ? 158 | calculatingStats.filter((stat) => stat.applicableToReference) 159 | : calculatingStats 160 | ; 161 | 162 | const resultRow: Cell[] = [header, statsToDisplay.map((stat) => stat.name).join('\n')]; 163 | 164 | for (let metricIndex = 0; metricIndex < this.config.metrics.length; metricIndex++) { 165 | const metric = this.config.metrics[metricIndex]; 166 | if (!metric.showInTable) { 167 | continue; 168 | } 169 | 170 | resultRow.push( 171 | paintedMetrics[siteIndex][metricIndex].join('\n'), 172 | paintedDiffs[siteIndex][metricIndex].join('\n'), 173 | ); 174 | } 175 | 176 | table.push(resultRow); 177 | } 178 | 179 | this.tableText = table.toString(); 180 | 181 | this.render(); 182 | } 183 | 184 | public render = () => { 185 | const rows = stdoutRect()[0]; 186 | 187 | const tableLines = this.tableText.split('\n'); 188 | 189 | const tableHeight = (tableLines.length); 190 | const maxLogs = rows - tableHeight - 1; 191 | 192 | if (this.logs.length > maxLogs) { 193 | this.logs.splice(0, this.logs.length - maxLogs); 194 | } 195 | 196 | this.box.setContent([...tableLines, ...this.logs].join('\n')); 197 | this.screen.render(); 198 | } 199 | 200 | public shutdown = (errorHappened: boolean) => { 201 | if (!this.config.withoutUi) { 202 | this.screen.destroy(); 203 | 204 | if (this.tableText) { 205 | // tslint:disable-next-line no-console 206 | console.log(this.tableText); 207 | } 208 | 209 | if (this.logs.length > 0) { 210 | // tslint:disable-next-line no-console 211 | console.error(`\nLast logs:\n${this.logs.join('\n')}`); 212 | } 213 | } 214 | 215 | process.exit(errorHappened ? 1 : 0); 216 | } 217 | 218 | public emergencyShutdown = (error: Error) => { 219 | this.viewConsole.log(error); 220 | this.shutdown(true); 221 | } 222 | 223 | public trimSitenames = (sites: string[]): string[] => { 224 | if (!this.paddedSitenames.length) { 225 | this.maxLength = Math.max(...sites.map((site) => site.length)); 226 | 227 | this.paddedSitenames = sites.map((site) => { 228 | const pad = Math.ceil((this.maxLength - site.length) / 2); 229 | 230 | return site.padEnd(pad).padStart(pad); 231 | }); 232 | 233 | this.spaceInsufficiency = this.maxLength - this.maxSitenameWidth; 234 | } 235 | 236 | const shifter = (Date.now() / 200) % (this.spaceInsufficiency * 2.5) - this.spaceInsufficiency * 0.25; 237 | let position: number; 238 | if (shifter < 0) { 239 | position = 0; 240 | } else if (shifter < this.spaceInsufficiency) { 241 | position = shifter; 242 | } else if (shifter < this.spaceInsufficiency * 1.25) { 243 | position = this.spaceInsufficiency; 244 | } else if (shifter < this.spaceInsufficiency * 2) { 245 | position = 2 * this.spaceInsufficiency - shifter; 246 | } else if (shifter < this.spaceInsufficiency * 2) { 247 | position = 0; 248 | } 249 | 250 | return this.paddedSitenames.map((site) => colors.green(site.slice(position))); 251 | } 252 | } 253 | 254 | const view: TableView = new TableView(); 255 | 256 | export const getView = (): TableView => { 257 | if (!view) { 258 | throw new Error('TableView not inited'); 259 | } 260 | 261 | return view; 262 | }; 263 | 264 | export const getViewConsole = (): Console => getView().viewConsole; 265 | -------------------------------------------------------------------------------- /src/lib/webdriverio.ts: -------------------------------------------------------------------------------- 1 | import {DesiredCapabilities, Options as WDOptions, remote} from 'webdriverio'; 2 | 3 | import {IBrowserProfile} from '@/lib/config'; 4 | import {getLogger} from '@/lib/logger'; 5 | import {ICheckOptions, IOriginalMetrics, ISite} from '@/types'; 6 | 7 | const logger = getLogger(); 8 | 9 | export async function runWebdriverCheck( 10 | site: ISite, 11 | _: number, 12 | options: ICheckOptions, 13 | ): Promise<(IOriginalMetrics|null)> { 14 | const {config} = options; 15 | const browserProfile = config.browserProfile; 16 | 17 | const wdOptions = validateWDOptions(config.webdriverOptions); 18 | 19 | if (wdOptions.desiredCapabilities.browserName === 'chrome') { 20 | setChromeFlags(wdOptions.desiredCapabilities, browserProfile); 21 | } 22 | 23 | const {height, width} = browserProfile; 24 | 25 | try { 26 | const metrics = await remote(wdOptions) 27 | .init(wdOptions.desiredCapabilities) 28 | .setViewportSize({width, height}) 29 | // @ts-ignore FIXME Property 'url' does not exist on type 'never'. 30 | .url(site.url) 31 | .execute(getMetricsFromBrowser); 32 | 33 | return metrics.value; 34 | 35 | } catch (e) { 36 | logger.error(e); 37 | return null; 38 | } 39 | } 40 | 41 | function getMetricsFromBrowser() { 42 | const timings = performance.getEntriesByType('navigation')[0].toJSON(); 43 | const paintEntries = performance.getEntriesByType('paint'); 44 | for (const entry of paintEntries) { 45 | timings[entry.name] = entry.startTime; 46 | } 47 | return timings; 48 | } 49 | 50 | type ValidWDOptions = WDOptions & { 51 | desiredCapabilities: DesiredCapabilities & { 52 | browserName: string, 53 | version: string, 54 | }, 55 | }; 56 | 57 | function validateWDOptions(options: WDOptions): ValidWDOptions { 58 | if ( 59 | typeof options === 'object' && 60 | typeof options.desiredCapabilities === 'object' && 61 | typeof options.desiredCapabilities.browserName === 'string' && 62 | typeof options.desiredCapabilities.version === 'string' 63 | ) { 64 | // @ts-ignore 65 | return options; 66 | } 67 | throw new TypeError('invalid desiredCapabilities object!'); 68 | } 69 | 70 | function setChromeFlags(desiredCapabilities: DesiredCapabilities, browserProfile: IBrowserProfile) { 71 | if (!desiredCapabilities.chromeOptions) { 72 | desiredCapabilities.chromeOptions = {}; 73 | } 74 | 75 | if (!desiredCapabilities.chromeOptions.args) { 76 | desiredCapabilities.chromeOptions.args = []; 77 | } 78 | 79 | if (browserProfile.userAgent) { 80 | desiredCapabilities.chromeOptions.args.push(`user-agent=${browserProfile.userAgent}`); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/lib/workerFarm.ts: -------------------------------------------------------------------------------- 1 | import { 2 | IOriginalMetrics, 3 | } from '@/types'; 4 | 5 | import {IComparison, IConfig} from '@/lib/config'; 6 | import {DataProcessor} from '@/lib/dataProcessor'; 7 | import {indexOfMin, sleep} from '@/lib/helpers'; 8 | import {getLogger} from '@/lib/logger'; 9 | import {close, runPuppeteerCheck} from '@/lib/puppeteer'; 10 | import {getView} from '@/lib/view'; 11 | import {runWebdriverCheck} from '@/lib/webdriverio'; 12 | 13 | const logger = getLogger(); 14 | const view = getView(); 15 | 16 | const workerSet = new Set(); 17 | 18 | let waitForCompleteInterval: NodeJS.Timeout; 19 | 20 | function waitForComplete(check: () => boolean): Promise { 21 | return new Promise((resolve) => { 22 | waitForCompleteInterval = setInterval(() => { 23 | if (check()) { 24 | resolve(); 25 | } 26 | }, 100); 27 | }); 28 | } 29 | 30 | function clearWaitForComplete() { 31 | clearInterval(waitForCompleteInterval); 32 | } 33 | 34 | export default async function startWorking( 35 | compareId: number, 36 | comparision: IComparison, 37 | dataProcessor: DataProcessor, 38 | config: IConfig, 39 | ) { 40 | let workersDone = 0; 41 | 42 | logger.info(`[startWorking] start: comparison=${comparision.name} id=${compareId}`); 43 | 44 | const runCheck = (config.mode === 'webdriver' ? runWebdriverCheck : runPuppeteerCheck); 45 | 46 | const totalIterationCount = config.mode === 'puppeteer' && config.puppeteerOptions.useWpr 47 | ? config.iterations * (compareId + 1) 48 | : config.iterations; 49 | 50 | function getNextSiteIndex(): (number|null) { 51 | if (dataProcessor.getLeastIterations() >= totalIterationCount) { 52 | return null; 53 | } 54 | 55 | const totalTests = []; 56 | for (let siteIndex = 0; siteIndex < dataProcessor.sites.length; siteIndex++) { 57 | totalTests[siteIndex] = dataProcessor.iterations[siteIndex] + dataProcessor.activeTests[siteIndex]; 58 | } 59 | 60 | return indexOfMin(totalTests); 61 | } 62 | 63 | // Controls the number of workers, spawns new ones, stops process when everything's done 64 | async function populateWorkers() { 65 | while (workersDone < config.workers) { 66 | while (config.workers - workersDone > workerSet.size) { 67 | const nextSiteIndex = getNextSiteIndex(); 68 | 69 | if (nextSiteIndex === null) { 70 | workersDone++; 71 | continue; 72 | } 73 | 74 | const job = runWorker(nextSiteIndex, comparision, config).catch(handleWorkerError); 75 | 76 | workerSet.add(job); 77 | dataProcessor.reportTestStart(nextSiteIndex, job); 78 | 79 | const clearJob = () => { workerSet.delete(job); }; 80 | job.then(clearJob, clearJob); 81 | } 82 | 83 | await Promise.race(Array.prototype.slice.call(workerSet.entries()).concat(sleep(100))); 84 | } 85 | 86 | // render last results 87 | view.renderTable(dataProcessor.calculateResults()); 88 | 89 | logger.info( 90 | `[startWorking] complete: comparison=${comparision.name}, id=${compareId}, workersDone=${workersDone}`, 91 | ); 92 | } 93 | 94 | function handleWorkerError(error: Error): void { 95 | logger.error(error); 96 | } 97 | 98 | function registerMetrics([originalMetrics, siteIndex]: [IOriginalMetrics, number]): void { 99 | const transformedMetrics: number[] = []; 100 | 101 | logger.trace('workerFarm registerMetrics:', siteIndex, originalMetrics); 102 | 103 | for (let metricIndex = 0; metricIndex < config.metrics.length; metricIndex++) { 104 | const metricName = config.metrics[metricIndex].name; 105 | logger.trace(`workerFarm registerMetrics: ${metricIndex}, ${metricName}, ${originalMetrics[metricName]}`); 106 | transformedMetrics[metricIndex] = originalMetrics[metricName]; 107 | } 108 | 109 | logger.trace(`workerFarm registerMetrics: transformedMetrics ${transformedMetrics}`); 110 | 111 | dataProcessor.registerMetrics(siteIndex, transformedMetrics); 112 | } 113 | 114 | async function runWorker( 115 | siteIndex: number, 116 | workerComparision: IComparison, 117 | workerConfig: IConfig, 118 | ): Promise { 119 | const workerSites = workerComparision.sites; 120 | 121 | const metrics = await Promise.race([ 122 | sleep(workerConfig.pageTimeout).then(() => { 123 | throw new Error(`Timeout on site #${siteIndex}, ${workerSites[siteIndex].url}`); 124 | }), 125 | runCheck(workerSites[siteIndex], siteIndex, { 126 | comparison: workerComparision, 127 | config: workerConfig, 128 | compareId, 129 | warmIterations: config.puppeteerOptions.warmIterations, 130 | }), 131 | ]); 132 | 133 | if (metrics !== null) { 134 | registerMetrics([metrics, siteIndex]); 135 | } 136 | } 137 | 138 | populateWorkers().catch((error) => { 139 | logger.error(error); 140 | }); 141 | 142 | await waitForComplete(() => { 143 | return workersDone >= config.workers; 144 | }); 145 | 146 | clearWaitForComplete(); 147 | 148 | if (config.mode === 'puppeteer') { 149 | await close(); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/lib/wpr/WprAbstract.ts: -------------------------------------------------------------------------------- 1 | import {ChildProcess, spawn} from 'child_process'; 2 | import * as fs from 'fs'; 3 | 4 | import {assertNonNull} from '@/lib/helpers'; 5 | import {getLogger} from '@/lib/logger'; 6 | import {IWprConfig, IWprProcess} from '@/lib/wpr/types'; 7 | 8 | export type BuildCmd = (wprConfig: IWprConfig, inputWprFilepath: string) => {command: string, args: string[]}; 9 | 10 | const logger = getLogger(); 11 | 12 | export const WAIT_TIMEOUT = 10000; 13 | 14 | export default abstract class WprAbstract implements IWprProcess { 15 | protected _config: IWprConfig; 16 | 17 | protected _process: ChildProcess | null = null; 18 | 19 | protected abstract _name: string = 'WprAbstract'; 20 | 21 | protected abstract _buildCmd: BuildCmd; 22 | 23 | constructor(config: IWprConfig) { 24 | this._config = config; 25 | } 26 | 27 | get process(): ChildProcess { 28 | if (!this._process) { 29 | throw new Error(`no process for ${this._name}`); 30 | } 31 | return this._process; 32 | } 33 | 34 | public async start() { 35 | const {wprArchiveFilepath, stdoutFilepath, stderrFilepath} = this._config; 36 | 37 | const cmd = this._buildCmd(this._config, wprArchiveFilepath); 38 | 39 | logger.debug(`starting ${this._name}: ${cmd.command} ${cmd.args.join(' ')}`); 40 | 41 | this._process = spawn(cmd.command, cmd.args); 42 | 43 | this._process.on('error', (error) => { 44 | logger.error(error); 45 | throw error; 46 | }); 47 | 48 | this._process.on('close', (code: number) => { 49 | if (code > 0) { 50 | throw new Error(`${this._name} process exit with code: ${code}`); 51 | } 52 | }); 53 | 54 | logger.debug(`started ${this._name} process: pid=${this._process.pid}`); 55 | 56 | // ChildProcess's stdout and stderr might be null if spawned with stdio other then `pipe`. 57 | // not this case 58 | assertNonNull(this.process.stdout); 59 | this.process.stdout.pipe(fs.createWriteStream(stdoutFilepath)); 60 | assertNonNull(this.process.stderr); 61 | this.process.stderr.pipe(fs.createWriteStream(stderrFilepath)); 62 | } 63 | 64 | public async stop() { 65 | logger.debug(`stopping ${this._name} process: pid=${this.process.pid}`); 66 | this.process.kill('SIGINT'); 67 | await this.wait(); 68 | } 69 | 70 | public async kill() { 71 | logger.debug(`killing ${this._name} process: pid=${this.process.pid}`); 72 | await this.process.kill(); 73 | } 74 | 75 | public onClose(cb: (code: number) => void) { 76 | this.process.on('close', cb); 77 | } 78 | 79 | public onError(cb: (err: Error) => void) { 80 | this.process.on('error', cb); 81 | } 82 | 83 | public wait() { 84 | return new Promise((resolve, reject) => { 85 | logger.debug(`wait while ${this._name} process (pid=${this.process.pid}) stopping`); 86 | 87 | const timeout = setTimeout(() => { 88 | reject(new Error(`wait timeout for ${this._name}: ${WAIT_TIMEOUT}ms`)); 89 | }, WAIT_TIMEOUT); 90 | 91 | this.onClose((code) => { 92 | clearTimeout(timeout); 93 | 94 | if (code > 0) { 95 | return reject(new Error(`${this._name} process (pid=${this.process.pid}) exit with code: ${code}`)); 96 | } 97 | 98 | logger.debug(`${this._name} process (pid=${this.process.pid}) exit with code: 0`); 99 | 100 | resolve(); 101 | }); 102 | }); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/lib/wpr/WprRecord.ts: -------------------------------------------------------------------------------- 1 | import {IWprConfig, IWprProcess} from './types'; 2 | import WprAbstract from './WprAbstract'; 3 | 4 | export default class WprRecord extends WprAbstract implements IWprProcess { 5 | protected _name: string = 'WprRecord'; 6 | 7 | protected _buildCmd = (wprConfig: IWprConfig, outputWprFilepath: string) => { 8 | return { 9 | command: wprConfig.bin, 10 | args: [ 11 | 'record', 12 | '--https_cert_file', wprConfig.certFile, 13 | '--https_key_file', wprConfig.keyFile, 14 | '--http_port', String(wprConfig.httpPort), 15 | '--https_port', String(wprConfig.httpsPort), 16 | '--inject_scripts', wprConfig.injectScripts, 17 | outputWprFilepath, 18 | ], 19 | }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/lib/wpr/WprReplay.ts: -------------------------------------------------------------------------------- 1 | import {IWprConfig, IWprProcess} from './types'; 2 | import WprAbstract from './WprAbstract'; 3 | 4 | export default class WprReplay extends WprAbstract implements IWprProcess { 5 | protected _name: string = 'WprReplay'; 6 | 7 | protected _buildCmd = (wprConfig: IWprConfig, inputWprFilepath: string) => { 8 | return { 9 | command: wprConfig.bin, 10 | args: [ 11 | 'replay', 12 | '--https_cert_file', wprConfig.certFile, 13 | '--https_key_file', wprConfig.keyFile, 14 | '--http_port', String(wprConfig.httpPort), 15 | '--https_port', String(wprConfig.httpsPort), 16 | '--inject_scripts', wprConfig.injectScripts, 17 | inputWprFilepath, 18 | ], 19 | }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/lib/wpr/index.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra'; 2 | import path from 'path'; 3 | import puppeteer, {Page} from 'puppeteer'; 4 | 5 | import {IComparison, IConfig} from '@/lib/config'; 6 | import {findTwoFreePorts} from '@/lib/findFreePorts'; 7 | import { 8 | getComparisonDir, 9 | getPageStructureSizesAfterLoadedFilepath, getPageStructureSizesFilepath, 10 | getWprArchiveFilepath, getWprRecordScreenshotFilepath, 11 | getWprRecordStderrFilepath, 12 | getWprRecordStdoutFilepath, getWprReplayStderrFilepath, getWprReplayStdoutFilepath, 13 | } from '@/lib/fs'; 14 | import {sleep} from '@/lib/helpers'; 15 | import {getLogger} from '@/lib/logger'; 16 | import { 17 | createPage, 18 | IPageProfile, 19 | launchBrowser, 20 | prepareBrowserLaunchOptions, 21 | preparePageProfile, 22 | } from '@/lib/puppeteer'; 23 | import {getPageStructureSizes} from '@/lib/puppeteer/pageStructureSizes'; 24 | import {ISite} from '@/types'; 25 | import {IBaseWprConfig, IWprConfig, IWprProcessOptions} from './types'; 26 | import WprRecord from './WprRecord'; 27 | import WprReplay from './WprReplay'; 28 | 29 | const logger = getLogger(); 30 | 31 | const rootDir = path.resolve(__dirname, '../../..'); 32 | const wprToolDir = path.resolve(`${rootDir}/wpr`); 33 | 34 | const baseConfig: IBaseWprConfig = { 35 | bin: path.resolve(wprToolDir, `wpr`), 36 | certFile: path.resolve(wprToolDir, 'wpr_cert.pem'), 37 | keyFile: path.resolve(wprToolDir, 'wpr_key.pem'), 38 | injectScripts: path.resolve(wprToolDir, 'deterministic.js'), 39 | }; 40 | 41 | export const createWprRecordProcess = (options: IWprProcessOptions) => { 42 | const config: IWprConfig = { 43 | ...baseConfig, 44 | ...options, 45 | }; 46 | 47 | return new WprRecord(config); 48 | }; 49 | 50 | export const createWprReplayProcess = (options: IWprProcessOptions) => { 51 | const config: IWprConfig = { 52 | ...baseConfig, 53 | ...options, 54 | }; 55 | 56 | return new WprReplay(config); 57 | }; 58 | 59 | const openPageWithRetries = async ( 60 | page: puppeteer.Page, 61 | site: ISite, 62 | retryCount: number, 63 | onVerifyWprHook: () => Promise, 64 | ): Promise => { 65 | let retry = 0; 66 | 67 | while (retry <= retryCount) { 68 | try { 69 | await page.goto(site.url, {waitUntil: 'networkidle0'}); 70 | 71 | if (onVerifyWprHook) { 72 | logger.trace(`[recordWprArchives] verify page ${site.name} (${site.url}) with onVerifyWpr hook`); 73 | await onVerifyWprHook(); 74 | } 75 | 76 | return page; 77 | } catch (error) { 78 | retry++; 79 | 80 | if (retry > retryCount) { 81 | logger.error(`[recordWprArchives] ${retryCount} retries exceed for site ${site.name} (${site.url})`); 82 | throw error; 83 | } else { 84 | logger.error(error); 85 | logger.warn(`[recordWprArchives] retry #${retry} page open: ${site.name} (${site.url})`); 86 | } 87 | } 88 | } 89 | 90 | return page; 91 | }; 92 | 93 | const fetchPageStructureSizes = ( 94 | {page, site, filepath, onVerifyWprHook, retryCount}: { 95 | page: Page, 96 | site: ISite, 97 | filepath: string, 98 | onVerifyWprHook: () => Promise, 99 | retryCount: number, 100 | }, 101 | ) => { 102 | return openPageWithRetries(page, site, retryCount, onVerifyWprHook) 103 | .then(() => getPageStructureSizes(page)) 104 | .then((sizes) => fs.writeJson(filepath, sizes)); 105 | }; 106 | 107 | export const recordWprArchives = async (comparison: IComparison, config: IConfig): Promise => { 108 | logger.info(`[recordWprArchives] start: record wpr archives for comparison: ${comparison.name}`); 109 | 110 | const sites = comparison.sites; 111 | 112 | // check workDir 113 | const comparisonDir = getComparisonDir(config.workDir, comparison); 114 | 115 | await fs.ensureDir(comparisonDir); 116 | 117 | const {recordWprCount} = config.puppeteerOptions; 118 | 119 | for (let id = 0; id < recordWprCount; id++) { 120 | logger.info( 121 | `[recordWprArchives] record wpr archives: ${id + 1} of ${recordWprCount}`, 122 | ); 123 | const wprRecordProcesses = []; 124 | const wprReplayProcesses = []; 125 | const launchBrowserPromises = []; 126 | 127 | for (const site of comparison.sites) { 128 | const [httpPort, httpsPort] = await findTwoFreePorts(); 129 | 130 | const wprArchiveFilepath = getWprArchiveFilepath(comparisonDir, site, id); 131 | 132 | const wprRecordProcess = createWprRecordProcess({ 133 | httpPort, 134 | httpsPort, 135 | stdoutFilepath: getWprRecordStdoutFilepath(comparisonDir, site, id), 136 | stderrFilepath: getWprRecordStderrFilepath(comparisonDir, site, id), 137 | wprArchiveFilepath, 138 | }); 139 | wprRecordProcesses.push(wprRecordProcess); 140 | 141 | const wprReplayProcess = createWprReplayProcess({ 142 | httpPort, 143 | httpsPort, 144 | stdoutFilepath: getWprReplayStdoutFilepath(comparisonDir, site, 0, id), 145 | stderrFilepath: getWprReplayStderrFilepath(comparisonDir, site, 0, id), 146 | wprArchiveFilepath, 147 | }); 148 | wprReplayProcesses.push(wprReplayProcess); 149 | 150 | const launchOptions = { 151 | ...prepareBrowserLaunchOptions(config), 152 | wpr: {httpPort, httpsPort}, 153 | }; 154 | 155 | const browser = launchBrowser(launchOptions); 156 | launchBrowserPromises.push(browser); 157 | } 158 | 159 | // start and wait wpr record processes and browsers 160 | await Promise.all([ 161 | Promise.all(wprRecordProcesses.map((p) => p.start())), 162 | Promise.all(launchBrowserPromises), 163 | ]); 164 | 165 | // get launched browsers 166 | const browsers = await Promise.all(launchBrowserPromises); 167 | 168 | // ready 169 | const recordPageWprPromises = []; 170 | 171 | for (const siteIndex of comparison.sites.keys()) { 172 | const site = sites[siteIndex]; 173 | const browser = browsers[siteIndex]; 174 | 175 | logger.trace(`[recordWprArchives] record wpr archive for ${site.name}`); 176 | 177 | const pageProfile = preparePageProfile(config); 178 | 179 | pageProfile.cacheEnabled = false; 180 | pageProfile.cpuThrottling = null; 181 | pageProfile.networkThrottling = null; 182 | 183 | const page = await createPage(browser, pageProfile); 184 | 185 | const recordPageWprPromise = fetchPageStructureSizes({ 186 | page, 187 | site, 188 | filepath: getPageStructureSizesAfterLoadedFilepath(comparisonDir, site, id), 189 | retryCount: config.puppeteerOptions.retryCount, 190 | onVerifyWprHook: () => 191 | config.hooks && config.hooks.onVerifyWpr 192 | ? config.hooks.onVerifyWpr({logger, page, comparison, site}) 193 | : Promise.resolve(), 194 | }) 195 | .then(() => page.screenshot({ 196 | fullPage: true, 197 | path: getWprRecordScreenshotFilepath(comparisonDir, site, id), 198 | })) 199 | .then(() => page.close()); 200 | 201 | recordPageWprPromises.push(recordPageWprPromise); 202 | } 203 | 204 | await Promise.all(recordPageWprPromises); 205 | 206 | // close wpr record processes 207 | await Promise.all(wprRecordProcesses.map((p) => p.stop())); 208 | 209 | // start wpr replay process on same ports 210 | await Promise.all(wprReplayProcesses.map( 211 | (p) => p.start().then(() => sleep(100)), 212 | )); 213 | 214 | const pageStructureSizesPromises = []; 215 | 216 | // get page structure sizes without javascript 217 | for (const siteIndex of comparison.sites.keys()) { 218 | const site = sites[siteIndex]; 219 | const browser = browsers[siteIndex]; 220 | 221 | logger.trace(`[recordWprArchives] get page structure sizes for ${site.name}`); 222 | 223 | const pageProfile: IPageProfile = { 224 | ...preparePageProfile(config), 225 | cpuThrottling: null, 226 | networkThrottling: null, 227 | javascriptEnabled: false, 228 | }; 229 | const page = await createPage(browser, pageProfile); 230 | 231 | const pageStructureSizesPromise = fetchPageStructureSizes({ 232 | page, 233 | site, 234 | filepath: getPageStructureSizesFilepath(comparisonDir, site, id), 235 | retryCount: config.puppeteerOptions.retryCount, 236 | onVerifyWprHook: () => Promise.resolve(), 237 | }) 238 | .then(() => page.close()); 239 | 240 | pageStructureSizesPromises.push(pageStructureSizesPromise); 241 | } 242 | 243 | await Promise.all(pageStructureSizesPromises); 244 | 245 | // close wpr processes and browsers 246 | await Promise.all( 247 | [ 248 | ...browsers.map((bro) => bro.close()), 249 | ...wprReplayProcesses.map((p) => p.stop()), 250 | ], 251 | ); 252 | } 253 | 254 | logger.info(`[recordWprArchives] complete: record wpr archives for comparison: ${comparison.name}`); 255 | }; 256 | -------------------------------------------------------------------------------- /src/lib/wpr/select.ts: -------------------------------------------------------------------------------- 1 | import {IConfig} from '@/lib/config'; 2 | import {getPageStructureSizesFilepath} from '@/lib/fs'; 3 | import {getLogger} from '@/lib/logger'; 4 | import {ISelectedWprArchives, IWprArchive} from '@/lib/wpr/types'; 5 | import {ISite} from '@/types'; 6 | import * as fs from 'fs-extra'; 7 | import jstat = require('jstat'); 8 | import * as path from 'path'; 9 | 10 | const logger = getLogger(); 11 | 12 | const parseWprArchiveFilenameRegex = /(.*)-(\d+)\.wprgo/; 13 | 14 | const getWprArchiveInfo = async (comparisonDir: string, filename: string): Promise => { 15 | const match = parseWprArchiveFilenameRegex.exec(filename); 16 | 17 | if (!match) { 18 | throw new Error(`can't parse wpr filename: ${filename}`); 19 | } 20 | 21 | const siteName = match[1]; 22 | const wprArchiveId = Number(match[2]); 23 | 24 | const [stat, structureSizes] = await Promise.all([ 25 | fs.stat(path.resolve(comparisonDir, filename)), 26 | fs.readJson(getPageStructureSizesFilepath(comparisonDir, {name: siteName, url: ''}, wprArchiveId)) 27 | .catch((error) => { 28 | if (error.code === 'ENOENT') { 29 | logger.warn( 30 | `skip wpr for site=${siteName} wprArchiveId=${wprArchiveId}, no pageStructureSizes`, 31 | ); 32 | return null; 33 | } 34 | 35 | throw error; 36 | }), 37 | ]); 38 | 39 | return { 40 | siteName, 41 | wprArchiveId, 42 | size: stat.size, 43 | structureSizes, 44 | }; 45 | }; 46 | 47 | export async function getWprArchives(comparisonDir: string, sites: ISite[]): Promise { 48 | const files = await fs.readdir(comparisonDir); 49 | const wprFiles = files.filter((filename: string) => /.*\.wprgo$/.exec(filename)); 50 | const wprs = await Promise.all(wprFiles.map((filename) => getWprArchiveInfo(comparisonDir, filename))); 51 | 52 | const siteNames = sites.map((site) => site.name); 53 | return wprs.filter((wpr) => siteNames.includes(wpr.siteName) && wpr.structureSizes); 54 | } 55 | 56 | const selectWprByWprArchiveId = (wprs: IWprArchive[], site: ISite, wprArchiveId: number): IWprArchive => { 57 | const found = wprs.find((wpr) => wpr.siteName === site.name && wpr.wprArchiveId === wprArchiveId); 58 | 59 | if (!found) { 60 | throw new Error(`can't find wpr for site=${site.name}, wprArchiveId=${wprArchiveId}`); 61 | } 62 | 63 | return found; 64 | }; 65 | 66 | export enum WprArchiveSizeTypes { 67 | WPR_ARCHIVE_SIZE, 68 | HTML_SIZE, 69 | INLINE_SCRIPT_SIZE, 70 | } 71 | 72 | const getWprArchiveSize = (wprArchive: IWprArchive, type: WprArchiveSizeTypes): number => { 73 | switch (type) { 74 | case WprArchiveSizeTypes.WPR_ARCHIVE_SIZE: 75 | return wprArchive.size; 76 | case WprArchiveSizeTypes.HTML_SIZE: 77 | return wprArchive.structureSizes.bytes; 78 | case WprArchiveSizeTypes.INLINE_SCRIPT_SIZE: 79 | return wprArchive.structureSizes.script; 80 | default: 81 | throw new Error(`unknown WprArchive size type: ${type}`); 82 | } 83 | }; 84 | 85 | export async function selectWprArchivesSimple( 86 | wprs: IWprArchive[], 87 | sites: ISite[], 88 | count: number, 89 | ): Promise { 90 | const result: ISelectedWprArchives[] = []; 91 | 92 | for (let i = 0; i < count; i++) { 93 | const wprArchives: IWprArchive[] = []; 94 | 95 | for (const site of sites) { 96 | wprArchives.push(selectWprByWprArchiveId(wprs, site, i)); 97 | } 98 | 99 | const selected: ISelectedWprArchives = { 100 | wprArchives, 101 | diff: wprArchives.length === 2 102 | ? getWprArchiveSize(wprArchives[0], WprArchiveSizeTypes.HTML_SIZE) 103 | - getWprArchiveSize(wprArchives[1], WprArchiveSizeTypes.HTML_SIZE) 104 | : jstat.stdev( 105 | wprArchives.map((wprArchive) => getWprArchiveSize(wprArchive, WprArchiveSizeTypes.HTML_SIZE)), 106 | true, 107 | ), 108 | }; 109 | 110 | result.push(selected); 111 | } 112 | 113 | return result; 114 | } 115 | 116 | function getAllCombinations( 117 | variant: IWprArchive[], 118 | siteIndex: number, 119 | wprArchivesBySite: IWprArchive[][], 120 | siteWprArchivesCount: number, 121 | wprArchiveSizeType: WprArchiveSizeTypes, 122 | result: ISelectedWprArchives[], 123 | ) { 124 | const nextSiteIndex = siteIndex + 1; 125 | 126 | const currentSiteWprArchives = wprArchivesBySite[siteIndex]; 127 | const nextSiteWprArchives = wprArchivesBySite[nextSiteIndex]; 128 | 129 | for (let i = 0; i < siteWprArchivesCount; i++) { 130 | if (!nextSiteWprArchives) { 131 | const wprArchives = [...variant, currentSiteWprArchives[i]]; 132 | result.push({ 133 | wprArchives, 134 | diff: wprArchives.length === 2 135 | ? getWprArchiveSize(wprArchives[0], wprArchiveSizeType) 136 | - getWprArchiveSize(wprArchives[1], wprArchiveSizeType) 137 | : jstat.stdev( 138 | wprArchives.map((wprArchive) => getWprArchiveSize(wprArchive, wprArchiveSizeType)), 139 | true, 140 | ), 141 | }); 142 | } else { 143 | getAllCombinations( 144 | [...variant, currentSiteWprArchives[i]], 145 | nextSiteIndex, 146 | wprArchivesBySite, 147 | siteWprArchivesCount, 148 | wprArchiveSizeType, 149 | result, 150 | ); 151 | } 152 | } 153 | 154 | return result; 155 | } 156 | 157 | export const selectClosestWprArchives = ( 158 | wprArchives: IWprArchive[], 159 | sites: ISite[], 160 | wprArchiveSizeType: WprArchiveSizeTypes, 161 | count: number, 162 | ): ISelectedWprArchives[] => { 163 | const wprArchivesBySites: IWprArchive[][] = []; 164 | 165 | const siteNames = sites.map((site) => site.name); 166 | 167 | wprArchives.forEach((wprArchive) => { 168 | const siteIndex = siteNames.indexOf(wprArchive.siteName); 169 | 170 | if (siteIndex === -1) { 171 | return; 172 | } 173 | 174 | if (!wprArchivesBySites[siteIndex]) { 175 | wprArchivesBySites[siteIndex] = []; 176 | } 177 | 178 | wprArchivesBySites[siteIndex].push(wprArchive); 179 | }); 180 | 181 | const combinations = getAllCombinations( 182 | [], 0, wprArchivesBySites, wprArchivesBySites[0].length, wprArchiveSizeType, [], 183 | ); 184 | 185 | combinations.sort((a, b) => Math.abs(a.diff) - Math.abs(b.diff)); 186 | 187 | return combinations.slice(0, count); 188 | }; 189 | 190 | export async function selectWprArchives( 191 | config: IConfig, 192 | wprs: IWprArchive[], 193 | sites: ISite[], 194 | ): Promise { 195 | switch (config.puppeteerOptions.selectWprMethod) { 196 | case 'simple': 197 | return selectWprArchivesSimple(wprs, sites, config.puppeteerOptions.selectWprCount); 198 | case 'closestByWprSize': 199 | return selectClosestWprArchives( 200 | wprs, 201 | sites, 202 | WprArchiveSizeTypes.WPR_ARCHIVE_SIZE, 203 | config.puppeteerOptions.selectWprCount, 204 | ); 205 | case 'closestByHtmlSize': 206 | return selectClosestWprArchives( 207 | wprs, 208 | sites, 209 | WprArchiveSizeTypes.HTML_SIZE, 210 | config.puppeteerOptions.selectWprCount, 211 | ); 212 | case 'closestByScriptSize': 213 | return selectClosestWprArchives( 214 | wprs, 215 | sites, 216 | WprArchiveSizeTypes.INLINE_SCRIPT_SIZE, 217 | config.puppeteerOptions.selectWprCount, 218 | ); 219 | default: 220 | throw new Error( 221 | `unknown config.puppeteerOptions.selectWprCount: ${config.puppeteerOptions.selectWprCount}`, 222 | ); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/lib/wpr/types.ts: -------------------------------------------------------------------------------- 1 | import {IPageStructureSizes} from '@/lib/puppeteer'; 2 | import {ChildProcess} from 'child_process'; 3 | 4 | export interface IBaseWprConfig { 5 | bin: string; 6 | certFile: string; 7 | keyFile: string; 8 | injectScripts: string; 9 | } 10 | 11 | export interface IWprConfig extends IBaseWprConfig { 12 | httpPort: number; 13 | httpsPort: number; 14 | wprArchiveFilepath: string; 15 | stdoutFilepath: string; 16 | stderrFilepath: string; 17 | } 18 | 19 | export interface IWprProcess { 20 | process: ChildProcess; 21 | start(): Promise; 22 | stop(): Promise; 23 | kill(): Promise; 24 | onClose(cb: (code: number) => void): void; 25 | wait(): Promise; 26 | } 27 | 28 | export interface IWprProcessOptions { 29 | wprArchiveFilepath: string; 30 | httpPort: number; 31 | httpsPort: number; 32 | stdoutFilepath: string; 33 | stderrFilepath: string; 34 | } 35 | 36 | export interface IWprArchive { 37 | siteName: string; 38 | wprArchiveId: number; 39 | size: number; 40 | structureSizes: IPageStructureSizes; 41 | } 42 | 43 | export interface ISelectedWprArchives { 44 | wprArchives: IWprArchive[]; 45 | diff: number; 46 | } 47 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/README.md: -------------------------------------------------------------------------------- 1 | # porchmark-pretty-reporter 2 | 3 | Moved from https://github.com/re-gor/porchmark-pretty-reporter 4 | 5 | Reporter with simple charts and other stuff 6 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/__spec__/config.mock.ts: -------------------------------------------------------------------------------- 1 | import getDefaultConfig from '@/lib/config/default'; 2 | 3 | export default getDefaultConfig(); 4 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/__spec__/index.spec.ts: -------------------------------------------------------------------------------- 1 | import {ChartReport} from '../chartReport'; 2 | import config from './config.mock'; 3 | import rawReport from './jsonRawReport.mock'; 4 | 5 | describe('such usefull', () => { 6 | it('much test', () => { 7 | 8 | // tslint:disable-next-line:no-console 9 | console.info('Run reporter with mock...'); 10 | 11 | const reporter = new ChartReport(); 12 | reporter.prepareData({ 13 | startedAt: '', 14 | completedAt: '', 15 | status: '', 16 | statusMessage: '', 17 | config, 18 | report: rawReport, 19 | }); 20 | reporter.saveToFs('.', 'report'); 21 | 22 | expect(2 + 2).toBe(4); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/chartReport.ts: -------------------------------------------------------------------------------- 1 | import { IPrepareDataParams, IReport } from '@/types'; 2 | import {promises as fs} from 'fs'; 3 | import path from 'path'; 4 | 5 | import * as d3 from 'd3'; 6 | import { JSDOM } from 'jsdom'; 7 | import { AggregationBarChart } from './lib/aggregationBarChart'; 8 | import { LineChart } from './lib/lineChart'; 9 | import { html } from './lib/template'; 10 | 11 | export class ChartReport implements IReport { 12 | public result: string; 13 | 14 | constructor() { 15 | this.result = 'Nothing here. Use prepareData'; 16 | } 17 | 18 | public wrapHtml({body}: {body: string | Element}) { 19 | return ` 20 | 21 | 22 | 23 | 24 | Porchmark Report 25 | 26 | 27 | 28 | ${body} 29 | 30 | 31 | `; 32 | } 33 | 34 | public exposeInternalView() { 35 | throw new Error('exposeInternalView: Not implemented'); 36 | } 37 | 38 | public prepareData({report}: IPrepareDataParams) { 39 | const doc = (new JSDOM()).window.document; 40 | const body2 = d3.select(doc.body) 41 | .append('svg') 42 | .attr('viewBox', '0 0 300 300') 43 | .attr('style', 'max-width: 600px; max-heigth: 600px; font: 10px mono; display: block; border: 1px solid black'); 44 | 45 | body2.selectAll('rect') 46 | .data([0, 40, 100, 150, 200]) 47 | .join('rect') 48 | .attr('x', 0) 49 | .attr('y', (d) => d) 50 | .attr('width', '100') 51 | .attr('height', '20') 52 | .node(); 53 | 54 | const chartChunks = report.metrics.map((metric) => { 55 | const {name, title} = metric; 56 | 57 | const lineChart = report.data.allMetrics ? new LineChart().prepare({ 58 | metrics: report.data.allMetrics[name], 59 | sites: report.sites, 60 | }).node() : ''; 61 | 62 | const sortedLineChart = report.data.allMetrics ? new LineChart().prepare({ 63 | metrics: Object.entries(report.data.allMetrics[name]) 64 | .reduce((acc: {[i: string]: number[]}, [site, values]) => { 65 | acc[site] = values.sort((a, b) => a - b); 66 | return acc; 67 | }, {}), 68 | sites: report.sites, 69 | }).node() : ''; 70 | 71 | const barChart = new AggregationBarChart().prepare({ 72 | aggregations: report.data.metrics[name], 73 | metricName: name, 74 | sites: report.sites, 75 | }).node(); 76 | 77 | return html` 78 |
79 |

80 | ${title || name} 81 |

82 | 83 |
84 |

Runs

85 | ${lineChart} 86 |
87 | 88 |
89 |

Sorted Runs

90 | ${sortedLineChart} 91 |
92 | 93 |
94 |

Aggregations

95 | ${barChart} 96 |
97 |
98 | `; 99 | }); 100 | 101 | this.result = this.wrapHtml({ 102 | body: html` 103 |
104 | ${chartChunks} 105 |
106 | `.outerHTML, 107 | }); 108 | } 109 | 110 | public async saveToFs(workDir: string, id: string) { 111 | await fs.writeFile(path.resolve(workDir, `html-report-${id}.html`), this.result); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/index.ts: -------------------------------------------------------------------------------- 1 | export {ChartReport} from './chartReport'; 2 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/lib/aggregationBarChart.ts: -------------------------------------------------------------------------------- 1 | import { ISite } from '@/types'; 2 | import * as d3 from 'd3'; 3 | import {createSvg} from './utils'; 4 | 5 | interface MetricAggregation { 6 | [index: string]: { // key=aggregation 7 | [index: string]: number; // key=site name, value = metric value 8 | }; 9 | } 10 | 11 | interface Data { 12 | aggregations: MetricAggregation; 13 | metricName: string; 14 | sites: ISite[]; 15 | } 16 | 17 | interface Config { 18 | width?: number; 19 | height?: number; 20 | lowBound?: number; 21 | format?: string; 22 | labelSize?: number; 23 | valueSize?: number; 24 | } 25 | 26 | export class AggregationBarChart { 27 | public chart: d3.Selection; 28 | public config: Required; 29 | public format: (n: number) => string; 30 | public signedFormat: (n: number) => string; 31 | 32 | constructor( 33 | { 34 | height = 400, 35 | width = 1000, 36 | lowBound = 0, 37 | format = '.2f', 38 | labelSize = 16, 39 | valueSize = 12, 40 | }: Config = {}, 41 | ) { 42 | this.chart = createSvg(); 43 | this.config = { 44 | height, width, lowBound, format, labelSize, valueSize, 45 | }; 46 | 47 | this.format = d3.format(format); 48 | this.signedFormat = d3.format(`+${format}`); 49 | } 50 | 51 | public prepare( 52 | data: Data, 53 | ) { 54 | const chart = this.chart; 55 | const config = this.config; 56 | const {aggregations, sites} = data; 57 | const margin = {top: 30, right: 0, bottom: 30, left: 60}; 58 | 59 | const values: number[] = Object 60 | .values(aggregations) 61 | .reduce((acc: number[], agg) => { 62 | acc.push(...Object.values(agg)); 63 | 64 | return acc; 65 | }, []); 66 | 67 | const getName = (aggName: string, site: string) => `${aggName}-${site}`; 68 | 69 | const keys: string[] = Object 70 | .entries(aggregations) 71 | .reduce((acc: string[], [aggName, aggValues]) => { 72 | acc.push(...Object.keys(aggValues).map((site) => getName(aggName, site))); 73 | 74 | return acc; 75 | }, []); 76 | 77 | const marginTop = margin.top + sites.length * config.labelSize; 78 | 79 | const y = d3.scaleLinear() 80 | .domain([config.lowBound, d3.max(values) as number]) 81 | .range([0, config.height - margin.bottom - marginTop]); 82 | 83 | const x = d3.scaleBand() 84 | .domain(keys) 85 | .range([margin.left, config.width - margin.right]); 86 | 87 | const diffs = Object 88 | .values(aggregations) 89 | .reduce((acc: number[], aggValues) => { 90 | const baseline = aggValues[sites[0].name]; 91 | const aggrDiffs = Object 92 | .values(aggValues) 93 | .map((val, i) => i !== 0 ? val - baseline : NaN); 94 | 95 | acc.push(...aggrDiffs); 96 | 97 | return acc; 98 | }, []); 99 | 100 | // Bad typings in @types/d3 101 | // @ts-ignore 102 | const rectData: Array<[string, number, number]> = d3.zip(keys, values, diffs); 103 | 104 | // Root 105 | chart 106 | .attr('style', `max-width: ${config.width}px; width: 100%; height: ${config.height}px; font: ${config.valueSize}px monospace`) 107 | .attr('viewBox', `0 0 ${config.width} ${config.height}`); 108 | 109 | // Bars 110 | chart 111 | .append('g') 112 | .selectAll('rect') 113 | .data(rectData) 114 | .join('rect') 115 | .attr('fill', 'steelblue') 116 | .attr('width', x.bandwidth() - 5) 117 | .attr('height', ([, value]: [string, number, number]) => y(value) || 0) 118 | .attr('y', ([, value]: [string, number, number]) => config.height - margin.bottom - (y(value) || 0)) 119 | .attr('x', ([key]: [string, number, number]) => x(key) || 0) 120 | ; 121 | 122 | // Values 123 | chart.append('g') 124 | .selectAll('text') 125 | .data(rectData) 126 | .join('text') 127 | .attr('width', x.bandwidth()) 128 | .attr('height', 50) 129 | .attr('x', ([key]) => (x(key) || 0) + x.bandwidth() / 2) 130 | .attr('fill', 'white') 131 | .attr('text-anchor', 'middle') 132 | .text(([, value]) => this.format(value)) 133 | .attr('y', ( 134 | [, value]: [string, number, number], 135 | ) => config.height - margin.bottom - (y(value) || 0) + config.valueSize * 2, 136 | ) 137 | ; 138 | 139 | // diffs 140 | chart.append('g') 141 | .selectAll('text') 142 | .data(rectData) 143 | .join('text') 144 | .attr('width', x.bandwidth()) 145 | .attr('height', 50) 146 | .attr('x', ([key]) => (x(key) || 0) + x.bandwidth() / 2) 147 | .attr('fill', 'white') 148 | .attr('text-anchor', 'middle') 149 | .text(([, , diff]) => Number.isNaN(diff) ? '' : this.signedFormat(diff)) 150 | .attr('y', ([, value]) => config.height - margin.bottom - (y(value) || 0) + config.valueSize * 3) 151 | ; 152 | 153 | // x-axis 154 | chart.append('g') 155 | .attr('transform', `translate(0, ${config.height - margin.bottom})`) 156 | .call(d3.axisBottom(x)) 157 | ; 158 | 159 | // Legend 160 | chart.append('g') 161 | .attr('transform', `translate(0, 0)`) 162 | .attr('style', `font-size: ${config.labelSize}px`) 163 | .selectAll('text') 164 | .data(sites) 165 | .join('text') 166 | .attr('fill', 'black') 167 | .attr('y', (_, i) => (i + 1) * config.labelSize) 168 | .text((site) => `${site.name}: ${site.url}`) 169 | ; 170 | 171 | return chart; 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/lib/lineChart.ts: -------------------------------------------------------------------------------- 1 | import { ISite } from '@/types'; 2 | import * as d3 from 'd3'; 3 | import { createSvg } from './utils'; 4 | 5 | interface Config { 6 | width?: number; 7 | height?: number; 8 | format?: string; 9 | labelSize?: number; 10 | valueSize?: number; 11 | } 12 | 13 | interface Data { 14 | metrics: {[site: string]: number[]}; 15 | sites: ISite[]; 16 | } 17 | 18 | export class LineChart { 19 | public chart: d3.Selection; 20 | public config: Required; 21 | public format: (n: number) => string; 22 | 23 | constructor({ 24 | height = 400, 25 | width = 1000, 26 | format = '.2f', 27 | labelSize = 16, 28 | valueSize = 12, 29 | }: Config = {}) { 30 | this.chart = createSvg(); 31 | this.config = { 32 | height, 33 | width, 34 | format, 35 | labelSize, 36 | valueSize, 37 | }; 38 | 39 | this.format = d3.format(format); 40 | } 41 | 42 | public prepare(reportData: Data) { 43 | const chart = this.chart; 44 | const config = this.config; 45 | const {metrics, sites} = reportData; 46 | const margin = {top: 30, right: 60, bottom: 30, left: 60}; 47 | 48 | const marginTop = margin.top + sites.length * config.labelSize; 49 | 50 | const allValues = Object.values(metrics).reduce((acc: number[], siteValues) => { 51 | acc.push.apply(acc, siteValues); 52 | return acc; 53 | }, []); 54 | 55 | const valuesCount = Object.values(metrics)[0].length; 56 | 57 | const y = d3.scaleLinear() 58 | .domain([(d3.min(allValues) as number) * 0.995, (d3.max(allValues) as number) * 1.005]) 59 | .range([config.height - margin.bottom, marginTop]); 60 | 61 | const x = d3.scaleLinear() 62 | .domain([0, valuesCount - 1]) 63 | .range([margin.left, config.width - margin.right]); 64 | 65 | const xT = d3.scaleBand() 66 | .domain(d3.range(valuesCount - 1).map(String)) 67 | .range([margin.left, config.width - margin.right]); 68 | 69 | const colors = d3.scaleOrdinal() 70 | .domain(Object.keys(metrics)) 71 | .range(d3.schemeTableau10) as d3.ScaleOrdinal; 72 | 73 | // Root 74 | chart 75 | .attr('style', `max-width: ${config.width}px; width: 100%; height: ${config.height}px; font: ${config.valueSize}px monospace`) 76 | .attr('viewBox', `0 0 ${config.width} ${config.height}`) 77 | .append('style') 78 | .text('.title {fill: transparent} .title:hover {fill: rgba(70, 130, 180, 0.3); }; '); 79 | 80 | // Lines 81 | Object.entries(metrics).forEach(([site, data]) => { 82 | const line = d3.line() 83 | .x((_: number, i: number) => x(i) || 0) 84 | .y((d: number) => y(d) || 0); 85 | 86 | const g = chart.append('g'); 87 | 88 | g.append('path') 89 | .attr('d', line(data) || '') 90 | .attr('fill', 'none') 91 | .attr('stroke', colors(site)) 92 | .attr('stroke-width', 1.5) 93 | .attr('stroke-miterlimit', 1) 94 | ; 95 | 96 | g.selectAll('circle') 97 | .data(data) 98 | .join('circle') 99 | .attr('cx', (_: number, i: number) => x(i) || 0) 100 | .attr('cy', (d: number) => y(d) || 0) 101 | .attr('r', 2) 102 | .attr('fill', colors(site)) 103 | ; 104 | 105 | g.selectAll('rect') 106 | .data(data) 107 | .join('rect') 108 | .attr('class', 'title') 109 | .attr('width', xT.bandwidth()) 110 | .attr('height', config.height - marginTop - margin.bottom) 111 | .attr('x', (_, i) => (x(i) || 0) - xT.bandwidth() / 2) 112 | .attr('y', marginTop) 113 | .append('title') 114 | .text((d, i) => { 115 | const xString = `x: ${i}/${valuesCount - 1} (${this.format((i + 1) / valuesCount)});`; 116 | const yString = `y: ${this.format(d)};`; 117 | return `${xString} ${yString}`; 118 | }) 119 | ; 120 | }); 121 | 122 | // x-axis 123 | chart.append('g') 124 | .attr('transform', `translate(0, ${config.height - margin.bottom})`) 125 | .call(d3.axisBottom(x).ticks(valuesCount)) 126 | ; 127 | 128 | // y-axis 129 | chart.append('g') 130 | .attr('transform', `translate(${margin.left})`) 131 | .call(d3.axisLeft(y)) 132 | ; 133 | 134 | // y-axis right 135 | chart.append('g') 136 | .attr('transform', `translate(${config.width - margin.right})`) 137 | .call(d3.axisRight(y)) 138 | ; 139 | 140 | // Legend 141 | chart.append('g') 142 | .attr('transform', `translate(0, 0)`) 143 | .attr('style', `font-size: ${config.labelSize}px`) 144 | .selectAll('text') 145 | .data(sites) 146 | .join('text') 147 | .attr('fill', (site) => colors(site.name)) 148 | .attr('y', (_, i) => (i + 1) * config.labelSize) 149 | .text((site) => { 150 | const min = this.format(d3.min(metrics[site.name]) || 0); 151 | const max = this.format(d3.max(metrics[site.name]) || 0); 152 | const q95 = this.format(d3.quantile(metrics[site.name], 95) || 0); 153 | 154 | return `min: ${min}; max: ${max}; q95: ${q95}; ${site.name}: ${site.url};`; 155 | }) 156 | ; 157 | 158 | return chart; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/lib/template/html.ts: -------------------------------------------------------------------------------- 1 | 2 | import createTemplate from './template'; 3 | 4 | export default function createHtml(window: Window) { 5 | const template = createTemplate(window); 6 | const {document} = window; 7 | 8 | return template(function(str: string) { 9 | const tmp = document.createElement('template'); 10 | tmp.innerHTML = str.trim(); 11 | return document.importNode(tmp.content, true); 12 | }, function() { 13 | return document.createElement('span'); 14 | }); 15 | } 16 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/lib/template/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Many thanks to 3 | [observablehq/stdlib](https://github.com/observablehq/stdlib/tree/e9e9a6f322002fdc55a145342e393a52f6811f54) 4 | which distributed by ISC license 5 | */ 6 | 7 | import { Doc } from '../utils'; 8 | import createHtml from './html'; 9 | import createSvg from './svg'; 10 | 11 | const window = Doc.getWindow() as unknown as Window; 12 | 13 | export const html = createHtml(window); 14 | export const svg = createSvg(window); 15 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/lib/template/svg.ts: -------------------------------------------------------------------------------- 1 | import createTemplate from './template'; 2 | 3 | export default function createSvg(window: Window) { 4 | const template = createTemplate(window); 5 | const {document} = window; 6 | 7 | return template(function(str: string) { 8 | const root = document.createElementNS('http://www.w3.org/2000/svg', 'g'); 9 | root.innerHTML = str.trim(); 10 | return root; 11 | }, function() { 12 | return document.createElementNS('http://www.w3.org/2000/svg', 'g'); 13 | }); 14 | } 15 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/lib/template/template.ts: -------------------------------------------------------------------------------- 1 | type Render = (str: string) => T; 2 | type Wrapper = () => Element; 3 | type Other = Array>; 5 | 6 | export default function makeTemplate(window: Window) { 7 | // @ts-ignore 8 | const {document, Node, NodeFilter} = window; 9 | 10 | return function template(render: Render, wrapper: Wrapper) { 11 | return function(strings: TemplateStringsArray, ..._: Other): Element { 12 | let str = strings[0]; 13 | const parts = []; 14 | let part; 15 | let root = null; 16 | let node; let nodes; 17 | let walker; 18 | let i; 19 | let n; 20 | let j; 21 | let m; 22 | let k = -1; 23 | 24 | // Concatenate the text using comments as placeholders. 25 | for (i = 1, n = arguments.length; i < n; ++i) { 26 | part = arguments[i]; 27 | if (part instanceof Node) { 28 | parts[++k] = part; 29 | str += ''; 30 | } else if (Array.isArray(part)) { 31 | for (j = 0, m = part.length; j < m; ++j) { 32 | node = part[j]; 33 | if (node instanceof Node) { 34 | if (root === null) { 35 | parts[++k] = root = document.createDocumentFragment(); 36 | str += ''; 37 | } 38 | root.appendChild(node); 39 | } else { 40 | root = null; 41 | str += node; 42 | } 43 | } 44 | root = null; 45 | } else { 46 | str += part; 47 | } 48 | str += strings[i]; 49 | } 50 | 51 | // Render the text. 52 | root = render(str); 53 | 54 | // Walk the rendered content to replace comment placeholders. 55 | if (++k > 0) { 56 | nodes = new Array(k); 57 | walker = document.createTreeWalker(root, NodeFilter.SHOW_COMMENT, null, false); 58 | while (walker.nextNode()) { 59 | node = walker.currentNode; 60 | if (/^o:/.test(node.nodeValue as string)) { 61 | nodes[+(node.nodeValue as string).slice(2)] = node; 62 | } 63 | } 64 | for (i = 0; i < k; ++i) { 65 | node = nodes[i]; 66 | if (node) { 67 | node.parentNode.replaceChild(parts[i], node); 68 | } 69 | } 70 | } 71 | 72 | // Is the rendered content 73 | // … a parent of a single child? Detach and return the child. 74 | // … a document fragment? Replace the fragment with an element. 75 | // … some other node? Return it. 76 | return ( 77 | root.childNodes.length === 1 ? root.removeChild(root.firstChild as Element) 78 | : root.nodeType === 11 ? ((node = wrapper()).appendChild(root), node) 79 | : root 80 | ) as Element; 81 | }; 82 | }; 83 | } 84 | -------------------------------------------------------------------------------- /src/packages/porchmark-pretty-reporter/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import * as d3 from 'd3'; 2 | import { JSDOM } from 'jsdom'; 3 | 4 | export function createSvg() { 5 | const doc = Doc.getDocument(); 6 | const domSvg = doc.createElementNS('http://www.w3.org/2000/svg', 'svg'); 7 | const svg = d3.select(domSvg); 8 | 9 | Doc.clear(); 10 | 11 | return svg; 12 | } 13 | 14 | export class Doc { 15 | 16 | public static getWindow() { 17 | Doc.checkDom(); 18 | 19 | return Doc.dom.window; 20 | } 21 | 22 | public static getDocument() { 23 | return Doc.getWindow().document; 24 | } 25 | 26 | public static getBody() { 27 | return Doc.getDocument().body; 28 | } 29 | 30 | public static clear() { 31 | Doc.dom.window.document.write(); 32 | } 33 | private static dom: JSDOM; 34 | 35 | private static checkDom() { 36 | if (!Doc.dom) { 37 | Doc.dom = new JSDOM(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import {IComparison, IConfig, IConfigMetricsAggregation} from '@/lib/config'; 2 | 3 | export type RecursivePartial = { 4 | [P in keyof T]?: 5 | T[P] extends (infer U)[] ? RecursivePartial[] : 6 | T[P] extends object ? RecursivePartial : 7 | T[P]; 8 | }; 9 | 10 | export interface IOriginalMetrics { 11 | [index: string]: number; 12 | } 13 | 14 | export type SiteName = string; 15 | 16 | export interface ISite { 17 | name: SiteName; 18 | url: string; 19 | } 20 | 21 | export interface ICheckOptions { 22 | compareId: number; 23 | comparison: IComparison; 24 | config: IConfig; 25 | warmIterations: number; 26 | } 27 | 28 | export interface IMetric { 29 | name: string; 30 | title?: string; 31 | } 32 | 33 | export interface IJsonRawReportData { 34 | metrics: { 35 | [index: string]: { // key=metric, {DCL: {"q80": {"test1": 99, "test2": 100, "test3": 200}} 36 | [index: string]: { // key=aggregation 37 | [index: string]: number; // key=site name, value = metric value 38 | }; 39 | }; 40 | }; 41 | diffs: { 42 | [index: string]: { // key=metric, example {DCL: {"q80": {"test2": -12, "test3": 10}} 43 | [index: string]: { // key=aggregation 44 | [index: string]: number; // key=site name, value: diff with first site metric 45 | }, 46 | }, 47 | }; 48 | allMetrics: { 49 | // key=metric, {DCL: {"test1": [99, 98, 100], "test2": [100, 103, 102, "test3": [200, 200, 203]}} 50 | [index: string]: { 51 | [index: string]: number[], 52 | }; 53 | }; 54 | } 55 | 56 | export interface IJsonRawReport { 57 | sites: ISite[]; 58 | metrics: IMetric[]; 59 | metricAggregations: IConfigMetricsAggregation[]; 60 | data: IJsonRawReportData; 61 | } 62 | 63 | export interface IPrepareDataParams { 64 | startedAt: string; 65 | completedAt: string; 66 | status: string; 67 | statusMessage: string; 68 | config: IConfig; 69 | report: IJsonRawReport; 70 | } 71 | 72 | export interface IReport { 73 | /* Obtain and convert JsonReport to internal view */ 74 | prepareData(params: IPrepareDataParams): void; 75 | 76 | /* Flush internal data to file system */ 77 | saveToFs(workDir: string, id: string): void; 78 | 79 | /* For testing purposes only */ 80 | exposeInternalView(): any; 81 | } 82 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2017", 4 | "module": "commonjs", 5 | "allowJs": true, 6 | "checkJs": false, 7 | "outDir": "./dist", 8 | "rootDir": "./src", 9 | "resolveJsonModule": true, 10 | "strict": true, 11 | "noImplicitAny": true, 12 | "strictNullChecks": true, 13 | "strictFunctionTypes": true, 14 | "strictPropertyInitialization": true, 15 | "noImplicitThis": true, 16 | "alwaysStrict": true, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": true, 19 | "noImplicitReturns": true, 20 | "moduleResolution": "node", 21 | "baseUrl": "./src", 22 | "paths": { 23 | "@/*": ["./*"] 24 | }, 25 | "esModuleInterop": true, 26 | "sourceRoot": "src", 27 | "inlineSourceMap": true, 28 | "typeRoots": [ 29 | "./types", 30 | "./node_modules/@types" 31 | ] 32 | }, 33 | "exclude": [ 34 | "porchmark.conf.js", 35 | "jest.config.js", 36 | "node_modules", 37 | "dist", 38 | "types", 39 | "wpr", 40 | "install.js", 41 | "example" 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultSeverity": "error", 3 | "extends": [ 4 | "tslint:recommended" 5 | ], 6 | "rules": { 7 | "quotemark": [true, "single"], 8 | "only-arrow-functions": false, 9 | "array-type": false, 10 | "object-literal-sort-keys": false, 11 | "variable-name": { 12 | "options": [ 13 | "ban-keywords", 14 | "check-format", 15 | "allow-leading-underscore", 16 | "allow-pascal-case" 17 | ] 18 | }, 19 | "interface-name": false 20 | }, 21 | "jsRules": { 22 | "quotemark": [true, "single"], 23 | "only-arrow-functions": false, 24 | "array-type": false, 25 | "object-literal-sort-keys": false, 26 | "variable-name": { 27 | "options": [ 28 | "ban-keywords", 29 | "check-format", 30 | "allow-leading-underscore", 31 | "allow-pascal-case" 32 | ] 33 | } 34 | }, 35 | "rulesDirectory": [] 36 | } 37 | -------------------------------------------------------------------------------- /types/console.table/index.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'console.table'; 2 | -------------------------------------------------------------------------------- /types/jStat/index.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'jstat'; 2 | 3 | 4 | 5 | --------------------------------------------------------------------------------