├── .eslintignore ├── .eslintrc.json ├── .github └── workflows │ └── build.yml ├── .gitignore ├── .npmignore ├── .nycrc ├── .prettierrc.js ├── .prettierrc.js ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── playwright.config.ts ├── src ├── defaultReport.ts ├── index.ts ├── types.ts └── utils.ts ├── tests ├── defaultReport.spec.ts ├── reporter.spec.ts └── utils.spec.ts └── tsconfig.json /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true 5 | }, 6 | "extends": [ 7 | "airbnb-base" 8 | ], 9 | "parser": "@typescript-eslint/parser", 10 | "parserOptions": { 11 | "ecmaVersion": "latest", 12 | "sourceType": "module" 13 | }, 14 | "plugins": [ 15 | "@typescript-eslint", 16 | "prettier" 17 | ], 18 | "rules": { 19 | "import/extensions": 0, 20 | "import/no-extraneous-dependencies": 0, 21 | "import/no-unresolved": [0 22 | ], 23 | "no-empty-pattern": 0, 24 | "no-undef": 0, 25 | "no-use-before-define": 0 26 | } 27 | } -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Tests with Coverage 2 | on: 3 | push: 4 | branches: [main, dev] 5 | pull_request: 6 | branches: [main, dev] 7 | jobs: 8 | lint_test_coverage: 9 | timeout-minutes: 60 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-node@v3 14 | with: 15 | node-version: '16' 16 | - name: Install dependencies 17 | run: npm ci 18 | - name: Run ESLint 19 | run: npm run lint 20 | - name: tsc 21 | uses: icrawl/action-tsc@v1 22 | with: 23 | project: ./tsconfig.json 24 | - name: Install Playwright Chromium 25 | run: npx playwright install-deps chromium 26 | - name: Run Playwright tests 27 | run: npm run test:coverage 28 | - name: Coveralls GitHub Action 29 | uses: coverallsapp/github-action@1.1.3 30 | with: 31 | github-token: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | summary.txt 2 | results.txt 3 | 4 | # Logs 5 | logs 6 | *.log 7 | npm-debug.log* 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | 12 | # Diagnostic reports (https://nodejs.org/api/report.html) 13 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 14 | 15 | # Runtime data 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | 21 | # Directory for instrumented libs generated by jscoverage/JSCover 22 | lib-cov 23 | 24 | # Coverage directory used by tools like istanbul 25 | coverage 26 | *.lcov 27 | 28 | # nyc test coverage 29 | .nyc_output 30 | 31 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 32 | .grunt 33 | 34 | # Bower dependency directory (https://bower.io/) 35 | bower_components 36 | 37 | # node-waf configuration 38 | .lock-wscript 39 | 40 | # Compiled binary addons (https://nodejs.org/api/addons.html) 41 | build/Release 42 | 43 | # Dependency directories 44 | node_modules/ 45 | jspm_packages/ 46 | 47 | # TypeScript v1 declaration files 48 | typings/ 49 | 50 | # TypeScript cache 51 | *.tsbuildinfo 52 | 53 | # Optional npm cache directory 54 | .npm 55 | 56 | # Optional eslint cache 57 | .eslintcache 58 | 59 | # Microbundle cache 60 | .rpt2_cache/ 61 | .rts2_cache_cjs/ 62 | .rts2_cache_es/ 63 | .rts2_cache_umd/ 64 | 65 | # Optional REPL history 66 | .node_repl_history 67 | 68 | # Output of 'npm pack' 69 | *.tgz 70 | 71 | # Yarn Integrity file 72 | .yarn-integrity 73 | 74 | # dotenv environment variables file 75 | .env 76 | .env.test 77 | 78 | # parcel-bundler cache (https://parceljs.org/) 79 | .cache 80 | 81 | # Next.js build output 82 | .next 83 | 84 | # Nuxt.js build / generate output 85 | .nuxt 86 | dist 87 | 88 | # Gatsby files 89 | .cache/ 90 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 91 | # https://nextjs.org/blog/next-9-1#public-directory-support 92 | # public 93 | 94 | # vuepress build output 95 | .vuepress/dist 96 | 97 | # Serverless directories 98 | .serverless/ 99 | 100 | # FuseBox cache 101 | .fusebox/ 102 | 103 | # DynamoDB Local files 104 | .dynamodb/ 105 | 106 | # TernJS port file 107 | .tern-port 108 | /test-results/ 109 | /playwright-report/ 110 | /playwright/.cache/ 111 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stephenkilbourn/playwright-report-summary/ceb310e1382a784192821fe92e03e32bca7cc6aa/.npmignore -------------------------------------------------------------------------------- /.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@istanbuljs/nyc-config-typescript", 3 | "all": true, 4 | "include": [ 5 | "src/**/*.ts" 6 | ], 7 | "check-coverage": true, 8 | "lines": 90, 9 | "statements": 90, 10 | "functions": 90, 11 | "branches": 90 12 | } -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | arrowParens: 'always', 3 | bracketSpacing: true, 4 | printWidth: 80, 5 | singleQuote: true, 6 | tabWidth: 2, 7 | trailingComma: 'all', 8 | }; -------------------------------------------------------------------------------- /.prettierrc.js : -------------------------------------------------------------------------------- 1 | module.exports = { 2 | arrowParens: 'always', 3 | bracketSpacing: true, 4 | printWidth: 80, 5 | singleQuote: true, 6 | tabWidth: 2, 7 | trailingComma: 'all', 8 | }; 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Stephen Kilbourn 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 📜 🎭 Playwright Report Summary 🎭 📜 2 | 3 | [![Coverage Status](https://coveralls.io/repos/github/stephenkilbourn/playwright-report-summary/badge.svg?branch=main)](https://coveralls.io/github/stephenkilbourn/playwright-report-summary?branch=main) 4 | 5 | Small text based custom reporter for Playwright. 6 | It can be handy to publish test results for things such as an SNS message or minimal Slack update. This Tool allows you to generate smaller reports with basic info about your test run. 7 | 8 | ## Table of Contents 9 | 10 | * [✨ Installation ✨](#-configuration-) 11 | * [📍 Configuration 📍](#-configuration-) 12 | * [ Default Output 📜](#default-output-) 13 | * [Customizing Outputs 👨‍💻](#customizing-outputs-) 14 | * [Available Stats 🧰](#available-stats-) 15 | 16 | ## ✨ Installation ✨ 17 | 18 | Run following commands: 19 | 20 | ### npm 21 | 22 | `npm install @skilbourn/playwright-report-summary --save-dev` 23 | 24 | ### yarn 25 | 26 | `yarn add @skilbourn/playwright-report-summary --dev` 27 | 28 | ## 📍 Configuration 📍 29 | 30 | Modify your `playwright.config.ts` file to include the reporter: 31 | 32 | ```typescript 33 | reporter: [ 34 | ['@skilbourn/playwright-report-summary', { outputFile: 'custom-summary.txt' }]], 35 | ['html'], // other reporters 36 | ['dot'] 37 | ], 38 | ``` 39 | 40 | The default output location will be to your root as `summary.txt` Including the optional `outputFile` parameter allows you to specify a custom report location. 41 | 42 | ## Default Output 📜 43 | 44 | If you do not pass an `outputFile` option, then the summary will be generated to a `summary.txt` file in the following format: 45 | 46 | ```txt 47 | Total Tests in Suite: 30, 48 | Total Tests Completed: 30, 49 | Tests Passed: 27, 50 | Tests Failed: 0, 51 | Flaky Tests: 0, 52 | Test run was failure free? true, 53 | Test Skipped: 3, 54 | Duration of CPU usage in ms: 75188, 55 | Duration of entire test run in ms: 12531, 56 | Average Test Duration in ms:2506.3, 57 | Test Suite Duration: 00:13 (mm:ss), 58 | Average Test Duration: 00:03 (mm:ss), 59 | Number of workers used for test run: 6 60 | ``` 61 | 62 | ## Customizing Outputs 👨‍💻 63 | 64 | You may also create a custom report by leveraging the values in the [`stats`](#available-stats-🧰) object. To add a custom report leveraging your stats, create a function in the format: 65 | 66 | ```typescript 67 | import type { Stats } from '@skilbourn/playwright-report-summary'; 68 | 69 | function customReport(stats: Stats) { 70 | return `Greetings, hello, ${stats.expectedResults} tests passed as expected in ${stats.formattedDurationSuite}`; 71 | } 72 | 73 | export default customReport; 74 | ``` 75 | 76 | and then modify your `playwright.config.ts` file with the following: 77 | 78 | ```typescript 79 | import type { PlaywrightTestConfig } from '@playwright/test'; 80 | import { devices } from '@playwright/test'; 81 | 82 | import customReport from './customReport'; 83 | // Your custom report path and preferred name 84 | 85 | 86 | const config: PlaywrightTestConfig = { 87 | ... 88 | reporter: [ 89 | ['@skilbourn/playwright-report-summary', { outputFile: 'custom-summary.txt', inputTemplate: customReport }]] 90 | ], 91 | 92 | ``` 93 | 94 | this will generate a `custom-summary.txt` file such as : 95 | 96 | ```txt 97 | hello, 50 tests passed as expected in 03:51 (mm:ss) 98 | ``` 99 | 100 | ## Available Stats 🧰 101 | 102 | The `stats` object provides information on your test suite: 103 | 104 | | **Name** | **type** | **Description** | 105 | |--------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| 106 | | testsInSuite | number | Total number of tests in suite | 107 | | totalTestsRun | number | total tests run. Retried tests can make this value larger than testsInSuite | 108 | | expectedResults | number | total test finished as [expected](https://playwright.dev/docs/api/class-testcase#test-case-expected-status) | 109 | | unexpectedResults | number | total tests not finished as expected | 110 | | flakyTests | number | total of tests that passed when retried | 111 | | testMarkedSkipped | number | total tests marked as test.skip() or test.fixme() | 112 | | failureFree | boolean | returns `true` if suite completes with all test completing as expected after retries | 113 | | durationCPU | number | total milliseconds spent run tests. If tests run parallel with multiple workers, this value will be larger than the duration of running the suite | 114 | | durationSuite | number | milliseconds to complete all tests in suite | 115 | | avgTestDuration | number | average test duration of all tests in milliseconds | 116 | | formattedDurationSuite | string | duration to complete all tests in mm:ss format | 117 | | formattedAvgTestDuration | string | average test duration of all tests in mm:ss format | 118 | | failures | object | an object containing each failure in the format `{[test.title: result.status]}` Retries with failures will populate this with multiple entries of the same test | 119 | | workers | number | total number of workers used to run the suite | 120 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@skilbourn/playwright-report-summary", 3 | "version": "1.1.1", 4 | "description": "generate a customizable text summary of your playwright test results", 5 | "main": "./dist/index.js", 6 | "types": "./dist/types.d.ts", 7 | "files": [ 8 | "/dist" 9 | ], 10 | "scripts": { 11 | "build": "tsc -p ./tsconfig.json", 12 | "lint": "npx eslint . --ext .ts", 13 | "prettier": "prettier --write --loglevel warn \"**/**/*.ts\"", 14 | "prepublish": "npm run build", 15 | "test": "npx playwright test", 16 | "typecheck": "tsc", 17 | "test:coverage": "nyc playwright test && nyc report --reporter=lcov --reporter=html" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "git+https://github.com/stephenkilbourn/playwright-report-summary.git" 22 | }, 23 | "keywords": [ 24 | "playwright", 25 | "report" 26 | ], 27 | "author": "Stephen Kilbourn ", 28 | "license": "MIT", 29 | "bugs": { 30 | "url": "https://github.com/stephenkilbourn/playwright-report-summary/issues" 31 | }, 32 | "homepage": "https://github.com/stephenkilbourn/playwright-report-summary#readme", 33 | "devDependencies": { 34 | "@istanbuljs/nyc-config-typescript": "^1.0.2", 35 | "@playwright/test": "^1.27.1", 36 | "@sinonjs/fake-timers": "^9.1.2", 37 | "@types/node": "^18.7.23", 38 | "@types/sinonjs__fake-timers": "^8.1.2", 39 | "@typescript-eslint/eslint-plugin": "^5.38.1", 40 | "@typescript-eslint/parser": "^5.38.1", 41 | "babel-plugin-istanbul": "^6.1.1", 42 | "eslint": "^8.24.0", 43 | "eslint-config-airbnb-base": "^15.0.0", 44 | "eslint-config-prettier": "^8.5.0", 45 | "eslint-plugin-prettier": "^4.2.1", 46 | "mock-fs": "^5.1.4", 47 | "nyc": "^15.1.0", 48 | "playwright": "^1.27.1", 49 | "playwright-test-coverage": "^1.2.0", 50 | "prettier": "^2.7.1", 51 | "source-map-support": "^0.5.21", 52 | "typescript": "^4.8.4" 53 | } 54 | } -------------------------------------------------------------------------------- /playwright.config.ts: -------------------------------------------------------------------------------- 1 | import type { PlaywrightTestConfig } from '@playwright/test'; 2 | import { devices } from '@playwright/test'; 3 | 4 | /** 5 | * See https://playwright.dev/docs/test-configuration. 6 | */ 7 | 8 | const config: PlaywrightTestConfig = { 9 | testDir: './tests', 10 | /* Maximum time one test can run for. */ 11 | timeout: 30 * 1000, 12 | expect: { 13 | /** 14 | * Maximum time expect() should wait for the condition to be met. 15 | * For example in `await expect(locator).toHaveText();` 16 | */ 17 | timeout: 5000, 18 | }, 19 | /* Run tests in files in parallel */ 20 | fullyParallel: true, 21 | /* Fail the build on CI if you accidentally left test.only in the source code. */ 22 | forbidOnly: !!process.env.CI, 23 | /* Retry on CI only */ 24 | retries: process.env.CI ? 2 : 0, 25 | /* Opt out of parallel tests on CI. */ 26 | workers: 1, 27 | /* Reporter to use. See https://playwright.dev/docs/test-reporters */ 28 | reporter: [['dot']], 29 | /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ 30 | use: { 31 | actionTimeout: 0, 32 | trace: 'on-first-retry', 33 | }, 34 | 35 | /* Configure projects for major browsers */ 36 | projects: [ 37 | { 38 | name: 'chromium', 39 | use: { 40 | ...devices['Desktop Chrome'], 41 | }, 42 | }, 43 | ], 44 | }; 45 | 46 | export default config; 47 | -------------------------------------------------------------------------------- /src/defaultReport.ts: -------------------------------------------------------------------------------- 1 | import type { Stats } from './types'; 2 | 3 | export default class DefaultReport { 4 | stats: Stats; 5 | 6 | constructor(stats) { 7 | this.stats = stats; 8 | } 9 | 10 | templateReport(): string { 11 | return ( 12 | // eslint-disable-next-line indent 13 | `Total Tests in Suite: ${this.stats.testsInSuite}, 14 | Total Tests Completed: ${this.stats.totalTestsRun}, 15 | Tests Passed: ${this.stats.expectedResults}, 16 | Tests Failed: ${this.stats.unexpectedResults}, 17 | Flaky Tests: ${this.stats.flakyTests}, 18 | Test Skipped: ${this.stats.testMarkedSkipped}, 19 | Test run was failure free? ${this.stats.failureFree}, 20 | Duration of CPU usage in ms: ${this.stats.durationCPU}, 21 | Duration of entire test run in ms: ${this.stats.durationSuite}, 22 | Average Test Duration in ms: ${this.stats.avgTestDuration}, 23 | Test Suite Duration: ${this.stats.formattedDurationSuite}, 24 | Average Test Duration: ${this.stats.formattedAvgTestDuration}, 25 | Number of workers used for test run: ${this.stats.workers}` 26 | ); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs'; 2 | 3 | import * as path from 'path'; 4 | 5 | import { Reporter, TestCase, TestResult } from '@playwright/test/reporter'; 6 | import type { Stats, InputTemplate, OutputFile } from './types'; 7 | import millisToMinuteSeconds from './utils'; 8 | import DefaultReport from './defaultReport'; 9 | 10 | const initialStats = (): Stats => ({ 11 | testsInSuite: 0, 12 | totalTestsRun: 0, 13 | expectedResults: 0, 14 | unexpectedResults: 0, 15 | flakyTests: 0, 16 | testMarkedSkipped: 0, 17 | failureFree: true, 18 | durationCPU: 0, 19 | durationSuite: 0, 20 | avgTestDuration: 0, 21 | formattedDurationSuite: '', 22 | formattedAvgTestDuration: '', 23 | failures: {}, 24 | workers: 1, 25 | }); 26 | 27 | class PlaywrightReportSummary implements Reporter { 28 | outputFile: OutputFile; 29 | 30 | private startTime: number; 31 | 32 | private endTime: number; 33 | 34 | inputTemplate: InputTemplate; 35 | 36 | stats: Stats; 37 | 38 | constructor( 39 | options: { outputFile?: string; inputTemplate?: () => string } = {}, 40 | ) { 41 | this.outputFile = options.outputFile; 42 | this.inputTemplate = options.inputTemplate; 43 | } 44 | 45 | onBegin(config, suite) { 46 | this.startTime = Date.now(); 47 | this.stats = initialStats(); 48 | this.stats.testsInSuite = suite.allTests().length; 49 | this.stats.workers = config.workers; 50 | } 51 | 52 | async onTestEnd(test: TestCase, result: TestResult) { 53 | const outcome = test.outcome(); 54 | const { retry } = result; 55 | 56 | if (outcome === 'expected') this.stats.expectedResults += 1; 57 | if (outcome === 'skipped') this.stats.testMarkedSkipped += 1; 58 | if (outcome === 'flaky') this.stats.flakyTests += 1; 59 | if (outcome === 'unexpected') { 60 | this.stats.failures[test.title] = result.status; 61 | if (retry === 0) { 62 | this.stats.unexpectedResults += 1; 63 | } 64 | } 65 | this.stats.totalTestsRun += 1; 66 | this.stats.durationCPU += result.duration; 67 | this.stats.failureFree = (this.stats.unexpectedResults - this.stats.flakyTests) === 0; 68 | } 69 | 70 | async onEnd() { 71 | this.endTime = Date.now(); 72 | this.stats.durationSuite = this.endTime - this.startTime; 73 | this.stats.avgTestDuration = Math.ceil( 74 | this.stats.durationCPU / (this.stats.totalTestsRun || 1), 75 | ); 76 | this.stats.formattedAvgTestDuration = millisToMinuteSeconds( 77 | this.stats.avgTestDuration, 78 | ); 79 | this.stats.formattedDurationSuite = millisToMinuteSeconds( 80 | this.stats.durationSuite, 81 | ); 82 | outputReport(this.stats, this.inputTemplate, this.outputFile); 83 | } 84 | } 85 | 86 | function outputReport( 87 | stats: Stats, 88 | inputTemplate?: Function, 89 | outputFile: string = 'results.txt', 90 | ) { 91 | let reportString: string; 92 | const report = new DefaultReport(stats); 93 | if (typeof inputTemplate === 'undefined') { 94 | reportString = report.templateReport(); 95 | } else { 96 | reportString = inputTemplate(stats); 97 | if (typeof reportString !== 'string') { 98 | throw new Error('custom input templates must return a string'); 99 | } 100 | } 101 | 102 | fs.mkdirSync(path.dirname(outputFile), { recursive: true }); 103 | fs.writeFileSync(outputFile, reportString); 104 | } 105 | 106 | export default PlaywrightReportSummary; 107 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export type Stats = { 2 | testsInSuite: number; 3 | totalTestsRun: number; 4 | expectedResults: number; 5 | unexpectedResults: number; 6 | flakyTests: number; 7 | testMarkedSkipped: number; 8 | failureFree: boolean; 9 | durationCPU: number; 10 | durationSuite: number; 11 | avgTestDuration: number; 12 | formattedDurationSuite: string; 13 | formattedAvgTestDuration: string; 14 | failures: object; 15 | workers: number; 16 | }; 17 | 18 | export type OutputFile = string; 19 | 20 | export type InputTemplate = () => string; 21 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | function millisToMinuteSeconds(milliseconds: number) { 2 | const min = Math.floor(milliseconds / 60000); 3 | const sec = Math.ceil((milliseconds % 60000) / 1000); 4 | const minString = pad(min); 5 | const secString = pad(sec); 6 | 7 | if (milliseconds > 0) { 8 | if (milliseconds < 1000) { 9 | return '00:01 (mm:ss)'; 10 | } 11 | if (sec < 10) { 12 | return `${minString}:${secString} (mm:ss)`; 13 | } 14 | return `${minString}:${secString} (mm:ss)`; 15 | } 16 | return '00:00 (mm:ss)'; 17 | } 18 | 19 | function pad(value: number) { 20 | if (value < 10) { 21 | return `0${value}`; 22 | } 23 | return value; 24 | } 25 | 26 | export default millisToMinuteSeconds; 27 | -------------------------------------------------------------------------------- /tests/defaultReport.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from '@playwright/test'; 2 | import DefaultReport from '../src/defaultReport'; 3 | 4 | const testStats = { 5 | testsInSuite: 0, 6 | totalTestsRun: 0, 7 | expectedResults: 0, 8 | unexpectedResults: 0, 9 | flakyTests: 0, 10 | testMarkedSkipped: 0, 11 | failureFree: true, 12 | durationCPU: 0, 13 | durationSuite: 0, 14 | avgTestDuration: 0, 15 | formattedDurationSuite: '00:00 (mm:ss)', 16 | formattedAvgTestDuration: '00:01 (mm:ss)', 17 | failures: {}, 18 | workers: 1, 19 | }; 20 | 21 | const defaultString = `Total Tests in Suite: 0, 22 | Total Tests Completed: 0, 23 | Tests Passed: 0, 24 | Tests Failed: 0, 25 | Flaky Tests: 0, 26 | Test Skipped: 0, 27 | Test run was failure free? true, 28 | Duration of CPU usage in ms: 0, 29 | Duration of entire test run in ms: 0, 30 | Average Test Duration in ms: 0, 31 | Test Suite Duration: 00:00 (mm:ss), 32 | Average Test Duration: 00:01 (mm:ss), 33 | Number of workers used for test run: 1`; 34 | 35 | test.describe('default report format', () => { 36 | test('initial stats', () => { 37 | const report = new DefaultReport(testStats); 38 | expect(report.templateReport()).toEqual(defaultString); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /tests/reporter.spec.ts: -------------------------------------------------------------------------------- 1 | import { Config, expect, test } from '@playwright/test'; 2 | import { Suite, TestCase, TestResult } from '@playwright/test/reporter'; 3 | import { readFileSync } from 'fs'; 4 | import mock from 'mock-fs'; 5 | import PlaywrightReportSummary from '../src/index'; 6 | 7 | const FakeTimers = require('@sinonjs/fake-timers'); 8 | 9 | type MockConfig = Pick; 10 | type MockSuite = Pick; 11 | type MockTestCase = Pick; 12 | type MockTestResult = Pick; 13 | 14 | const mockedPassingTest: MockTestCase = { 15 | expectedStatus: 'passed', 16 | outcome: () => 'expected', 17 | title: 'mocked test', 18 | }; 19 | const mockedPassingResult: MockTestResult = { 20 | status: 'passed', 21 | duration: 10000, 22 | retry: 0, 23 | }; 24 | 25 | const mockedSkippedTest: MockTestCase = { 26 | expectedStatus: 'skipped', 27 | outcome: () => 'skipped', 28 | title: 'mocked test', 29 | }; 30 | const mockedSkippedResult: MockTestResult = { 31 | status: 'skipped', 32 | duration: 0, 33 | retry: 0, 34 | }; 35 | 36 | const mockedFailingTest: MockTestCase = { 37 | expectedStatus: 'passed', 38 | outcome: () => 'unexpected', 39 | title: 'failed mocked test', 40 | }; 41 | const mockedFailingResult: MockTestResult = { 42 | status: 'failed', 43 | duration: 10000, 44 | retry: 0, 45 | }; 46 | 47 | const mockedTimedOutTest: MockTestCase = { 48 | expectedStatus: 'passed', 49 | outcome: () => 'unexpected', 50 | title: 'timed out mocked test', 51 | }; 52 | const mockedTimedOutResult: MockTestResult = { 53 | status: 'timedOut', 54 | duration: 10000, 55 | retry: 1, 56 | }; 57 | 58 | const mockedPassingTestAfterRetries: MockTestCase = { 59 | expectedStatus: 'passed', 60 | outcome: () => 'flaky', 61 | title: 'timed out mocked test', 62 | }; 63 | 64 | const mockedPassingResultAfterRetries: MockTestResult = { 65 | status: 'passed', 66 | duration: 10000, 67 | retry: 2, 68 | }; 69 | 70 | test.describe('Reporter handles stats', () => { 71 | test('parses default results successfully', async () => { 72 | const mockConfig: MockConfig = { 73 | workers: 1, 74 | }; 75 | const mockSuite: MockSuite = { 76 | allTests: () => [], 77 | }; 78 | const playwrightReportSummary = new PlaywrightReportSummary(); 79 | 80 | const clock = FakeTimers.install(); 81 | 82 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 83 | clock.tick(10000); 84 | await playwrightReportSummary.onEnd(); 85 | clock.uninstall(); 86 | 87 | const results = await playwrightReportSummary.stats; 88 | expect(results).toEqual({ 89 | testsInSuite: 0, 90 | totalTestsRun: 0, 91 | expectedResults: 0, 92 | unexpectedResults: 0, 93 | flakyTests: 0, 94 | testMarkedSkipped: 0, 95 | failureFree: true, 96 | durationCPU: 0, 97 | durationSuite: 10000, 98 | avgTestDuration: 0, 99 | formattedDurationSuite: '00:10 (mm:ss)', 100 | formattedAvgTestDuration: '00:00 (mm:ss)', 101 | failures: {}, 102 | workers: 1, 103 | }); 104 | }); 105 | 106 | test('updates stats if test passes', async () => { 107 | const mockConfig: MockConfig = { 108 | workers: 1, 109 | }; 110 | const mockSuite: MockSuite = { 111 | // @ts-ignore 112 | allTests: () => [mockedPassingTest], 113 | }; 114 | const clock = FakeTimers.install(); 115 | const playwrightReportSummary = new PlaywrightReportSummary(); 116 | 117 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 118 | await playwrightReportSummary.onTestEnd( 119 | // @ts-ignore 120 | mockedPassingTest, 121 | mockedPassingResult, 122 | ); 123 | clock.tick(10000); 124 | await playwrightReportSummary.onEnd(); 125 | clock.uninstall(); 126 | 127 | const results = await playwrightReportSummary.stats; 128 | expect(results).toEqual({ 129 | testsInSuite: 1, 130 | totalTestsRun: 1, 131 | expectedResults: 1, 132 | unexpectedResults: 0, 133 | flakyTests: 0, 134 | testMarkedSkipped: 0, 135 | failureFree: true, 136 | durationCPU: 10000, 137 | durationSuite: 10000, 138 | avgTestDuration: 10000, 139 | formattedDurationSuite: '00:10 (mm:ss)', 140 | formattedAvgTestDuration: '00:10 (mm:ss)', 141 | failures: {}, 142 | workers: 1, 143 | }); 144 | }); 145 | 146 | test('updates stats if test marked skipped', async () => { 147 | const mockConfig: MockConfig = { 148 | workers: 1, 149 | }; 150 | const mockSuite: MockSuite = { 151 | // @ts-ignore 152 | allTests: () => [mockedSkippedTest], 153 | }; 154 | const playwrightReportSummary = new PlaywrightReportSummary(); 155 | const clock = FakeTimers.install(); 156 | 157 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 158 | await playwrightReportSummary.onTestEnd( 159 | // @ts-ignore 160 | mockedSkippedTest, 161 | mockedSkippedResult, 162 | ); 163 | clock.tick(10000); 164 | await playwrightReportSummary.onEnd(); 165 | clock.uninstall(); 166 | 167 | const results = await playwrightReportSummary.stats; 168 | expect(results).toEqual({ 169 | testsInSuite: 1, 170 | totalTestsRun: 1, 171 | expectedResults: 0, 172 | unexpectedResults: 0, 173 | flakyTests: 0, 174 | testMarkedSkipped: 1, 175 | failureFree: true, 176 | durationCPU: 0, 177 | durationSuite: 10000, 178 | avgTestDuration: 0, 179 | formattedDurationSuite: '00:10 (mm:ss)', 180 | formattedAvgTestDuration: '00:00 (mm:ss)', 181 | failures: {}, 182 | workers: 1, 183 | }); 184 | }); 185 | 186 | test('updates stats if 2 tests pass', async () => { 187 | const mockConfig: MockConfig = { 188 | workers: 1, 189 | }; 190 | const mockSuite: MockSuite = { 191 | // @ts-ignore 192 | allTests: () => [mockedPassingTest, mockedPassingTest], 193 | }; 194 | const playwrightReportSummary = new PlaywrightReportSummary(); 195 | const clock = FakeTimers.install(); 196 | 197 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 198 | clock.tick(10000); 199 | await playwrightReportSummary.onTestEnd( 200 | // @ts-ignore 201 | mockedPassingTest, 202 | mockedPassingResult, 203 | ); 204 | clock.tick(10000); 205 | await playwrightReportSummary.onTestEnd( 206 | // @ts-ignore 207 | mockedPassingTest, 208 | mockedPassingResult, 209 | ); 210 | await playwrightReportSummary.onEnd(); 211 | clock.uninstall(); 212 | 213 | const results = await playwrightReportSummary.stats; 214 | expect(results).toEqual({ 215 | testsInSuite: 2, 216 | totalTestsRun: 2, 217 | expectedResults: 2, 218 | unexpectedResults: 0, 219 | flakyTests: 0, 220 | testMarkedSkipped: 0, 221 | failureFree: true, 222 | durationCPU: 20000, 223 | durationSuite: 20000, 224 | avgTestDuration: 10000, 225 | formattedDurationSuite: '00:20 (mm:ss)', 226 | formattedAvgTestDuration: '00:10 (mm:ss)', 227 | failures: {}, 228 | workers: 1, 229 | }); 230 | }); 231 | 232 | test('show changed workers & suite duration if multiple workers', async () => { 233 | const mockConfig: MockConfig = { 234 | workers: 2, 235 | }; 236 | const mockSuite: MockSuite = { 237 | // @ts-ignore 238 | allTests: () => [mockedPassingTest, mockedPassingTest], 239 | }; 240 | const playwrightReportSummary = new PlaywrightReportSummary(); 241 | const clock = FakeTimers.install(); 242 | 243 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 244 | 245 | await playwrightReportSummary.onTestEnd( 246 | // @ts-ignore 247 | mockedPassingTest, 248 | mockedPassingResult, 249 | ); 250 | await playwrightReportSummary.onTestEnd( 251 | // @ts-ignore 252 | mockedPassingTest, 253 | mockedPassingResult, 254 | ); 255 | clock.tick(10000); 256 | await playwrightReportSummary.onEnd(); 257 | clock.uninstall(); 258 | 259 | const results = await playwrightReportSummary.stats; 260 | expect(results).toEqual({ 261 | testsInSuite: 2, 262 | totalTestsRun: 2, 263 | expectedResults: 2, 264 | unexpectedResults: 0, 265 | flakyTests: 0, 266 | testMarkedSkipped: 0, 267 | failureFree: true, 268 | durationCPU: 20000, 269 | durationSuite: 10000, 270 | avgTestDuration: 10000, 271 | formattedDurationSuite: '00:10 (mm:ss)', 272 | formattedAvgTestDuration: '00:10 (mm:ss)', 273 | failures: {}, 274 | workers: 2, 275 | }); 276 | }); 277 | 278 | test('show failure if tests fails', async () => { 279 | const mockConfig: MockConfig = { 280 | workers: 2, 281 | }; 282 | const mockSuite: MockSuite = { 283 | // @ts-ignore 284 | allTests: () => [mockedFailingTest, mockedPassingTest], 285 | }; 286 | const playwrightReportSummary = new PlaywrightReportSummary(); 287 | const clock = FakeTimers.install(); 288 | 289 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 290 | await playwrightReportSummary.onTestEnd( 291 | // @ts-ignore 292 | mockedFailingTest, 293 | mockedFailingResult, 294 | ); 295 | await playwrightReportSummary.onTestEnd( 296 | // @ts-ignore 297 | mockedPassingTest, 298 | mockedPassingResult, 299 | ); 300 | 301 | clock.tick(10000); 302 | await playwrightReportSummary.onEnd(); 303 | clock.uninstall(); 304 | 305 | const results = await playwrightReportSummary.stats; 306 | expect(results).toEqual({ 307 | testsInSuite: 2, 308 | totalTestsRun: 2, 309 | expectedResults: 1, 310 | unexpectedResults: 1, 311 | flakyTests: 0, 312 | testMarkedSkipped: 0, 313 | failureFree: false, 314 | durationCPU: 20000, 315 | durationSuite: 10000, 316 | avgTestDuration: 10000, 317 | formattedDurationSuite: '00:10 (mm:ss)', 318 | formattedAvgTestDuration: '00:10 (mm:ss)', 319 | failures: { 'failed mocked test': 'failed' }, 320 | workers: 2, 321 | }); 322 | }); 323 | 324 | test('count as flaky if tests fails and then passes', async () => { 325 | const mockConfig: MockConfig = { 326 | workers: 1, 327 | }; 328 | const mockSuite: MockSuite = { 329 | // @ts-ignore 330 | allTests: () => [mockedPassingTestAfterRetries], 331 | }; 332 | const playwrightReportSummary = new PlaywrightReportSummary(); 333 | const clock = FakeTimers.install(); 334 | 335 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 336 | await playwrightReportSummary.onTestEnd( 337 | // @ts-ignore 338 | mockedFailingTest, 339 | mockedFailingResult, 340 | ); 341 | await playwrightReportSummary.onTestEnd( 342 | // @ts-ignore 343 | mockedTimedOutTest, 344 | mockedTimedOutResult, 345 | ); 346 | await playwrightReportSummary.onTestEnd( 347 | // @ts-ignore 348 | mockedPassingTestAfterRetries, 349 | mockedPassingResultAfterRetries, 350 | ); 351 | 352 | clock.tick(30000); 353 | await playwrightReportSummary.onEnd(); 354 | clock.uninstall(); 355 | 356 | const results = await playwrightReportSummary.stats; 357 | expect(results).toEqual({ 358 | testsInSuite: 1, 359 | totalTestsRun: 3, 360 | expectedResults: 0, 361 | unexpectedResults: 1, 362 | flakyTests: 1, 363 | testMarkedSkipped: 0, 364 | failureFree: true, 365 | durationCPU: 30000, 366 | durationSuite: 30000, 367 | avgTestDuration: 10000, 368 | formattedDurationSuite: '00:30 (mm:ss)', 369 | formattedAvgTestDuration: '00:10 (mm:ss)', 370 | failures: { 371 | 'failed mocked test': 'failed', 372 | 'timed out mocked test': 'timedOut', 373 | }, 374 | workers: 1, 375 | }); 376 | }); 377 | }); 378 | 379 | test.describe('outputReport correctly writes files', () => { 380 | test.beforeAll(() => { 381 | mock({}); 382 | }); 383 | test.afterAll(() => { 384 | mock.restore(); 385 | }); 386 | test('write to default location if no outPut file provided', async () => { 387 | const defaultString = `Total Tests in Suite: 0, 388 | Total Tests Completed: 0, 389 | Tests Passed: 0, 390 | Tests Failed: 0, 391 | Flaky Tests: 0, 392 | Test Skipped: 0, 393 | Test run was failure free? true, 394 | Duration of CPU usage in ms: 0, 395 | Duration of entire test run in ms: 0, 396 | Average Test Duration in ms: 0, 397 | Test Suite Duration: 00:00 (mm:ss), 398 | Average Test Duration: 00:00 (mm:ss), 399 | Number of workers used for test run: 1`; 400 | 401 | const mockConfig: MockConfig = { 402 | workers: 1, 403 | }; 404 | const mockSuite: MockSuite = { 405 | allTests: () => [], 406 | }; 407 | 408 | const playwrightReportSummary = new PlaywrightReportSummary(); 409 | 410 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 411 | await playwrightReportSummary.onEnd(); 412 | 413 | const result = readFileSync('results.txt', 'utf8'); 414 | await expect(result).toEqual(defaultString); 415 | }); 416 | 417 | test('write to specified location if outPut filepath provided', async () => { 418 | const defaultString = `Total Tests in Suite: 0, 419 | Total Tests Completed: 0, 420 | Tests Passed: 0, 421 | Tests Failed: 0, 422 | Flaky Tests: 0, 423 | Test Skipped: 0, 424 | Test run was failure free? true, 425 | Duration of CPU usage in ms: 0, 426 | Duration of entire test run in ms: 0, 427 | Average Test Duration in ms: 0, 428 | Test Suite Duration: 00:00 (mm:ss), 429 | Average Test Duration: 00:00 (mm:ss), 430 | Number of workers used for test run: 1`; 431 | 432 | const mockConfig: MockConfig = { 433 | workers: 1, 434 | }; 435 | const mockSuite: MockSuite = { 436 | allTests: () => [], 437 | }; 438 | 439 | const playwrightReportSummary = new PlaywrightReportSummary({ 440 | outputFile: 'subdirectory/results.txt', 441 | }); 442 | 443 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 444 | await playwrightReportSummary.onEnd(); 445 | 446 | const result = readFileSync('subdirectory/results.txt', 'utf8'); 447 | await expect(result).toEqual(defaultString); 448 | }); 449 | 450 | test('write output of custom inputTemplate if provided', async () => { 451 | const testInputTemplate = () => 'my custom template'; 452 | 453 | const mockConfig: MockConfig = { 454 | workers: 1, 455 | }; 456 | const mockSuite: MockSuite = { 457 | allTests: () => [], 458 | }; 459 | 460 | const playwrightReportSummary = new PlaywrightReportSummary({ 461 | inputTemplate: testInputTemplate, 462 | }); 463 | 464 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 465 | await playwrightReportSummary.onEnd(); 466 | 467 | const result = readFileSync('results.txt', 'utf8'); 468 | await expect(result).toEqual('my custom template'); 469 | }); 470 | 471 | test('throw error if output of custom inputTemplate is not string', async () => { 472 | const testInputTemplate = () => true; 473 | 474 | const mockConfig: MockConfig = { 475 | workers: 1, 476 | }; 477 | const mockSuite: MockSuite = { 478 | allTests: () => [], 479 | }; 480 | 481 | const playwrightReportSummary = new PlaywrightReportSummary({ 482 | // ignoring the error to test scenario if someone ignores type 483 | // @ts-ignore 484 | inputTemplate: testInputTemplate, 485 | }); 486 | 487 | await playwrightReportSummary.onBegin(mockConfig, mockSuite); 488 | 489 | await expect(async () => { 490 | await playwrightReportSummary.onEnd(); 491 | }).rejects.toThrow(); 492 | }); 493 | }); 494 | -------------------------------------------------------------------------------- /tests/utils.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from '@playwright/test'; 2 | import millisToMinuteSeconds from '../src/utils'; 3 | 4 | test.describe('(millisToMinutesSeconds', () => { 5 | test('handles 0 millisceonds', () => { 6 | const result = millisToMinuteSeconds(0); 7 | expect(result).toEqual('00:00 (mm:ss)'); 8 | }); 9 | 10 | test('handles less than 1 second', () => { 11 | const result = millisToMinuteSeconds(100); 12 | expect(result).toEqual('00:01 (mm:ss)'); 13 | }); 14 | test('handles less than 10 seconds', () => { 15 | const result = millisToMinuteSeconds(9 * 1000); 16 | expect(result).toEqual('00:09 (mm:ss)'); 17 | }); 18 | test('handles 10 seconds', () => { 19 | const result = millisToMinuteSeconds(10 * 1000); 20 | expect(result).toEqual('00:10 (mm:ss)'); 21 | }); 22 | 23 | test('handles over 10 seconds', () => { 24 | const result = millisToMinuteSeconds(21 * 1000); 25 | expect(result).toEqual('00:21 (mm:ss)'); 26 | }); 27 | 28 | test('handles 60 seconds', () => { 29 | const result = millisToMinuteSeconds(60 * 1000); 30 | expect(result).toEqual('01:00 (mm:ss)'); 31 | }); 32 | 33 | test('handles 90 seconds', () => { 34 | const result = millisToMinuteSeconds(90 * 1000); 35 | expect(result).toEqual('01:30 (mm:ss)'); 36 | }); 37 | 38 | test('handles 120 seconds', () => { 39 | const result = millisToMinuteSeconds(120 * 1000); 40 | expect(result).toEqual('02:00 (mm:ss)'); 41 | }); 42 | 43 | test('handles 10 minutes', () => { 44 | const result = millisToMinuteSeconds(10 * 60 * 1000); 45 | expect(result).toEqual('10:00 (mm:ss)'); 46 | }); 47 | test('handles 100 minutes', () => { 48 | const result = millisToMinuteSeconds(100 * 60 * 1000); 49 | expect(result).toEqual('100:00 (mm:ss)'); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "node16", 5 | "moduleResolution": "node", 6 | "sourceMap": false, 7 | "outDir": "./dist", 8 | "declaration": true, 9 | "allowSyntheticDefaultImports": true 10 | }, 11 | "include": [ 12 | "./src" 13 | ] 14 | } --------------------------------------------------------------------------------