├── .eslintrc.js ├── .github └── workflows │ └── main.yaml ├── .gitignore ├── .npmignore ├── .prettierrc ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── jest.config.js ├── package-lock.json ├── package.json ├── src ├── generate-report.ts └── index.ts ├── tsconfig.json └── types └── ctrf.d.ts /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | es2021: true, 5 | }, 6 | extends: ['standard-with-typescript', 'plugin:prettier/recommended'], 7 | parserOptions: { 8 | ecmaVersion: 'latest', 9 | sourceType: 'module', 10 | }, 11 | rules: { 12 | 'prettier/prettier': 'error', 13 | }, 14 | plugins: ['prettier'], 15 | ignorePatterns: ['dist/', 'tests/'], 16 | } 17 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Use Node.js 18.x 16 | uses: actions/setup-node@v3 17 | with: 18 | node-version: '18.x' 19 | - run: npm ci 20 | - run: npm run build 21 | - run: npm run lint-check 22 | - run: npm run format-check 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | src/ 2 | tests/ 3 | .git/ 4 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "trailingComma": "es5" 5 | } 6 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | First off, thank you for considering contributing to this project. 4 | 5 | ## Reporting Issues 6 | 7 | Before submitting an issue, please check the issue tracker to ensure that the issue hasn't already been reported. If you find your issue already reported, you can subscribe to that issue to receive updates. If you have any additional information to add, please comment on the issue. If the issue is unnasigned and you'd like to contribute, assign the issue to yourself. 8 | 9 | ## How to Contribute 10 | 11 | If you'd like to contribute, start by searching through the issues and pull requests to see whether someone else has raised a similar idea or question. 12 | 13 | If you don't see your idea listed, and you think it fits into the goals of this project, do one of the following: 14 | 15 | - If your contribution is minor, such as a typo fix, open a pull request. 16 | - If your contribution is major, or you have not yet decided how to implement your idea, open an issue to discuss it. This allows other contributors to point out any potential flaws or to help you flesh out your idea. 17 | 18 | ### Pull Requests 19 | 20 | 1. Fork the repository and create your branch from `main`. 21 | 2. Write some code 22 | 3. Make sure your code lints. 23 | 4. Issue that pull request! 24 | 25 | ### Commit Messages 26 | 27 | Write meaningful commit messages that provide insight into the changes made. 28 | 29 | ## Finding Bugs 30 | 31 | If you find a bug, please report it in the issue tracker with a detailed description. 32 | 33 | ## Feature Requests 34 | 35 | Feature requests are welcome. But take a moment to find out whether your idea fits with the scope and aims of the project. Please provide as much detail and context as possible. 36 | 37 | ## License 38 | 39 | By contributing to this project, you agree that your contributions will be licensed under MIT. 40 | 41 | ## Acknowledgments 42 | 43 | Your contributions are sincerely appreciated. We want to make contributing to this project as easy and transparent as possible, whether it's: 44 | 45 | - Reporting a bug 46 | - Discussing the current state of the code 47 | - Submitting a fix 48 | - Proposing new features 49 | 50 | Thank you for your interest in contributing 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Matthew Thomas 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Playwright JSON Test Results Report 2 | 3 | > Save Playwright test results as a JSON file 4 | 5 | A Playwright JSON test reporter to create test reports that follow the CTRF standard. 6 | 7 | [Common Test Report Format](https://ctrf.io) ensures the generation of uniform JSON test reports, independent of programming languages or test framework in use. 8 | 9 |
10 |
11 | 💚 12 |

CTRF tooling is open source and free to use

13 |

You can support the project with a follow and a star

14 | 15 |
16 | 17 | GitHub stars 18 | 19 | 20 | GitHub followers 21 | 22 |
23 |
24 | 25 |

26 | Maintained by Matthew Thomas
27 | Contributions are very welcome!
28 | Explore more integrations 29 |

30 |
31 | 32 | ## Features 33 | 34 | ![Static Badge](https://img.shields.io/badge/official-red?label=ctrf&labelColor=green) 35 | [![build](https://github.com/ctrf-io/playwright-ctrf-json-report/actions/workflows/main.yaml/badge.svg)](https://github.com/ctrf-io/playwright-ctrf-json-report/actions/workflows/main.yaml) 36 | ![NPM Downloads](https://img.shields.io/npm/d18m/playwright-ctrf-json-reporter?logo=npm) 37 | ![npm bundle size](https://img.shields.io/bundlephobia/minzip/playwright-ctrf-json-reporter?label=Size) 38 | ![GitHub Repo stars](https://img.shields.io/github/stars/ctrf-io/playwright-ctrf-json-report) 39 | 40 | - Generate JSON test reports that are [CTRF](https://ctrf.io) compliant 41 | - Customizable output options, minimal or comprehensive reports 42 | - Straightforward integration with Playwright 43 | - Enhanced test insights with detailed test information, environment details, and more. 44 | 45 | ```json 46 | { 47 | "results": { 48 | "tool": { 49 | "name": "playwright" 50 | }, 51 | "summary": { 52 | "tests": 1, 53 | "passed": 1, 54 | "failed": 0, 55 | "pending": 0, 56 | "skipped": 0, 57 | "other": 0, 58 | "start": 1706828654274, 59 | "stop": 1706828655782 60 | }, 61 | "tests": [ 62 | { 63 | "name": "ctrf should generate the same report with any tool", 64 | "status": "passed", 65 | "duration": 100 66 | } 67 | ], 68 | "environment": { 69 | "appName": "MyApp", 70 | "buildName": "MyBuild", 71 | "buildNumber": "1" 72 | } 73 | } 74 | } 75 | ``` 76 | 77 | ## Installation 78 | 79 | ```bash 80 | npm install --save-dev playwright-ctrf-json-reporter 81 | ``` 82 | 83 | Add the reporter to your playwright.config.ts file: 84 | 85 | ```javascript 86 | reporter: [ 87 | ['list'], // You can combine multiple reporters 88 | ['playwright-ctrf-json-reporter', {}] 89 | ], 90 | ``` 91 | 92 | Run your tests: 93 | 94 | ```bash 95 | npx playwright test 96 | ``` 97 | 98 | You'll find a JSON file named `ctrf-report.json` in the `ctrf` directory. 99 | 100 | ## Reporter Options 101 | 102 | The reporter supports several configuration options: 103 | 104 | ```javascript 105 | reporter: [ 106 | ['playwright-ctrf-json-reporter', { 107 | outputFile: 'custom-name.json', // Optional: Output file name. Defaults to 'ctrf-report.json'. 108 | outputDir: 'custom-directory', // Optional: Output directory path. Defaults to '.' (project root). 109 | minimal: true, // Optional: Generate a minimal report. Defaults to 'false'. Overrides screenshot and testType when set to true 110 | screenshot: false, // Optional: Include screenshots in the report. Defaults to 'false'. 111 | annotations: false, // Optional: Include annotations in the report. Defaults to 'false'. 112 | testType: 'e2e', // Optional: Specify the test type (e.g., 'api', 'e2e'). Defaults to 'e2e'. 113 | appName: 'MyApp', // Optional: Specify the name of the application under test. 114 | appVersion: '1.0.0', // Optional: Specify the version of the application under test. 115 | osPlatform: 'linux', // Optional: Specify the OS platform. 116 | osRelease: '18.04', // Optional: Specify the OS release version. 117 | osVersion: '5.4.0', // Optional: Specify the OS version. 118 | buildName: 'MyApp Build', // Optional: Specify the build name. 119 | buildNumber: '100', // Optional: Specify the build number. 120 | buildUrl: "https://ctrf.io", // Optional: Specify the build url. 121 | repositoryName: "ctrf-json", // Optional: Specify the repository name. 122 | repositoryUrl: "https://gh.io", // Optional: Specify the repository url. 123 | branchName: "main", // Optional: Specify the branch name. 124 | testEnvironment: "staging" // Optional: Specify the test environment (e.g. staging, production). 125 | }] 126 | ], 127 | ``` 128 | 129 | A comprehensive report is generated by default, with the exception of screenshots, which you must explicitly set to true. 130 | 131 | To disable stdio use Playwright built in `quiet` configuration option. 132 | 133 | ## Merge reports 134 | 135 | When running tests in parallel, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them. 136 | 137 | The [ctrf-cli](https://github.com/ctrf-io/ctrf-cli) package provides a method to merge multiple ctrf json files into a single file. 138 | 139 | After executing your tests, use the following command: 140 | 141 | ```sh 142 | npx ctrf merge 143 | ``` 144 | 145 | Replace directory with the path to the directory containing the CTRF reports you want to merge. 146 | 147 | ## Test Object Properties 148 | 149 | The test object in the report includes the following [CTRF properties](https://ctrf.io/docs/schema/test): 150 | 151 | | Name | Type | Required | Details | 152 | | ------------- | ---------------- | -------- | ----------------------------------------------------------------------------------- | 153 | | `name` | String | Required | The name of the test. | 154 | | `status` | String | Required | The outcome of the test. One of: `passed`, `failed`, `skipped`, `pending`, `other`. | 155 | | `duration` | Number | Required | The time taken for the test execution, in milliseconds. | 156 | | `start` | Number | Optional | The start time of the test as a Unix epoch timestamp. | 157 | | `stop` | Number | Optional | The end time of the test as a Unix epoch timestamp. | 158 | | `suite` | String | Optional | The suite or group to which the test belongs. | 159 | | `message` | String | Optional | The failure message if the test failed. | 160 | | `trace` | String | Optional | The stack trace captured if the test failed. | 161 | | `rawStatus` | String | Optional | The original playwright status of the test before mapping to CTRF status. | 162 | | `tags` | Array of Strings | Optional | The tags retrieved from the test name | 163 | | `type` | String | Optional | The type of test (e.g., `api`, `e2e`). | 164 | | `filepath` | String | Optional | The file path where the test is located in the project. | 165 | | `retries` | Number | Optional | The number of retries attempted for the test. | 166 | | `flaky` | Boolean | Optional | Indicates whether the test result is flaky. | 167 | | `browser` | String | Optional | The browser used for the test. | 168 | | `attachments` | Array of Objects | Optional | The attachments attached to the test. | 169 | | `stdout` | Array of Strings | Optional | The standard output of the test. | 170 | | `stderr` | Array of Strings | Optional | The standard error of the test. | 171 | | `screenshot` | String | Optional | A base64 encoded screenshot taken during the test. | 172 | | `screenshot` | String | Optional | A base64 encoded screenshot taken during the test. | 173 | | `steps` | Array of Objects | Optional | Individual steps in the test, especially for BDD-style testing. | 174 | 175 | ## Advanced usage 176 | 177 | Some features require additional setup or usage considerations. 178 | 179 | ### Annotations 180 | 181 | By setting `annotations: true` you can include annotations in the test extra property. 182 | 183 | ### Screenshots 184 | 185 | You can include base-64 screenshots in your test report, you'll need to capture and attach screenshots in your Playwright tests: 186 | 187 | ```javascript 188 | import { test, expect } from '@playwright/test' 189 | 190 | test('basic test', async ({ page }, testInfo) => { 191 | await page.goto('https://playwright.dev') 192 | const screenshot = await page.screenshot({ quality: 50, type: 'jpeg' }) 193 | await testInfo.attach('screenshot', { 194 | body: screenshot, 195 | contentType: 'image/jpeg', 196 | }) 197 | }) 198 | ``` 199 | 200 | #### Supported Formats 201 | 202 | Both JPEG and PNG formats are supported, only the last screenshot attached from each test will be included in the report. 203 | 204 | #### Size Considerations 205 | 206 | Base64-encoded image data can greatly increase the size of your report, it's recommended to use screenshots with a lower quality setting (less than 50%) to reduce file size, particularly if you are generating JPEG images. 207 | 208 | ### Browser 209 | 210 | You can include browser information in your test report. You will need to extend Playwright's test object to capture and attach browser metadata for each test: 211 | 212 | ```javascript 213 | // tests/helpers.ts 214 | import { test as _test, expect } from '@playwright/test'; 215 | import os from 'os'; 216 | 217 | export const test = _test.extend<{ _autoAttachMetadata: void }>({ 218 | _autoAttachMetadata: [async ({ browser, browserName }, use, testInfo) => { 219 | // BEFORE: Generate an attachment for the test with the required info 220 | await testInfo.attach('metadata.json', { 221 | body: JSON.stringify({ 222 | name: browserName, 223 | version: browser.version(), 224 | }) 225 | }) 226 | 227 | // --------------------------------------------------------- 228 | await use(/** our test doesn't need this fixture direcly */); 229 | // --------------------------------------------------------- 230 | 231 | // AFTER: There's nothing to cleanup in this fixutre 232 | }, { auto: true }], 233 | }) 234 | 235 | export { expect }; 236 | ``` 237 | 238 | Replace the standard Playwright test import with the custom test fixture in your test files: 239 | 240 | ```javascript 241 | // tests/my-test.spec.ts 242 | import { test, expect } from './helpers' // Adjust the path as necessary 243 | 244 | test('example test', async ({ page }) => { 245 | // ... your test logic ... 246 | }) 247 | ``` 248 | 249 | The browser metadata file must be called metadata.json and contain properties name and version in the body. 250 | 251 | ## What is CTRF? 252 | 253 | CTRF is a universal JSON test report schema that addresses the lack of a standardized format for JSON test reports. 254 | 255 | **Consistency Across Tools:** Different testing tools and frameworks often produce reports in varied formats. CTRF ensures a uniform structure, making it easier to understand and compare reports, regardless of the testing tool used. 256 | 257 | **Language and Framework Agnostic:** It provides a universal reporting schema that works seamlessly with any programming language and testing framework. 258 | 259 | **Facilitates Better Analysis:** With a standardized format, programatically analyzing test outcomes across multiple platforms becomes more straightforward. 260 | 261 | ## Support Us 262 | 263 | If you find this project useful, consider giving it a GitHub star ⭐ It means a lot to us. 264 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | module.exports = { 3 | preset: 'ts-jest', 4 | testEnvironment: 'node', 5 | reporters: ['default'], 6 | } 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "playwright-ctrf-json-reporter", 3 | "version": "0.0.20", 4 | "description": "A Playwright JSON test reporter to create test results reports", 5 | "main": "dist/index.js", 6 | "scripts": { 7 | "build": "tsc", 8 | "test": "jest", 9 | "lint": "eslint . --ext .ts --fix", 10 | "lint-check": "eslint . --ext .ts", 11 | "format": "prettier --write .", 12 | "format-check": "prettier --check ." 13 | }, 14 | "repository": "github:ctrf-io/playwright-ctrf-json-report", 15 | "homepage": "https://ctrf.io", 16 | "files": [ 17 | "dist/", 18 | "README.md" 19 | ], 20 | "author": "Matthew Thomas", 21 | "license": "MIT", 22 | "devDependencies": { 23 | "@playwright/test": "^1.39.0", 24 | "@types/jest": "^29.5.6", 25 | "@types/node": "^20.8.7", 26 | "@typescript-eslint/eslint-plugin": "^6.12.0", 27 | "eslint": "^8.54.0", 28 | "eslint-config-prettier": "^9.0.0", 29 | "eslint-config-standard-with-typescript": "^40.0.0", 30 | "eslint-plugin-import": "^2.29.0", 31 | "eslint-plugin-n": "^16.3.1", 32 | "eslint-plugin-prettier": "^5.0.1", 33 | "eslint-plugin-promise": "^6.1.1", 34 | "jest": "^29.7.0", 35 | "prettier": "^3.1.0", 36 | "ts-jest": "^29.1.1", 37 | "typescript": "^5.3.2" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/generate-report.ts: -------------------------------------------------------------------------------- 1 | import path from 'path' 2 | import fs from 'fs' 3 | 4 | import { 5 | type Suite, 6 | type Reporter, 7 | type TestCase, 8 | type TestResult, 9 | type FullConfig, 10 | type TestStep, 11 | } from '@playwright/test/reporter' 12 | 13 | import { 14 | type CtrfTestState, 15 | type CtrfReport, 16 | type CtrfTest, 17 | type CtrfEnvironment, 18 | type CtrfAttachment, 19 | } from '../types/ctrf' 20 | 21 | interface ReporterConfigOptions { 22 | outputFile?: string 23 | outputDir?: string 24 | minimal?: boolean 25 | screenshot?: boolean 26 | annotations?: boolean 27 | testType?: string 28 | appName?: string | undefined 29 | appVersion?: string | undefined 30 | osPlatform?: string | undefined 31 | osRelease?: string | undefined 32 | osVersion?: string | undefined 33 | buildName?: string | undefined 34 | buildNumber?: string | undefined 35 | buildUrl?: string | undefined 36 | repositoryName?: string | undefined 37 | repositoryUrl?: string | undefined 38 | branchName?: string | undefined 39 | testEnvironment?: string | undefined 40 | } 41 | 42 | class GenerateCtrfReport implements Reporter { 43 | readonly ctrfReport: CtrfReport 44 | readonly ctrfEnvironment: CtrfEnvironment 45 | readonly reporterConfigOptions: ReporterConfigOptions 46 | readonly reporterName = 'playwright-ctrf-json-reporter' 47 | readonly defaultOutputFile = 'ctrf-report.json' 48 | readonly defaultOutputDir = 'ctrf' 49 | private suite: Suite | undefined 50 | private startTime: number | undefined 51 | 52 | constructor(config?: Partial) { 53 | this.reporterConfigOptions = { 54 | outputFile: config?.outputFile ?? this.defaultOutputFile, 55 | outputDir: config?.outputDir ?? this.defaultOutputDir, 56 | minimal: config?.minimal ?? false, 57 | screenshot: config?.screenshot ?? false, 58 | annotations: config?.annotations ?? false, 59 | testType: config?.testType ?? 'e2e', 60 | appName: config?.appName ?? undefined, 61 | appVersion: config?.appVersion ?? undefined, 62 | osPlatform: config?.osPlatform ?? undefined, 63 | osRelease: config?.osRelease ?? undefined, 64 | osVersion: config?.osVersion ?? undefined, 65 | buildName: config?.buildName ?? undefined, 66 | buildNumber: config?.buildNumber ?? undefined, 67 | buildUrl: config?.buildUrl ?? undefined, 68 | repositoryName: config?.repositoryName ?? undefined, 69 | repositoryUrl: config?.repositoryUrl ?? undefined, 70 | branchName: config?.branchName ?? undefined, 71 | testEnvironment: config?.testEnvironment ?? undefined, 72 | } 73 | 74 | this.ctrfReport = { 75 | results: { 76 | tool: { 77 | name: 'playwright', 78 | }, 79 | summary: { 80 | tests: 0, 81 | passed: 0, 82 | failed: 0, 83 | pending: 0, 84 | skipped: 0, 85 | other: 0, 86 | start: 0, 87 | stop: 0, 88 | }, 89 | tests: [], 90 | }, 91 | } 92 | 93 | this.ctrfEnvironment = {} 94 | } 95 | 96 | onBegin(_config: FullConfig, suite: Suite): void { 97 | this.suite = suite 98 | this.startTime = Date.now() 99 | this.ctrfReport.results.summary.start = this.startTime 100 | 101 | if ( 102 | !fs.existsSync( 103 | this.reporterConfigOptions.outputDir ?? this.defaultOutputDir 104 | ) 105 | ) { 106 | fs.mkdirSync( 107 | this.reporterConfigOptions.outputDir ?? this.defaultOutputDir, 108 | { recursive: true } 109 | ) 110 | } 111 | 112 | this.setEnvironmentDetails(this.reporterConfigOptions) 113 | 114 | if (this.hasEnvironmentDetails(this.ctrfEnvironment)) { 115 | this.ctrfReport.results.environment = this.ctrfEnvironment 116 | } 117 | 118 | this.setFilename( 119 | this.reporterConfigOptions.outputFile ?? this.defaultOutputFile 120 | ) 121 | } 122 | 123 | onEnd(): void { 124 | this.ctrfReport.results.summary.stop = Date.now() 125 | 126 | if (this.suite !== undefined) { 127 | if (this.suite.allTests().length > 0) { 128 | this.processSuite(this.suite) 129 | 130 | this.ctrfReport.results.summary.suites = this.countSuites(this.suite) 131 | } 132 | } 133 | this.writeReportToFile(this.ctrfReport) 134 | } 135 | 136 | printsToStdio(): boolean { 137 | return false 138 | } 139 | 140 | processSuite(suite: Suite): void { 141 | for (const test of suite.tests) { 142 | this.processTest(test) 143 | } 144 | 145 | for (const childSuite of suite.suites) { 146 | this.processSuite(childSuite) 147 | } 148 | } 149 | 150 | processTest(testCase: TestCase): void { 151 | if (testCase.results.length === 0) { 152 | return 153 | } 154 | const latestResult = testCase.results[testCase.results.length - 1] 155 | if (latestResult !== undefined) { 156 | this.updateCtrfTestResultsFromTestResult( 157 | testCase, 158 | latestResult, 159 | this.ctrfReport 160 | ) 161 | this.updateSummaryFromTestResult(latestResult, this.ctrfReport) 162 | } 163 | } 164 | 165 | setFilename(filename: string): void { 166 | if (filename.endsWith('.json')) { 167 | this.reporterConfigOptions.outputFile = filename 168 | } else { 169 | this.reporterConfigOptions.outputFile = `${filename}.json` 170 | } 171 | } 172 | 173 | updateCtrfTestResultsFromTestResult( 174 | testCase: TestCase, 175 | testResult: TestResult, 176 | ctrfReport: CtrfReport 177 | ): void { 178 | const test: CtrfTest = { 179 | name: testCase.title, 180 | status: this.mapPlaywrightStatusToCtrf(testResult.status), 181 | duration: testResult.duration, 182 | } 183 | 184 | if (this.reporterConfigOptions.minimal === false) { 185 | test.start = this.updateStart(testResult.startTime) 186 | test.stop = this.calculateStopTime( 187 | testResult.startTime, 188 | testResult.duration 189 | ) 190 | test.message = this.extractFailureDetails(testResult).message 191 | test.trace = this.extractFailureDetails(testResult).trace 192 | test.rawStatus = testResult.status 193 | test.tags = this.extractTagsFromTitle(testCase.title) 194 | test.type = this.reporterConfigOptions.testType ?? 'e2e' 195 | test.filePath = testCase.location.file 196 | test.retries = testResult.retry 197 | test.flaky = testResult.status === 'passed' && testResult.retry > 0 198 | test.steps = [] 199 | if (testResult.steps.length > 0) { 200 | testResult.steps.forEach((step) => { 201 | this.processStep(test, step) 202 | }) 203 | } 204 | if (this.reporterConfigOptions.screenshot === true) { 205 | test.screenshot = this.extractScreenshotBase64(testResult) 206 | } 207 | test.suite = this.buildSuitePath(testCase) 208 | if ( 209 | this.extractMetadata(testResult)?.name !== undefined || 210 | this.extractMetadata(testResult)?.version !== undefined 211 | ) 212 | test.browser = `${this.extractMetadata(testResult) 213 | ?.name} ${this.extractMetadata(testResult)?.version}` 214 | test.attachments = this.filterValidAttachments(testResult.attachments) 215 | test.stdout = testResult.stdout.map((item) => 216 | Buffer.isBuffer(item) ? item.toString() : String(item) 217 | ) 218 | test.stderr = testResult.stderr.map((item) => 219 | Buffer.isBuffer(item) ? item.toString() : String(item) 220 | ) 221 | if (this.reporterConfigOptions.annotations !== undefined) { 222 | test.extra = { annotations: testCase.annotations } 223 | } 224 | } 225 | 226 | ctrfReport.results.tests.push(test) 227 | } 228 | 229 | updateSummaryFromTestResult( 230 | testResult: TestResult, 231 | ctrfReport: CtrfReport 232 | ): void { 233 | ctrfReport.results.summary.tests++ 234 | 235 | const ctrfStatus = this.mapPlaywrightStatusToCtrf(testResult.status) 236 | 237 | if (ctrfStatus in ctrfReport.results.summary) { 238 | ctrfReport.results.summary[ctrfStatus]++ 239 | } else { 240 | ctrfReport.results.summary.other++ 241 | } 242 | } 243 | 244 | mapPlaywrightStatusToCtrf(testStatus: string): CtrfTestState { 245 | switch (testStatus) { 246 | case 'passed': 247 | return 'passed' 248 | case 'failed': 249 | case 'timedOut': 250 | case 'interrupted': 251 | return 'failed' 252 | case 'skipped': 253 | return 'skipped' 254 | case 'pending': 255 | return 'pending' 256 | default: 257 | return 'other' 258 | } 259 | } 260 | 261 | setEnvironmentDetails(reporterConfigOptions: ReporterConfigOptions): void { 262 | if (reporterConfigOptions.appName !== undefined) { 263 | this.ctrfEnvironment.appName = reporterConfigOptions.appName 264 | } 265 | if (reporterConfigOptions.appVersion !== undefined) { 266 | this.ctrfEnvironment.appVersion = reporterConfigOptions.appVersion 267 | } 268 | if (reporterConfigOptions.osPlatform !== undefined) { 269 | this.ctrfEnvironment.osPlatform = reporterConfigOptions.osPlatform 270 | } 271 | if (reporterConfigOptions.osRelease !== undefined) { 272 | this.ctrfEnvironment.osRelease = reporterConfigOptions.osRelease 273 | } 274 | if (reporterConfigOptions.osVersion !== undefined) { 275 | this.ctrfEnvironment.osVersion = reporterConfigOptions.osVersion 276 | } 277 | if (reporterConfigOptions.buildName !== undefined) { 278 | this.ctrfEnvironment.buildName = reporterConfigOptions.buildName 279 | } 280 | if (reporterConfigOptions.buildNumber !== undefined) { 281 | this.ctrfEnvironment.buildNumber = reporterConfigOptions.buildNumber 282 | } 283 | if (reporterConfigOptions.buildUrl !== undefined) { 284 | this.ctrfEnvironment.buildUrl = reporterConfigOptions.buildUrl 285 | } 286 | if (reporterConfigOptions.repositoryName !== undefined) { 287 | this.ctrfEnvironment.repositoryName = reporterConfigOptions.repositoryName 288 | } 289 | if (reporterConfigOptions.repositoryUrl !== undefined) { 290 | this.ctrfEnvironment.repositoryUrl = reporterConfigOptions.repositoryUrl 291 | } 292 | if (reporterConfigOptions.branchName !== undefined) { 293 | this.ctrfEnvironment.branchName = reporterConfigOptions.branchName 294 | } 295 | if (reporterConfigOptions.testEnvironment !== undefined) { 296 | this.ctrfEnvironment.testEnvironment = 297 | reporterConfigOptions.testEnvironment 298 | } 299 | } 300 | 301 | hasEnvironmentDetails(environment: CtrfEnvironment): boolean { 302 | return Object.keys(environment).length > 0 303 | } 304 | 305 | extractMetadata(testResult: TestResult): any { 306 | const metadataAttachment = testResult.attachments.find( 307 | (attachment) => attachment.name === 'metadata.json' 308 | ) 309 | if ( 310 | metadataAttachment?.body !== null && 311 | metadataAttachment?.body !== undefined 312 | ) { 313 | try { 314 | const metadataRaw = metadataAttachment.body.toString('utf-8') 315 | return JSON.parse(metadataRaw) 316 | } catch (e) { 317 | if (e instanceof Error) { 318 | console.error(`Error parsing browser metadata: ${e.message}`) 319 | } else { 320 | console.error('An unknown error occurred in parsing browser metadata') 321 | } 322 | } 323 | } 324 | return null 325 | } 326 | 327 | updateStart(startTime: Date): number { 328 | const date = new Date(startTime) 329 | const unixEpochTime = Math.floor(date.getTime() / 1000) 330 | return unixEpochTime 331 | } 332 | 333 | calculateStopTime(startTime: Date, duration: number): number { 334 | const startDate = new Date(startTime) 335 | const stopDate = new Date(startDate.getTime() + duration) 336 | return Math.floor(stopDate.getTime() / 1000) 337 | } 338 | 339 | buildSuitePath(test: TestCase): string { 340 | const pathComponents = [] 341 | let currentSuite: Suite | undefined = test.parent 342 | 343 | while (currentSuite !== undefined) { 344 | if (currentSuite.title !== '') { 345 | pathComponents.unshift(currentSuite.title) 346 | } 347 | currentSuite = currentSuite.parent 348 | } 349 | 350 | return pathComponents.join(' > ') 351 | } 352 | 353 | extractTagsFromTitle(title: string): string[] { 354 | const tagPattern = /@\w+/g 355 | const tags = title.match(tagPattern) 356 | return tags ?? [] 357 | } 358 | 359 | extractScreenshotBase64(testResult: TestResult): string | undefined { 360 | const screenshotAttachment = testResult.attachments.find( 361 | (attachment) => 362 | attachment.name === 'screenshot' && 363 | (attachment.contentType === 'image/jpeg' || 364 | attachment.contentType === 'image/png') 365 | ) 366 | 367 | return screenshotAttachment?.body?.toString('base64') 368 | } 369 | 370 | extractFailureDetails(testResult: TestResult): Partial { 371 | if ( 372 | (testResult.status === 'failed' || 373 | testResult.status === 'timedOut' || 374 | testResult.status === 'interrupted') && 375 | testResult.error !== undefined 376 | ) { 377 | const failureDetails: Partial = {} 378 | if (testResult.error.message !== undefined) { 379 | failureDetails.message = testResult.error.message 380 | } 381 | if (testResult.error.stack !== undefined) { 382 | failureDetails.trace = testResult.error.stack 383 | } 384 | return failureDetails 385 | } 386 | return {} 387 | } 388 | 389 | countSuites(suite: Suite): number { 390 | let count = 0 391 | 392 | suite.suites.forEach((childSuite) => { 393 | count += this.countSuites(childSuite) 394 | }) 395 | 396 | return count 397 | } 398 | 399 | writeReportToFile(data: CtrfReport): void { 400 | const filePath = path.join( 401 | this.reporterConfigOptions.outputDir ?? this.defaultOutputDir, 402 | this.reporterConfigOptions.outputFile ?? this.defaultOutputFile 403 | ) 404 | const str = JSON.stringify(data, null, 2) 405 | try { 406 | fs.writeFileSync(filePath, str + '\n') 407 | console.log( 408 | `${this.reporterName}: successfully written ctrf json to %s/%s`, 409 | this.reporterConfigOptions.outputDir, 410 | this.reporterConfigOptions.outputFile 411 | ) 412 | } catch (error) { 413 | console.error(`Error writing ctrf json report:, ${String(error)}`) 414 | } 415 | } 416 | 417 | processStep(test: CtrfTest, step: TestStep): void { 418 | if (step.category === 'test.step') { 419 | const stepStatus = 420 | step.error === undefined 421 | ? this.mapPlaywrightStatusToCtrf('passed') 422 | : this.mapPlaywrightStatusToCtrf('failed') 423 | const currentStep = { 424 | name: step.title, 425 | status: stepStatus, 426 | } 427 | test.steps?.push(currentStep) 428 | } 429 | 430 | const childSteps = step.steps 431 | 432 | if (childSteps.length > 0) { 433 | childSteps.forEach((cStep) => { 434 | this.processStep(test, cStep) 435 | }) 436 | } 437 | } 438 | 439 | filterValidAttachments( 440 | attachments: TestResult['attachments'] 441 | ): CtrfAttachment[] { 442 | return attachments 443 | .filter((attachment) => attachment.path !== undefined) 444 | .map((attachment) => ({ 445 | name: attachment.name, 446 | contentType: attachment.contentType, 447 | path: attachment.path ?? '', 448 | })) 449 | } 450 | } 451 | 452 | export default GenerateCtrfReport 453 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { default } from './generate-report' 2 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016", 4 | "module": "commonjs", 5 | "strict": true, 6 | "esModuleInterop": true, 7 | "outDir": "./dist", 8 | "rootDir": "./src", 9 | "declaration": true, 10 | "forceConsistentCasingInFileNames": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /types/ctrf.d.ts: -------------------------------------------------------------------------------- 1 | export interface CtrfReport { 2 | results: Results 3 | } 4 | 5 | export interface Results { 6 | tool: Tool 7 | summary: Summary 8 | tests: CtrfTest[] 9 | environment?: CtrfEnvironment 10 | extra?: Record 11 | } 12 | 13 | export interface Summary { 14 | tests: number 15 | passed: number 16 | failed: number 17 | skipped: number 18 | pending: number 19 | other: number 20 | suites?: number 21 | start: number 22 | stop: number 23 | extra?: Record 24 | } 25 | 26 | export interface CtrfTest { 27 | name: string 28 | status: CtrfTestState 29 | duration: number 30 | start?: number 31 | stop?: number 32 | suite?: string 33 | message?: string 34 | trace?: string 35 | rawStatus?: string 36 | tags?: string[] 37 | type?: string 38 | filePath?: string 39 | retries?: number 40 | flaky?: boolean 41 | attachments?: CtrfAttachment[] 42 | stdout?: string[] 43 | stderr?: string[] 44 | browser?: string 45 | device?: string 46 | screenshot?: string 47 | parameters?: Record 48 | steps?: Step[] 49 | extra?: Record 50 | } 51 | 52 | export interface CtrfEnvironment { 53 | appName?: string 54 | appVersion?: string 55 | osPlatform?: string 56 | osRelease?: string 57 | osVersion?: string 58 | buildName?: string 59 | buildNumber?: string 60 | buildUrl?: string 61 | repositoryName?: string 62 | repositoryUrl?: string 63 | branchName?: string 64 | testEnvironment?: string 65 | extra?: Record 66 | } 67 | 68 | export interface Tool { 69 | name: string 70 | version?: string 71 | extra?: Record 72 | } 73 | 74 | export interface Step { 75 | name: string 76 | status: CtrfTestState 77 | } 78 | 79 | export interface CtrfAttachment { 80 | name: string 81 | contentType: string 82 | path: string 83 | } 84 | 85 | export type CtrfTestState = 86 | | 'passed' 87 | | 'failed' 88 | | 'skipped' 89 | | 'pending' 90 | | 'other' 91 | --------------------------------------------------------------------------------