├── .gitattributes ├── .eslintrc.json ├── img ├── situation-now.png └── situation-expected.png ├── .gitignore ├── .mailmap ├── SECURITY.md ├── authors.js ├── test ├── fixtures │ ├── integration │ │ ├── qunit-todo.js │ │ ├── mocha.js │ │ ├── jasmine.js │ │ └── qunit.js │ └── unit.js ├── versions │ ├── versions.js │ ├── failing-versions.js │ └── versions-reporting.js ├── unit │ ├── console-reporter.js │ ├── helpers.js │ ├── summary-reporter.js │ └── tap-reporter.js └── integration │ ├── adapters-run.js │ ├── reference-data-todo.js │ ├── adapters.js │ └── reference-data.js ├── .github └── workflows │ ├── browsers.yaml │ └── CI.yaml ├── RELEASE.md ├── index.js ├── lib ├── auto.js ├── helpers.js ├── reporters │ ├── ConsoleReporter.js │ ├── SummaryReporter.js │ └── TapReporter.js └── adapters │ ├── JasmineAdapter.js │ ├── MochaAdapter.js │ └── QUnitAdapter.js ├── LICENSE ├── rollup.config.js ├── karma.conf.js ├── package.json ├── karma.conf.sauce.js ├── CHANGELOG.md ├── docs ├── example.md └── frameworks.md ├── README.md └── spec └── cri-draft.adoc /.gitattributes: -------------------------------------------------------------------------------- 1 | package-lock.json binary 2 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "extends": "semistandard" 4 | } 5 | -------------------------------------------------------------------------------- /img/situation-now.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qunitjs/js-reporters/HEAD/img/situation-now.png -------------------------------------------------------------------------------- /img/situation-expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qunitjs/js-reporters/HEAD/img/situation-expected.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.nyc_output 2 | /coverage 3 | /dist 4 | /node_modules 5 | /log 6 | /*.log 7 | /package-lock.json 8 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Florentin Simion 2 | Jiahao Guo 3 | Trent Willis 4 | Trent Willis 5 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security policy 2 | 3 | ## Supported versions 4 | 5 | The latest release is supported with security updates. 6 | 7 | ## Reporting a vulnerability 8 | 9 | If you discover a vulnerability, we would like to know about it so we can take steps to address it as quickly as possible. 10 | 11 | E-mail your findings to security@jquery.com. Thanks! 12 | -------------------------------------------------------------------------------- /authors.js: -------------------------------------------------------------------------------- 1 | const cp = require('child_process'); 2 | 3 | function getAuthors () { 4 | const orderedAuthors = cp.execFileSync( 5 | 'git', 6 | ['log', '--format=%aN', '--reverse'], 7 | { encoding: 'utf8' } 8 | ); 9 | const unique = orderedAuthors 10 | .trim() 11 | .split('\n') 12 | .filter((author, i, arr) => arr.indexOf(author) === i); 13 | 14 | unique[unique.length - 1] = 'and ' + unique[unique.length - 1]; 15 | 16 | return unique.join(', ') + '.'; 17 | } 18 | 19 | process.stdout.write(getAuthors() + '\n'); 20 | -------------------------------------------------------------------------------- /test/fixtures/integration/qunit-todo.js: -------------------------------------------------------------------------------- 1 | /* global QUnit */ 2 | 3 | QUnit.test('global pass', function (assert) { 4 | assert.ok(true); 5 | }); 6 | 7 | QUnit.todo('global todo', function (assert) { 8 | assert.ok(false); 9 | }); 10 | 11 | QUnit.module('suite with a todo test'); 12 | QUnit.test('should pass', function (assert) { 13 | assert.ok(true); 14 | }); 15 | QUnit.todo('should todo', function (assert) { 16 | assert.ok(false); 17 | }); 18 | 19 | QUnit.module.todo('todo suite'); 20 | QUnit.test('should todo', function (assert) { 21 | assert.ok(false); 22 | }); 23 | -------------------------------------------------------------------------------- /.github/workflows/browsers.yaml: -------------------------------------------------------------------------------- 1 | name: Browsers 2 | on: 3 | push: 4 | branches: 5 | - main 6 | # Or manually 7 | workflow_dispatch: 8 | 9 | jobs: 10 | 11 | test: 12 | name: SauceLabs 13 | runs-on: ubuntu-latest 14 | if: ${{ github.repository == 'qunitjs/js-reporters' }} # skip on forks, needs secret 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - run: npm install 19 | 20 | - run: npm run lint 21 | 22 | - run: npm run test-browser-sauce 23 | env: 24 | SAUCE_USERNAME: "${{ secrets.SAUCE_USERNAME }}" 25 | SAUCE_REGION: "${{ secrets.SAUCE_REGION }}" 26 | SAUCE_ACCESS_KEY: "${{ secrets.SAUCE_ACCESS_KEY }}" 27 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | 3 | 1. Update `CHANGELOG.md` 4 | * Run the following command, substituting `v1.2.3` for the last release tag. 5 | `git log --format="* %s. (%aN)" --no-merges v1.2.3...HEAD | sort` 6 | * Copy the output to a changelog section at the top of the file. 7 | * Remove entries that don't affect users of the package (e.g. "Build", "Test", or "Spec"). 8 | * Format using [Keep a changelog](https://keepachangelog.com/en/1.0.0/) conventions. 9 | 2. Update `package.json` version, and stage the changes. 10 | 3. Commit with message `Release X.Y.Z`, and create a signed tag `git tag -s "vX.Y.Z" -m "Release X.Y.Z"` 11 | 4. Push the commit and tag to GitHub. 12 | 5. Run `npm run build` (may run in isolation), followed by `npm publish`. 13 | 14 | That's all! 15 | -------------------------------------------------------------------------------- /test/versions/versions.js: -------------------------------------------------------------------------------- 1 | /* eslint-env qunit */ 2 | const { test } = QUnit; 3 | const failingVersionsRef = require('./failing-versions.js'); 4 | const failingVersions = require('./versions-reporting.js'); 5 | 6 | QUnit.module('Versions', function () { 7 | test('qunit versions', assert => { 8 | assert.deepEqual(failingVersions.qunit, failingVersionsRef.qunit); 9 | }); 10 | 11 | test('qunitjs versions', assert => { 12 | assert.deepEqual(failingVersions.qunitjs, failingVersionsRef.qunitjs); 13 | }); 14 | 15 | test('jasmine versions', assert => { 16 | assert.deepEqual(failingVersions.jasmine, failingVersionsRef.jasmine); 17 | }); 18 | 19 | test('mocha versions', assert => { 20 | assert.deepEqual(failingVersions.mocha, failingVersionsRef.mocha); 21 | }); 22 | }); 23 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const QUnitAdapter = require('./lib/adapters/QUnitAdapter.js'); 3 | const JasmineAdapter = require('./lib/adapters/JasmineAdapter.js'); 4 | const MochaAdapter = require('./lib/adapters/MochaAdapter.js'); 5 | const TapReporter = require('./lib/reporters/TapReporter.js'); 6 | const ConsoleReporter = require('./lib/reporters/ConsoleReporter.js'); 7 | const SummaryReporter = require('./lib/reporters/SummaryReporter.js'); 8 | const { 9 | createTestStart 10 | } = require('./lib/helpers.js'); 11 | const { autoRegister } = require('./lib/auto.js'); 12 | 13 | module.exports = { 14 | QUnitAdapter, 15 | JasmineAdapter, 16 | MochaAdapter, 17 | TapReporter, 18 | ConsoleReporter, 19 | SummaryReporter, 20 | EventEmitter, 21 | createTestStart, 22 | autoRegister 23 | }; 24 | -------------------------------------------------------------------------------- /.github/workflows/CI.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | - push 4 | - pull_request 5 | 6 | jobs: 7 | 8 | test: 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | include: 13 | - node: 14.x 14 | - node: 16.x 15 | - node: 18.x 16 | - node: 20.x 17 | 18 | name: Node.js ${{ matrix.node }} 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - name: Install Node.js 24 | uses: actions/setup-node@v4 25 | with: 26 | node-version: ${{ matrix.node }} 27 | 28 | - run: npm install 29 | 30 | - run: npm test 31 | 32 | coverage: 33 | name: Coverage 34 | runs-on: ubuntu-latest 35 | steps: 36 | - uses: actions/checkout@v4 37 | 38 | - run: npm install 39 | 40 | - run: npm run coverage 41 | -------------------------------------------------------------------------------- /lib/auto.js: -------------------------------------------------------------------------------- 1 | /* global QUnit, mocha, jasmine */ 2 | 3 | const QUnitAdapter = require('./adapters/QUnitAdapter.js'); 4 | const MochaAdapter = require('./adapters/MochaAdapter.js'); 5 | const JasmineAdapter = require('./adapters/JasmineAdapter.js'); 6 | 7 | /** 8 | * Auto registers the adapter for the respective testing framework and 9 | * returns the runner for event listening. 10 | */ 11 | function autoRegister () { 12 | let runner; 13 | 14 | if (QUnit) { 15 | runner = new QUnitAdapter(QUnit); 16 | } else if (mocha) { 17 | runner = new MochaAdapter(mocha); 18 | } else if (jasmine) { 19 | runner = new JasmineAdapter(jasmine); 20 | } else { 21 | throw new Error('Failed to register js-reporters adapter. Supported ' + 22 | 'frameworks are: QUnit, Mocha, Jasmine'); 23 | } 24 | 25 | return runner; 26 | } 27 | 28 | module.exports = { 29 | autoRegister 30 | }; 31 | -------------------------------------------------------------------------------- /test/versions/failing-versions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Known issues. 3 | */ 4 | 5 | module.exports = { 6 | qunitjs: [], 7 | qunit: [], 8 | jasmine: [ 9 | '2.0.1', 10 | // jasmine@2.3.0: Jasmine kills the process without an error message. 11 | // Fixed in 2.3.1, . 12 | '2.3.0', 13 | // Jasmine 2.5.0: Same bug as in the 2.3.0 version. 14 | // Fixed in 2.5.1, . 15 | '2.5.0' 16 | ], 17 | mocha: [ 18 | // mocha@2.1.0: mocha.run() throws "fn is not a function" 19 | // Fixed in 2.2.0, . 20 | '2.1.0', 21 | // mocha 2.5.0: Fails due to missing dependency. 22 | // Fixed in 2.5.1, . 23 | '2.5.0' 24 | ] 25 | }; 26 | -------------------------------------------------------------------------------- /lib/helpers.js: -------------------------------------------------------------------------------- 1 | function aggregateTests (all) { 2 | const testCounts = { 3 | passed: all.filter((test) => test.status === 'passed').length, 4 | failed: all.filter((test) => test.status === 'failed').length, 5 | skipped: all.filter((test) => test.status === 'skipped').length, 6 | todo: all.filter((test) => test.status === 'todo').length, 7 | total: all.length 8 | }; 9 | const status = testCounts.failed ? 'failed' : 'passed'; 10 | 11 | let runtime = 0; 12 | all.forEach((test) => { 13 | runtime += test.runtime || 0; 14 | }); 15 | 16 | return { 17 | status, 18 | testCounts, 19 | runtime 20 | }; 21 | } 22 | 23 | function createTestStart (testEnd) { 24 | return { 25 | name: testEnd.name, 26 | suiteName: testEnd.suiteName, 27 | fullName: testEnd.fullName.slice() 28 | }; 29 | } 30 | 31 | module.exports = { 32 | aggregateTests, 33 | createTestStart 34 | }; 35 | -------------------------------------------------------------------------------- /test/fixtures/integration/mocha.js: -------------------------------------------------------------------------------- 1 | /* global describe, it */ 2 | 3 | it('global test', function () { 4 | 5 | }); 6 | 7 | describe('suite with passing test', function () { 8 | it('should pass', function () { 9 | 10 | }); 11 | }); 12 | 13 | describe('suite with skipped test', function () { 14 | it.skip('should skip', function () { 15 | 16 | }); 17 | }); 18 | 19 | describe('suite with failing test', function () { 20 | it('should fail', function () { 21 | throw new Error('error'); 22 | }); 23 | }); 24 | 25 | describe('suite with tests', function () { 26 | it('should pass', function () { 27 | 28 | }); 29 | 30 | it.skip('should skip', function () { 31 | 32 | }); 33 | 34 | it('should fail', function () { 35 | throw new Error('error'); 36 | }); 37 | }); 38 | 39 | describe('outer suite', function () { 40 | describe('inner suite', function () { 41 | it('inner test', function () { 42 | 43 | }); 44 | }); 45 | 46 | it('outer test', function () { 47 | 48 | }); 49 | }); 50 | -------------------------------------------------------------------------------- /lib/reporters/ConsoleReporter.js: -------------------------------------------------------------------------------- 1 | module.exports = class ConsoleReporter { 2 | constructor (runner, options = {}) { 3 | // Cache references to console methods to ensure we can report failures 4 | // from tests tests that mock the console object itself. 5 | // https://github.com/qunitjs/js-reporters/issues/125 6 | this.log = options.log || console.log.bind(console); 7 | 8 | runner.on('runStart', this.onRunStart.bind(this)); 9 | runner.on('testStart', this.onTestStart.bind(this)); 10 | runner.on('testEnd', this.onTestEnd.bind(this)); 11 | runner.on('runEnd', this.onRunEnd.bind(this)); 12 | } 13 | 14 | static init (runner) { 15 | return new ConsoleReporter(runner); 16 | } 17 | 18 | onRunStart (runStart) { 19 | this.log('runStart', runStart); 20 | } 21 | 22 | onTestStart (test) { 23 | this.log('testStart', test); 24 | } 25 | 26 | onTestEnd (test) { 27 | this.log('testEnd', test); 28 | } 29 | 30 | onRunEnd (runEnd) { 31 | this.log('runEnd', runEnd); 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright JS Reporters 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /test/fixtures/integration/jasmine.js: -------------------------------------------------------------------------------- 1 | /* global describe, it, xit, expect */ 2 | 3 | it('global test', function () { 4 | expect(true).toBeTruthy(); 5 | }); 6 | 7 | describe('suite with passing test', function () { 8 | it('should pass', function () { 9 | expect(true).toBeTruthy(); 10 | }); 11 | }); 12 | 13 | describe('suite with skipped test', function () { 14 | xit('should skip', function () { 15 | 16 | }); 17 | }); 18 | 19 | describe('suite with failing test', function () { 20 | it('should fail', function () { 21 | throw new Error('error'); 22 | }); 23 | }); 24 | 25 | describe('suite with tests', function () { 26 | it('should pass', function () { 27 | expect(true).toBeTruthy(); 28 | }); 29 | 30 | xit('should skip', function () { 31 | 32 | }); 33 | 34 | it('should fail', function () { 35 | throw new Error('error'); 36 | }); 37 | }); 38 | 39 | describe('outer suite', function () { 40 | it('outer test', function () { 41 | expect(true).toBeTruthy(); 42 | }); 43 | 44 | describe('inner suite', function () { 45 | it('inner test', function () { 46 | expect(true).toBeTruthy(); 47 | }); 48 | }); 49 | }); 50 | -------------------------------------------------------------------------------- /test/fixtures/integration/qunit.js: -------------------------------------------------------------------------------- 1 | /* global QUnit */ 2 | 3 | // Add dummy assertions in passing tests, as QUnit by defaul requires at least one assertion. 4 | 5 | QUnit.test('global test', function (assert) { 6 | assert.ok(true); 7 | }); 8 | 9 | QUnit.module('suite with passing test'); 10 | QUnit.test('should pass', function (assert) { 11 | assert.ok(true); 12 | }); 13 | 14 | QUnit.module('suite with skipped test'); 15 | QUnit.skip('should skip', function () { 16 | 17 | }); 18 | 19 | QUnit.module('suite with failing test'); 20 | QUnit.test('should fail', function () { 21 | throw new Error('error'); 22 | }); 23 | 24 | QUnit.module('suite with tests'); 25 | QUnit.test('should pass', function (assert) { 26 | assert.ok(true); 27 | }); 28 | QUnit.skip('should skip', function () { 29 | 30 | }); 31 | QUnit.test('should fail', function () { 32 | throw new Error('error'); 33 | }); 34 | 35 | QUnit.module('outer suite', function () { 36 | QUnit.module('inner suite', function () { 37 | QUnit.test('inner test', function (assert) { 38 | assert.ok(true); 39 | }); 40 | }); 41 | 42 | QUnit.test('outer test', function (assert) { 43 | assert.ok(true); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /test/unit/console-reporter.js: -------------------------------------------------------------------------------- 1 | /* eslint-env qunit */ 2 | const { test } = QUnit; 3 | const sinon = require('sinon'); 4 | const JsReporters = require('../../index.js'); 5 | 6 | QUnit.module('ConsoleReporter', hooks => { 7 | let emitter, sandbox, con; 8 | 9 | hooks.beforeEach(function () { 10 | emitter = new JsReporters.EventEmitter(); 11 | sandbox = sinon.sandbox.create(); 12 | con = { 13 | log: sandbox.stub() 14 | }; 15 | // eslint-disable-next-line no-new 16 | new JsReporters.ConsoleReporter(emitter, con); 17 | }); 18 | 19 | hooks.afterEach(function () { 20 | sandbox.restore(); 21 | }); 22 | 23 | test('Event "runStart"', assert => { 24 | emitter.emit('runStart', {}); 25 | assert.equal(con.log.callCount, 1); 26 | }); 27 | 28 | test('Event "runEnd"', assert => { 29 | emitter.emit('runEnd', {}); 30 | assert.equal(con.log.callCount, 1); 31 | }); 32 | 33 | test('Event "testStart"', assert => { 34 | emitter.emit('testStart', {}); 35 | assert.equal(con.log.callCount, 1); 36 | }); 37 | 38 | test('Event "testEnd"', assert => { 39 | emitter.emit('testEnd', {}); 40 | assert.equal(con.log.callCount, 1); 41 | }); 42 | }); 43 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | const { babel } = require('@rollup/plugin-babel'); 2 | const commonjs = require('@rollup/plugin-commonjs'); 3 | const { nodeResolve } = require('@rollup/plugin-node-resolve'); 4 | 5 | const isCoverage = process.env.BUILD_TARGET === 'dev'; 6 | const version = require('./package.json').version; 7 | 8 | module.exports = { 9 | input: 'index.js', 10 | output: { 11 | file: 'dist/js-reporters.js', 12 | sourcemap: isCoverage, 13 | format: 'umd', 14 | name: 'JsReporters', 15 | exports: 'auto', 16 | banner: `/*! JsReporters ${version} | Copyright JS Reporters https://github.com/qunitjs/js-reporters/ | https://opensource.org/licenses/MIT */` 17 | }, 18 | plugins: [ 19 | // For 'events' and 'kleur' 20 | nodeResolve({ 21 | preferBuiltins: false 22 | }), 23 | commonjs({ 24 | // This makes require() work like in Node.js, 25 | // instead of wrapped in a {default:…} object. 26 | requireReturnsDefault: 'preferred' 27 | }), 28 | babel({ 29 | babelHelpers: 'bundled', 30 | babelrc: false, 31 | presets: [ 32 | ['@babel/preset-env', { 33 | targets: { 34 | ie: 9 35 | } 36 | }] 37 | ] 38 | }) 39 | ] 40 | }; 41 | -------------------------------------------------------------------------------- /karma.conf.js: -------------------------------------------------------------------------------- 1 | module.exports = function (config) { 2 | const { babel } = require('@rollup/plugin-babel'); 3 | const commonjs = require('@rollup/plugin-commonjs'); 4 | const { nodeResolve } = require('@rollup/plugin-node-resolve'); 5 | 6 | config.set({ 7 | files: [ 8 | 'node_modules/sinon/pkg/sinon.js', 9 | 'test/unit/tap-reporter.js', 10 | 'test/unit/helpers.js' 11 | ], 12 | preprocessors: { 13 | 'test/unit/**.js': ['rollup'] 14 | }, 15 | rollupPreprocessor: { 16 | external: ['sinon'], 17 | output: { 18 | globals: { 19 | sinon: 'sinon' 20 | }, 21 | format: 'iife', 22 | name: 'JsReporters_Test', 23 | sourcemap: true 24 | }, 25 | // See rollup.config.js 26 | plugins: [ 27 | nodeResolve({ 28 | preferBuiltins: false 29 | }), 30 | commonjs({ 31 | requireReturnsDefault: 'preferred' 32 | }), 33 | babel({ 34 | babelHelpers: 'bundled', 35 | babelrc: false, 36 | presets: [ 37 | ['@babel/preset-env', { 38 | targets: { 39 | ie: 9 40 | } 41 | }] 42 | ] 43 | }) 44 | ] 45 | }, 46 | frameworks: ['qunit'], 47 | browsers: ['FirefoxHeadless'], 48 | singleRun: true, 49 | autoWatch: false 50 | }); 51 | }; 52 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "js-reporters", 3 | "version": "2.1.0", 4 | "description": "Common reporter interface for JavaScript testing frameworks.", 5 | "main": "dist/js-reporters.js", 6 | "files": [ 7 | "dist/" 8 | ], 9 | "license": "MIT", 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/qunitjs/js-reporters.git" 13 | }, 14 | "engines": { 15 | "node": ">=10" 16 | }, 17 | "scripts": { 18 | "build": "rollup -c", 19 | "lint": "semistandard", 20 | "test": "npm run test-unit && npm run test-integration && npm run test-browser && npm run lint", 21 | "test-unit": "qunit 'test/unit/*.js'", 22 | "test-integration": "qunit test/integration/adapters.js", 23 | "test-browser": "karma start", 24 | "test-browser-sauce": "karma start karma.conf.sauce.js", 25 | "test-versions": "qunit test/versions/versions.js", 26 | "coverage": "nyc qunit 'test/unit/*.js' test/integration/adapters.js" 27 | }, 28 | "devDependencies": { 29 | "@babel/core": "7.13.14", 30 | "@babel/preset-env": "7.13.12", 31 | "@rollup/plugin-babel": "5.3.0", 32 | "@rollup/plugin-commonjs": "18.0.0", 33 | "@rollup/plugin-node-resolve": "11.2.1", 34 | "events": "3.3.0", 35 | "jasmine": "3.7.0", 36 | "karma": "6.4.4", 37 | "karma-chrome-launcher": "3.2.0", 38 | "karma-firefox-launcher": "2.1.3", 39 | "karma-qunit": "4.2.1", 40 | "karma-rollup-preprocessor": "7.0.7", 41 | "karma-sauce-launcher": "4.3.6", 42 | "kleur": "4.1.4", 43 | "mocha": "8.3.2", 44 | "nyc": "15.1.0", 45 | "qunit": "2.20.1", 46 | "qunitjs": "1.23.1", 47 | "rimraf": "3.0.2", 48 | "rollup": "2.44.0", 49 | "semistandard": "16.0.0", 50 | "semver": "7.3.5", 51 | "sinon": "1.17.4" 52 | }, 53 | "overrides": { 54 | "@types/puppeteer-core": "7.0.4" 55 | }, 56 | "nyc": { 57 | "include": [ 58 | "lib/**" 59 | ], 60 | "reporter": [ 61 | "text", 62 | "html", 63 | "lcovonly" 64 | ], 65 | "report-dir": "coverage" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /test/unit/helpers.js: -------------------------------------------------------------------------------- 1 | /* eslint-env qunit */ 2 | const { test } = QUnit; 3 | const sinon = require('sinon'); 4 | const JsReporters = require('../../index.js'); 5 | const data = require('../fixtures/unit.js'); 6 | 7 | QUnit.module('Helpers', function () { 8 | const dummyFunc = function () {}; 9 | 10 | QUnit.module('autoregister', hooks => { 11 | hooks.beforeEach(function () { 12 | global.QUnit = undefined; 13 | global.mocha = undefined; 14 | global.jasmine = undefined; 15 | }); 16 | 17 | hooks.afterEach(function () { 18 | delete global.QUnit; 19 | delete global.mocha; 20 | delete global.jasmine; 21 | }); 22 | 23 | test('register the QUnitAdapter', assert => { 24 | global.QUnit = { 25 | begin: sinon.stub(), 26 | testStart: dummyFunc, 27 | log: dummyFunc, 28 | testDone: dummyFunc, 29 | done: dummyFunc 30 | }; 31 | 32 | JsReporters.autoRegister(); 33 | 34 | assert.true(global.QUnit.begin.calledOnce); 35 | }); 36 | 37 | test('register the MochaAdapter', assert => { 38 | global.mocha = { 39 | reporter: sinon.stub() 40 | }; 41 | 42 | JsReporters.autoRegister(); 43 | 44 | assert.true(global.mocha.reporter.calledOnce); 45 | }); 46 | 47 | test('register the JasmineAdapter', assert => { 48 | const spy = sinon.stub(); 49 | global.jasmine = { 50 | getEnv: function () { 51 | return { 52 | addReporter: spy 53 | }; 54 | } 55 | }; 56 | 57 | JsReporters.autoRegister(); 58 | 59 | assert.true(spy.calledOnce); 60 | }); 61 | 62 | test('should throw an error if no testing framework was found', assert => { 63 | assert.throws(JsReporters.autoRegister, Error); 64 | }); 65 | }); 66 | 67 | QUnit.module('create functions', () => { 68 | test('return a test start', assert => { 69 | assert.propEqual( 70 | JsReporters.createTestStart(data.passingTest), 71 | data.passingTestStart 72 | ); 73 | }); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /karma.conf.sauce.js: -------------------------------------------------------------------------------- 1 | module.exports = function (config) { 2 | const { babel } = require('@rollup/plugin-babel'); 3 | const commonjs = require('@rollup/plugin-commonjs'); 4 | const { nodeResolve } = require('@rollup/plugin-node-resolve'); 5 | 6 | config.set({ 7 | files: [ 8 | 'node_modules/sinon/pkg/sinon.js', 9 | 'test/unit/tap-reporter.js', 10 | 'test/unit/helpers.js' 11 | ], 12 | preprocessors: { 13 | 'test/unit/**.js': ['rollup'] 14 | }, 15 | rollupPreprocessor: { 16 | external: ['sinon'], 17 | output: { 18 | globals: { 19 | sinon: 'sinon' 20 | }, 21 | format: 'iife', 22 | name: 'JsReporters_Test', 23 | sourcemap: true 24 | }, 25 | // See rollup.config.js 26 | plugins: [ 27 | nodeResolve({ 28 | preferBuiltins: false 29 | }), 30 | commonjs({ 31 | requireReturnsDefault: 'preferred' 32 | }), 33 | babel({ 34 | babelHelpers: 'bundled', 35 | babelrc: false, 36 | presets: [ 37 | ['@babel/preset-env', { 38 | targets: { 39 | ie: 9 40 | } 41 | }] 42 | ] 43 | }) 44 | ] 45 | }, 46 | frameworks: ['qunit'], 47 | sauceLabs: { 48 | username: process.env.SAUCE_USERNAME, 49 | accessKey: process.env.SAUCE_ACCESS_KEY, 50 | region: process.env.SAUCE_REGION || 'us' 51 | }, 52 | customLaunchers: { 53 | firefox78: { 54 | base: 'SauceLabs', 55 | browserName: 'firefox', 56 | version: '78.0' 57 | }, 58 | ie9: { 59 | base: 'SauceLabs', 60 | browserName: 'internet explorer', 61 | version: '9' 62 | }, 63 | ie10: { 64 | base: 'SauceLabs', 65 | browserName: 'internet explorer', 66 | version: '10' 67 | }, 68 | ie11: { 69 | base: 'SauceLabs', 70 | browserName: 'internet explorer', 71 | version: '11' 72 | }, 73 | safari9: { 74 | base: 'SauceLabs', 75 | browserName: 'safari', 76 | version: '9' 77 | }, 78 | safari10: { 79 | base: 'SauceLabs', 80 | browserName: 'safari', 81 | version: '10' 82 | }, 83 | safari: { 84 | base: 'SauceLabs', 85 | browserName: 'safari' 86 | }, 87 | edge15: { 88 | // Edge 40 (EdgeHTML 15) 89 | base: 'SauceLabs', 90 | browserName: 'MicrosoftEdge', 91 | version: '15.15063' 92 | }, 93 | edge: { 94 | // Edge 80+ (Chromium) 95 | base: 'SauceLabs', 96 | browserName: 'MicrosoftEdge', 97 | version: 'latest' 98 | }, 99 | chrome80: { 100 | base: 'SauceLabs', 101 | browserName: 'chrome', 102 | version: '80.0' 103 | } 104 | }, 105 | concurrency: 4, 106 | browsers: [ 107 | 'firefox78', 108 | 'ie9', 109 | 'ie10', 110 | 'ie11', 111 | 'safari9', 112 | 'safari10', 113 | 'safari', 114 | 'edge15', 115 | 'edge', 116 | 'chrome80' 117 | ], 118 | logLevel: 'WARN', 119 | singleRun: true, 120 | autoWatch: false 121 | }); 122 | }; 123 | -------------------------------------------------------------------------------- /test/integration/adapters-run.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const JsReporters = require('../../index.js'); 3 | 4 | const testDir = path.join(__dirname, '../fixtures/integration'); 5 | 6 | function rerequire (file) { 7 | const resolveOptions = process.env.JSREP_TMPDIR 8 | ? { 9 | // Only consider our temporary install. 10 | // Ignore the js-reporters own depDependencies. 11 | paths: [process.env.JSREP_TMPDIR] 12 | } 13 | : {}; 14 | const resolved = require.resolve(file, resolveOptions); 15 | delete require.cache[resolved]; 16 | return require(resolved); 17 | } 18 | 19 | /** 20 | * Exports a function for each adapter that will run 21 | * against a default test fixture. 22 | */ 23 | module.exports.main = { 24 | Jasmine: function (collectData) { 25 | const Jasmine = rerequire('jasmine'); 26 | const jasmine = new Jasmine(); 27 | 28 | jasmine.loadConfig({ 29 | spec_dir: 'test/fixtures/integration', 30 | // Jasmine 3.0.0 and later use randomized order by default 31 | // 32 | random: false, 33 | spec_files: ['jasmine.js'] 34 | }); 35 | 36 | if (jasmine.env && jasmine.env.clearReporters) { 37 | // Jasmine 2.5.2 and later no longer remove the default CLI reporters 38 | // when calling addReporter(). Instead, it introduced clearReporters() 39 | // to do this manually. 40 | jasmine.env.clearReporters(); 41 | } 42 | if (jasmine.completionReporter) { 43 | // Jasmine 2.5.2 and later no longer remove the default CLI reporters 44 | // when calling addReporter(). The clearReporters() method above removes 45 | // the ConsoleReporter, but theCompletionReporter that exits the process, 46 | // is registered late via jasmine.execute() in a way we can't remove in time. 47 | // Stub it instead? 48 | jasmine.completionReporter.onComplete(function () {}); 49 | } 50 | const jasmineRunner = new JsReporters.JasmineAdapter(jasmine); 51 | collectData(jasmineRunner); 52 | 53 | jasmine.execute(); 54 | }, 55 | 56 | 'QUnit (1.x)': function (collectData) { 57 | // Legacy npm package name 58 | const QUnit = rerequire('qunitjs'); 59 | global.QUnit = QUnit; 60 | QUnit.config.autorun = false; 61 | 62 | const qunitRunner = new JsReporters.QUnitAdapter(QUnit); 63 | collectData(qunitRunner); 64 | 65 | rerequire(path.join(testDir, 'qunit.js')); 66 | 67 | QUnit.load(); 68 | }, 69 | QUnit: function (collectData) { 70 | // Get a reporter context independent of the current integration test 71 | const QUnit = rerequire('qunit'); 72 | global.QUnit = QUnit; 73 | QUnit.config.autorun = false; 74 | 75 | const qunitRunner = new JsReporters.QUnitAdapter(QUnit); 76 | collectData(qunitRunner); 77 | 78 | rerequire(path.join(testDir, 'qunit.js')); 79 | 80 | QUnit.start(); 81 | }, 82 | Mocha: function (collectData) { 83 | const Mocha = rerequire('mocha'); 84 | const mocha = new Mocha(); 85 | mocha.addFile(path.join(testDir, 'mocha.js')); 86 | 87 | const mochaRunner = new JsReporters.MochaAdapter(mocha); 88 | 89 | collectData(mochaRunner); 90 | mocha.run(); 91 | } 92 | }; 93 | 94 | module.exports.todo = { 95 | QUnit: function (collectData) { 96 | const QUnit = rerequire('qunit'); 97 | global.QUnit = QUnit; 98 | QUnit.config.autorun = false; 99 | 100 | const qunitRunner = new JsReporters.QUnitAdapter(QUnit); 101 | collectData(qunitRunner); 102 | 103 | rerequire(path.join(testDir, 'qunit-todo.js')); 104 | 105 | QUnit.start(); 106 | } 107 | }; 108 | -------------------------------------------------------------------------------- /test/integration/reference-data-todo.js: -------------------------------------------------------------------------------- 1 | const failedAssertion = { 2 | passed: false 3 | }; 4 | 5 | const passedAssertion = { 6 | passed: true 7 | }; 8 | 9 | const globalPassStart = { 10 | name: 'global pass', 11 | suiteName: null, 12 | fullName: [ 13 | 'global pass' 14 | ] 15 | }; 16 | const globalPassEnd = { 17 | name: 'global pass', 18 | suiteName: null, 19 | fullName: [ 20 | 'global pass' 21 | ], 22 | status: 'passed', 23 | runtime: 42, 24 | errors: [], 25 | assertions: [ 26 | passedAssertion 27 | ] 28 | }; 29 | 30 | const globalTodoStart = { 31 | name: 'global todo', 32 | suiteName: null, 33 | fullName: [ 34 | 'global todo' 35 | ] 36 | }; 37 | const globalTodoEnd = { 38 | name: 'global todo', 39 | suiteName: null, 40 | fullName: [ 41 | 'global todo' 42 | ], 43 | status: 'todo', 44 | runtime: 42, 45 | errors: [ 46 | failedAssertion 47 | ], 48 | assertions: [ 49 | failedAssertion 50 | ] 51 | }; 52 | 53 | const suite1Start = { 54 | name: 'suite with a todo test', 55 | fullName: [ 56 | 'suite with a todo test' 57 | ] 58 | }; 59 | const passInSuite1Start = { 60 | name: 'should pass', 61 | suiteName: 'suite with a todo test', 62 | fullName: [ 63 | 'suite with a todo test', 64 | 'should pass' 65 | ] 66 | }; 67 | const passInSuite1End = { 68 | name: 'should pass', 69 | suiteName: 'suite with a todo test', 70 | fullName: [ 71 | 'suite with a todo test', 72 | 'should pass' 73 | ], 74 | status: 'passed', 75 | runtime: 42, 76 | errors: [], 77 | assertions: [ 78 | passedAssertion 79 | ] 80 | }; 81 | const todoInSuite1Start = { 82 | name: 'should todo', 83 | suiteName: 'suite with a todo test', 84 | fullName: [ 85 | 'suite with a todo test', 86 | 'should todo' 87 | ] 88 | }; 89 | const todoInSuite1End = { 90 | name: 'should todo', 91 | suiteName: 'suite with a todo test', 92 | fullName: [ 93 | 'suite with a todo test', 94 | 'should todo' 95 | ], 96 | status: 'todo', 97 | runtime: 42, 98 | errors: [ 99 | failedAssertion 100 | ], 101 | assertions: [ 102 | failedAssertion 103 | ] 104 | }; 105 | const suite1End = { 106 | name: 'suite with a todo test', 107 | fullName: [ 108 | 'suite with a todo test' 109 | ], 110 | status: 'passed', 111 | runtime: 42 112 | }; 113 | 114 | const suite2Start = { 115 | name: 'todo suite', 116 | fullName: [ 117 | 'todo suite' 118 | ] 119 | }; 120 | const todoInSuite2Start = { 121 | name: 'should todo', 122 | suiteName: 'todo suite', 123 | fullName: [ 124 | 'todo suite', 125 | 'should todo' 126 | ] 127 | }; 128 | const todoInSuite2End = { 129 | name: 'should todo', 130 | suiteName: 'todo suite', 131 | fullName: [ 132 | 'todo suite', 133 | 'should todo' 134 | ], 135 | status: 'todo', 136 | runtime: 42, 137 | errors: [ 138 | failedAssertion 139 | ], 140 | assertions: [ 141 | failedAssertion 142 | ] 143 | }; 144 | const suite2End = { 145 | name: 'todo suite', 146 | fullName: [ 147 | 'todo suite' 148 | ], 149 | status: 'passed', 150 | runtime: 42 151 | }; 152 | 153 | const runStart = { 154 | name: null, 155 | testCounts: { 156 | total: 7 157 | } 158 | }; 159 | const runEnd = { 160 | name: null, 161 | status: 'passed', 162 | testCounts: { 163 | passed: 4, 164 | failed: 0, 165 | skipped: 0, 166 | todo: 3, 167 | total: 7 168 | }, 169 | runtime: 42 170 | }; 171 | 172 | module.exports = [ 173 | ['runStart', runStart], 174 | 175 | ['testStart', globalPassStart], 176 | ['testEnd', globalPassEnd], 177 | 178 | ['testStart', globalTodoStart], 179 | ['testEnd', globalTodoEnd], 180 | 181 | ['suiteStart', suite1Start], 182 | ['testStart', passInSuite1Start], 183 | ['testEnd', passInSuite1End], 184 | ['testStart', todoInSuite1Start], 185 | ['testEnd', todoInSuite1End], 186 | ['suiteEnd', suite1End], 187 | 188 | ['suiteStart', suite2Start], 189 | ['testStart', todoInSuite2Start], 190 | ['testEnd', todoInSuite2End], 191 | ['suiteEnd', suite2End], 192 | 193 | ['runEnd', runEnd] 194 | ]; 195 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog for the [js-reporters](https://www.npmjs.com/package/js-reporters) package. See [spec/](./spec/cri-draft.adoc) for the CRI standard. 2 | 3 | 2.1.0 / 2021-06-06 4 | ================== 5 | 6 | ### Added 7 | 8 | * QUnitAdapter: Support for `todo` tests. (Timo Tijhof) [#140](https://github.com/qunitjs/js-reporters/pull/140) 9 | 10 | 2.0.0 / 2021-04-04 11 | ================== 12 | 13 | This release provides a simplified spec, with various properties and features removed. Overall the new spec is considered narrower than the previous one. Existing reporters that support old producers should naturally support new producers as well. Existing producers can choose to remain unchanged or to remove older portions in a future release. 14 | 15 | This release provides a simplified [spec](https://github.com/qunitjs/js-reporters/blob/v2.0.0/spec/cri-draft.adoc), with various properties and features removed. Overall the new spec is considered narrower than the previous one. Existing reporters that support old producers should naturally support new producers as well. Existing producers can choose to remain unchanged or to remove older portions in a future release. 16 | 17 | ### Added 18 | 19 | * Add `SummaryReporter` implementation. 20 | 21 | ### Changed 22 | 23 | * Spec: Rewrite current proposal into a formal [specification](https://github.com/qunitjs/js-reporters/blob/v2.0.0/spec/cri-draft.adoc). (Timo Tijhof) 24 | * Spec: Remove "todo" from Assertion event data. (Keith Cirkel) [#119](https://github.com/qunitjs/js-reporters/pull/119) 25 | * Spec: Remove "tests" and "childSuites" from SuiteStart and SuiteEnd event data. 26 | * Spec: Prefer `null` instead of `undefined` for optional fields. 27 | * TapReporter: Improve formatting of multi-line strings. [#109](https://github.com/qunitjs/js-reporters/issues/109) 28 | 29 | ### Fixed 30 | 31 | * TapReporter: Fix support objects with cycles, avoiding uncaught errors. (Zachary Mulgrew) [#104](https://github.com/qunitjs/js-reporters/issues/104) 32 | * TapReporter: Defend against mocked `console` object. [#125](https://github.com/qunitjs/js-reporters/issues/125) 33 | * MochaAdapter: Fix support for Mocha 8, due to changes in `STATE_PENDING`. [#116](https://github.com/qunitjs/js-reporters/issues/116) 34 | 35 | ### Removed 36 | 37 | * Remove support for Node.js 8 and older. This release requires Node.js 10 or later. (Browser support has not changed and remains IE 9+, see [README](./README.md#runtime-support).) 38 | * Helpers: Remove the `Assertion`, `Test`, and `Suite` classes. 39 | * Helpers: Remove `collectSuite{Start,StartData,EndData}` methods. 40 | 41 | 1.2.3 / 2020-09-07 42 | ================== 43 | 44 | ### Changed 45 | 46 | * TapReporter: Align `actual` with `expected` in TAP output. (Robert Jackson) [#107](https://github.com/qunitjs/js-reporters/pull/107) 47 | 48 | ### Fixed 49 | 50 | * Helpers: Correct spelling in `autoRegister()` error message. (P. Roebuck) [#108](https://github.com/qunitjs/js-reporters/issues/108) 51 | * TapReporter: Revert "Fix YAML syntax". [#110](https://github.com/qunitjs/js-reporters/issues/110) 52 | 53 | 1.2.2 / 2019-05-13 54 | ================== 55 | 56 | ### Fixed 57 | 58 | * TapReporter: Fix YAML syntax. (jeberger) [#110](https://github.com/qunitjs/js-reporters/issues/110) 59 | 60 | 1.2.1 / 2017-07-04 61 | ================== 62 | 63 | ### Changed 64 | 65 | * TapReporter: Print "actual:", "expected:" even if undefined. (Martin Olsson) 66 | 67 | ### Fixed 68 | 69 | * TapReporter: Drop accidentally committed `console.warn()` statement. (Martin Olsson) 70 | 71 | 1.2.0 / 2017-03-22 72 | ================== 73 | 74 | ### Added 75 | 76 | * TapReporter: Improve TAP information and styling. (Florentin Simion) 77 | * TapReporter: Support todo test in TAP reporter. (Trent Willis) 78 | * Docs: Add API docs for the js-reporters package. (Florentin Simion) 79 | 80 | 1.1.0 / 2016-08-10 81 | ================== 82 | 83 | * Fix IE8 support by removing ES5 getters. (Florentin Simion) [#82](https://github.com/qunitjs/js-reporters/pull/82) 84 | * Add testCounts property for Suites. (Florentin Simion) [#85](https://github.com/qunitjs/js-reporters/pull/85) 85 | * Add assertions property for tests. (Florentin Simion) [#31](https://github.com/qunitjs/js-reporters/issues/31) 86 | * Normalize assertions. (Florentin Simion) [#81](https://github.com/qunitjs/js-reporters/pull/81) 87 | * Change test `testName` property to `name`. (Florentin Simion) [#81](https://github.com/qunitjs/js-reporters/pull/81) 88 | * Add types to the event data. (Florentin Simion) [#84](https://github.com/qunitjs/js-reporters/pull/84) 89 | -------------------------------------------------------------------------------- /lib/reporters/SummaryReporter.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const hasOwn = Object.hasOwnProperty; 3 | 4 | module.exports = class SummaryReporter extends EventEmitter { 5 | constructor (runner) { 6 | super(); 7 | 8 | this.top = { 9 | name: null, 10 | tests: [], 11 | status: null, 12 | testCounts: null, 13 | runtime: null 14 | }; 15 | 16 | // Map of full name to test objects 17 | this.attachedTests = {}; 18 | 19 | // Map of full name to array of children test objects 20 | this.detachedChildren = {}; 21 | 22 | this.summary = null; 23 | 24 | runner.on('suiteEnd', (suiteEnd) => { 25 | const ownFull = suiteEnd.fullName.join('>'); 26 | const suiteName = suiteEnd.fullName.length >= 2 ? suiteEnd.fullName.slice(-2, -1)[0] : null; 27 | const parentFull = suiteEnd.fullName.length > 1 ? suiteEnd.fullName.slice(0, -1).join('>') : null; 28 | 29 | let children; 30 | if (hasOwn.call(this.detachedChildren, ownFull)) { 31 | children = this.detachedChildren[ownFull]; 32 | delete this.detachedChildren[ownFull]; 33 | } else { 34 | children = []; 35 | } 36 | 37 | const test = { 38 | ...suiteEnd, 39 | suiteName: suiteName, 40 | errors: [], 41 | assertions: [], 42 | tests: children 43 | }; 44 | 45 | if (parentFull === null) { 46 | this.top.tests.push(test); 47 | } else if (hasOwn.call(this.attachedTests, parentFull)) { 48 | this.attachedTests[parentFull].tests.push(test); 49 | } else if (hasOwn.call(this.detachedChildren, parentFull)) { 50 | this.detachedChildren[parentFull].push(test); 51 | } else { 52 | this.detachedChildren[parentFull] = [test]; 53 | } 54 | 55 | this.attachedTests[ownFull] = test; 56 | }); 57 | 58 | runner.on('testEnd', (testEnd) => { 59 | const ownFull = testEnd.fullName.join('>'); 60 | const parentFull = testEnd.fullName.length > 1 ? testEnd.fullName.slice(0, -1).join('>') : null; 61 | 62 | let children; 63 | if (hasOwn.call(this.detachedChildren, ownFull)) { 64 | children = this.detachedChildren[ownFull]; 65 | delete this.detachedChildren[ownFull]; 66 | } else { 67 | children = []; 68 | } 69 | 70 | const test = { 71 | ...testEnd, 72 | tests: children 73 | }; 74 | 75 | if (parentFull === null) { 76 | this.top.tests.push(test); 77 | } else if (hasOwn.call(this.attachedTests, parentFull)) { 78 | this.attachedTests[parentFull].tests.push(test); 79 | } else if (hasOwn.call(this.detachedChildren, parentFull)) { 80 | this.detachedChildren[parentFull].push(test); 81 | } else { 82 | this.detachedChildren[parentFull] = [test]; 83 | } 84 | 85 | this.attachedTests[ownFull] = test; 86 | }); 87 | 88 | runner.once('runEnd', (runEnd) => { 89 | this.top.name = runEnd.name; 90 | this.top.status = runEnd.status; 91 | this.top.testCounts = runEnd.testCounts; 92 | this.top.runtime = runEnd.runtime; 93 | 94 | this.summary = this.top; 95 | 96 | const problems = Object.keys(this.detachedChildren).join(', '); 97 | if (problems.length) { 98 | console.error('detachedChildren:', problems); 99 | } 100 | this.top = null; 101 | this.attachedTests = null; 102 | this.detachedChildren = null; 103 | 104 | this.emit('summary', this.summary); 105 | }); 106 | } 107 | 108 | static init (runner) { 109 | return new SummaryReporter(runner); 110 | } 111 | 112 | /** 113 | * Get the summary via callback or Promise. 114 | * 115 | * @param {Function} [callback] 116 | * @param {null|Error} [callback.err] 117 | * @param {Object} [callback.summary] 118 | * @return {undefined|Promise} 119 | */ 120 | getSummary (callback) { 121 | if (callback) { 122 | if (this.summary) { 123 | callback(null, this.summary); 124 | } else { 125 | this.once('summary', (summary) => { 126 | callback(null, summary); 127 | }); 128 | } 129 | } else { 130 | // Support IE 9-11: Only reference 'Promise' (which we don't polyfill) when needed. 131 | // If the user doesn't use this reporter, or only its callback signature, then it should 132 | // work in older browser as well. 133 | return new Promise((resolve) => { 134 | if (this.summary) { 135 | resolve(this.summary); 136 | } else { 137 | this.once('summary', (summary) => { 138 | resolve(summary); 139 | }); 140 | } 141 | }); 142 | } 143 | } 144 | }; 145 | -------------------------------------------------------------------------------- /docs/example.md: -------------------------------------------------------------------------------- 1 | # Example 2 | 3 | ```js 4 | 5 | it('global test', function() { 6 | 7 | }); 8 | 9 | describe('suite 1', function() { 10 | describe('suite 2', function() { 11 | it('test2', function() { 12 | expect(true).to.be.true; 13 | }); 14 | }); 15 | 16 | it('test1', function() { 17 | expect(true).to.be.true; 18 | expect(true).to.be.false; 19 | }); 20 | }); 21 | ``` 22 | 23 | For the above Mocha test suite, the following events and even data are expected to be emitted by a CRI-compliant interface (Common Reporter Interface): 24 | 25 | - **runStart**: 26 | 27 | ```js 28 | Suite { 29 | name: undefined, 30 | fullName: [], 31 | tests: [Test('global test')], 32 | childSuites: [Suite('suite1')], 33 | status: undefined, 34 | testCounts: { 35 | passed: undefined, 36 | failed: undefined, 37 | skipped: undefined, 38 | total: 2 39 | }, 40 | runtime: undefined 41 | } 42 | ``` 43 | 44 | - **testStart** 45 | 46 | ```js 47 | Test { 48 | name: 'global test', 49 | suiteName: undefined, 50 | fullName: ['global test'], 51 | status: undefined, 52 | runtime: undefined, 53 | errors: undefined, 54 | assertions: undefined 55 | } 56 | ``` 57 | 58 | - **testEnd** 59 | 60 | ```js 61 | Test { 62 | name: 'global test', 63 | suiteName: undefined, 64 | fullName: ['global test'], 65 | status: 'passed', 66 | runtime: 1, 67 | errors: [], 68 | assertions: [] 69 | ``` 70 | 71 | - **suiteStart** 72 | 73 | ```js 74 | Suite { 75 | name: 'suite1', 76 | fullName: ['suite1'], 77 | tests: [Test('test1')], 78 | childSuites: [Suite('suite2')], 79 | status: undefined, 80 | testCounts: { 81 | passed: undefined, 82 | failed: undefined, 83 | skipped: undefined, 84 | total: 2 85 | }, 86 | runtime: undefined 87 | ``` 88 | 89 | - **suiteStart** 90 | 91 | ```js 92 | Suite { 93 | name: 'suite2', 94 | fullName: ['suite1', 'suite2'], 95 | tests: [Test('test2')], 96 | childSuites: [], 97 | status: undefined, 98 | testCounts: { 99 | passed: undefined, 100 | failed: undefined, 101 | skipped: undefined, 102 | total: 1 103 | }, 104 | runtime: undefined 105 | ``` 106 | 107 | - **testStart** 108 | 109 | ```js 110 | Test { 111 | name: 'test2', 112 | suiteName: 'suite2', 113 | fullName: ['suite1', 'suite2', 'test2'], 114 | status: undefined, 115 | runtime: undefined, 116 | errors: undefined, 117 | assertions: undefined 118 | ``` 119 | 120 | - **testEnd**: 121 | 122 | ```js 123 | Test { 124 | name: 'test2', 125 | suiteName: 'suite2', 126 | fullName: ['suite1', 'suite2', 'test2'], 127 | status: `passed`, 128 | runtime: 1, 129 | errors: [], 130 | assertions: [{ 131 | passed: true, 132 | actual: true, 133 | expected: true, 134 | message: `some message`, 135 | stack: undefined 136 | }] 137 | ``` 138 | 139 | - **suiteEnd** 140 | 141 | ```js 142 | Suite { 143 | name: 'suite2', 144 | fullName: ['suite1', 'suite2'], 145 | tests: [Test('test2')], 146 | childSuites: [], 147 | status: `passed`, 148 | testCounts: { 149 | passed: 1, 150 | failed: 0, 151 | skipped: 0, 152 | total: 1 153 | }, 154 | runtime: 1 155 | ``` 156 | 157 | - **testStart** 158 | 159 | ```js 160 | Test { 161 | name: 'test1', 162 | suiteName: 'suite1', 163 | fullName: ['suite1', 'test'], 164 | status: undefined, 165 | runtime: undefined, 166 | errors: undefined, 167 | assertions: undefined 168 | ``` 169 | 170 | - **testEnd**: 171 | 172 | ```js 173 | Test { 174 | name: 'test1', 175 | suiteName: 'suite1', 176 | fullName: ['suite1', 'test'], 177 | status: `failed`, 178 | runtime: 2, 179 | errors: [{ 180 | passed: false, 181 | actual: true, 182 | expected: false, 183 | message: `some message`, 184 | stack: `stack trace` 185 | }], 186 | assertions: [{ 187 | passed: true, 188 | actual: true, 189 | expected: true, 190 | message: `some message`, 191 | stack: undefined 192 | }, { 193 | passed: false, 194 | actual: true, 195 | expected: false, 196 | message: `some message`, 197 | stack: `stack trace` 198 | }] 199 | ``` 200 | 201 | - **suiteEnd** 202 | 203 | ```js 204 | Suite { 205 | name: 'suite1', 206 | fullName: ['suite1'], 207 | tests: [Test('test1')], 208 | childSuites: [], 209 | status: `failed`, 210 | testCounts: { 211 | passed: 1, 212 | failed: 1, 213 | skipped: 0, 214 | total: 2 215 | }, 216 | runtime: 3 217 | ``` 218 | 219 | - **runEnd**: 220 | 221 | ```js 222 | Suite { 223 | name: undefined, 224 | fullName: [], 225 | tests: [Test('global test')], 226 | childSuites: [Suite('suite1')], 227 | status: `failed`, 228 | testCounts: { 229 | passed: 2, 230 | failed: 1, 231 | skipped: 0, 232 | total: 3 233 | }, 234 | runtime: 4 235 | } 236 | ``` 237 | -------------------------------------------------------------------------------- /test/versions/versions-reporting.js: -------------------------------------------------------------------------------- 1 | const childProcess = require('child_process'); 2 | const fs = require('fs'); 3 | const os = require('os'); 4 | const path = require('path'); 5 | 6 | const kleur = require('kleur'); 7 | const rimraf = require('rimraf'); 8 | const semver = require('semver'); 9 | 10 | const rootDir = path.join(__dirname, '..', '..'); 11 | const logDir = path.join(rootDir, 'log'); 12 | const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'jsreptest-')); 13 | const rootPkg = require('../../package.json'); 14 | const packages = [ 15 | { name: 'jasmine', range: '*', default: rootPkg.devDependencies.jasmine }, 16 | 17 | // Our adapter supports QUnit 1.19, but our integration test exercises the 18 | // "nested module" feature which was first released in QUnit 1.20. 19 | // This package goes until qunitjs@2.4.1, later releases are under 'qunit'. 20 | { name: 'qunitjs', range: '>= 1.20.0', default: rootPkg.devDependencies.qunitjs }, 21 | 22 | // Ignore versions 0.x and 1.x versions of 'qunit', which were something else. 23 | { name: 'qunit', range: '>= 2.0.0', default: rootPkg.devDependencies.qunit }, 24 | 25 | // Our adapter supports Mocha 1.18.0 and later. 26 | // Various features were missing before then. 27 | { name: 'mocha', range: '>= 1.18.0', default: rootPkg.devDependencies.mocha } 28 | ]; 29 | 30 | rimraf.sync(logDir); 31 | rimraf.sync(tmpDir); 32 | 33 | console.log(); 34 | 35 | /** 36 | * Takes each version of each framework available on npm and runs the tests 37 | * against it, making in the end a summary of versions which are working with 38 | * the js-reporters adapters and which don't. 39 | */ 40 | for (const pkg of packages) { 41 | const output = childProcess.execSync(`npm view ${pkg.name} versions`) 42 | .toString() 43 | .replace(/[[]|]|'| |\n/g, ''); 44 | const versions = output.split(','); 45 | const workingVersions = []; 46 | const notWorkingVersions = []; 47 | 48 | console.log(kleur.green().bold().underline(pkg.name)); 49 | console.log(); 50 | 51 | versions.forEach(function (version) { 52 | if (!semver.satisfies(version, pkg.range)) { 53 | return; 54 | } 55 | process.stdout.write(kleur.dim(`Testing version: ${version}`)); 56 | 57 | // Install this package in a temporary location rather overriding 58 | // the main package in our woring copy. 59 | // - Allows us to keep using the (latest) 'qunit' CLI for the integration 60 | // test itself. 61 | // - Avoid dirtying the main package.json/package-lock.json files. 62 | // - Avoid false positive tests where a broken release appears to 63 | // work due to other packages we have installed. 64 | // Note that: 65 | // - npm-install is "smart" and operates on the nearest "real" 66 | // package even when in a sub directory. We create a local package.json 67 | // file so that the directory is considered its own "package". 68 | // - Even when limiting require.resolve() in integration/adapters-run.js, 69 | // to a specific directory, if that directory is a subdirectory of 70 | // our working copy, it will still fallback to our dependencies, 71 | // which can cause additional false positives. As such, create this 72 | // install under os.tmpdir() instead. 73 | fs.mkdirSync(tmpDir); 74 | fs.writeFileSync(path.join(tmpDir, 'package.json'), '{}'); 75 | const others = packages 76 | .filter(other => other.name !== pkg.name) 77 | .map(other => `${other.name}@${other.default}`) 78 | .join(' '); 79 | childProcess.execSync(`npm install ${pkg.name}@${version} ${others}`, { 80 | cwd: tmpDir, 81 | stdio: ['ignore', 'ignore', 'ignore'] 82 | }); 83 | 84 | const details = childProcess.spawnSync('npm', ['run', 'test-integration'], { 85 | env: { 86 | ...process.env, 87 | JSREP_TMPDIR: tmpDir 88 | } 89 | }); 90 | 91 | // Clean up temporary install 92 | rimraf.sync(tmpDir); 93 | 94 | let logFile; 95 | if (details.status === 0) { 96 | workingVersions.push(version); 97 | logFile = path.join(logDir, `passed--${pkg.name}--${version}.log`); 98 | process.stdout.write(kleur.dim('.') + '\n'); 99 | } else { 100 | notWorkingVersions.push(version); 101 | logFile = path.join(logDir, `failed--${pkg.name}--${version}.log`); 102 | const relative = path.relative(rootDir, logFile); 103 | process.stdout.write(`, saved failure output to ${relative}.\n`); 104 | } 105 | 106 | fs.mkdirSync(logDir, { recursive: true }); 107 | fs.writeFileSync( 108 | logFile, 109 | `stdout:\n\n${details.stdout}\n\n\n\nstderr:\n\n${details.stderr}\n` 110 | ); 111 | }); 112 | 113 | module.exports[pkg.name] = notWorkingVersions; 114 | 115 | console.log(); 116 | console.log(kleur.green('Working: ' + workingVersions.join(', ') + ';')); 117 | console.log(kleur.red('Not working: ' + notWorkingVersions.join(', ') + ';')); 118 | console.log(); 119 | } 120 | -------------------------------------------------------------------------------- /test/unit/summary-reporter.js: -------------------------------------------------------------------------------- 1 | /* eslint-env qunit */ 2 | const { test } = QUnit; 3 | const JsReporters = require('../../index.js'); 4 | 5 | // Generally in test frameworks, a parent ends after its children 6 | // have finished. Thus we build the tree "upwards", from the inside out. 7 | function playUpwardRun (emitter) { 8 | emitter.emit('runStart', { 9 | name: null, 10 | testCounts: { 11 | total: 2 12 | } 13 | }); 14 | emitter.emit('testEnd', { 15 | name: 'foo', 16 | suiteName: 'Inner suite', 17 | fullName: ['Outer suite', 'Inner suite', 'foo'], 18 | status: 'passed', 19 | runtime: 42, 20 | errors: [], 21 | assertions: [ 22 | { passed: true } 23 | ] 24 | }); 25 | emitter.emit('testEnd', { 26 | name: 'bar', 27 | suiteName: 'Inner suite', 28 | fullName: ['Outer suite', 'Inner suite', 'bar'], 29 | status: 'passed', 30 | runtime: 42, 31 | errors: [], 32 | assertions: [ 33 | { passed: true } 34 | ] 35 | }); 36 | emitter.emit('suiteEnd', { 37 | name: 'Inner suite', 38 | fullName: ['Outer suite', 'Inner suite'], 39 | status: 'passed', 40 | runtime: 42 41 | }); 42 | emitter.emit('suiteEnd', { 43 | name: 'Outer suite', 44 | fullName: ['Outer suite'], 45 | status: 'passed', 46 | runtime: 42 47 | }); 48 | emitter.emit('runEnd', { 49 | name: null, 50 | status: 'passed', 51 | testCounts: { 52 | passed: 4, 53 | failed: 0, 54 | skipped: 0, 55 | todo: 0, 56 | total: 4 57 | }, 58 | runtime: 42 59 | }); 60 | } 61 | 62 | // But we support receiving events in any order, 63 | // so test the reverse as well. 64 | function playDownwardRun (emitter) { 65 | emitter.emit('runStart', { 66 | name: null, 67 | testCounts: { 68 | total: 2 69 | } 70 | }); 71 | emitter.emit('suiteEnd', { 72 | name: 'Outer suite', 73 | fullName: ['Outer suite'], 74 | status: 'passed', 75 | runtime: 42 76 | }); 77 | emitter.emit('suiteEnd', { 78 | name: 'Inner suite', 79 | fullName: ['Outer suite', 'Inner suite'], 80 | status: 'passed', 81 | runtime: 42 82 | }); 83 | emitter.emit('testEnd', { 84 | name: 'foo', 85 | suiteName: 'Inner suite', 86 | fullName: ['Outer suite', 'Inner suite', 'foo'], 87 | status: 'passed', 88 | runtime: 42, 89 | errors: [], 90 | assertions: [ 91 | { passed: true } 92 | ] 93 | }); 94 | emitter.emit('testEnd', { 95 | name: 'bar', 96 | suiteName: 'Inner suite', 97 | fullName: ['Outer suite', 'Inner suite', 'bar'], 98 | status: 'passed', 99 | runtime: 42, 100 | errors: [], 101 | assertions: [ 102 | { passed: true } 103 | ] 104 | }); 105 | emitter.emit('runEnd', { 106 | name: null, 107 | status: 'passed', 108 | testCounts: { 109 | passed: 4, 110 | failed: 0, 111 | skipped: 0, 112 | todo: 0, 113 | total: 4 114 | }, 115 | runtime: 42 116 | }); 117 | } 118 | 119 | const expectedSummary = { 120 | name: null, 121 | tests: [ 122 | { 123 | name: 'Outer suite', 124 | suiteName: null, 125 | fullName: ['Outer suite'], 126 | status: 'passed', 127 | runtime: 42, 128 | errors: [], 129 | assertions: [], 130 | tests: [ 131 | { 132 | name: 'Inner suite', 133 | suiteName: 'Outer suite', 134 | fullName: ['Outer suite', 'Inner suite'], 135 | status: 'passed', 136 | runtime: 42, 137 | errors: [], 138 | assertions: [], 139 | tests: [ 140 | { 141 | name: 'foo', 142 | suiteName: 'Inner suite', 143 | fullName: ['Outer suite', 'Inner suite', 'foo'], 144 | status: 'passed', 145 | runtime: 42, 146 | errors: [], 147 | assertions: [ 148 | { passed: true } 149 | ], 150 | tests: [] 151 | }, 152 | { 153 | name: 'bar', 154 | suiteName: 'Inner suite', 155 | fullName: ['Outer suite', 'Inner suite', 'bar'], 156 | status: 'passed', 157 | runtime: 42, 158 | errors: [], 159 | assertions: [ 160 | { passed: true } 161 | ], 162 | tests: [] 163 | } 164 | ] 165 | } 166 | ] 167 | } 168 | ], 169 | status: 'passed', 170 | testCounts: { 171 | passed: 4, 172 | failed: 0, 173 | skipped: 0, 174 | todo: 0, 175 | total: 4 176 | }, 177 | runtime: 42 178 | }; 179 | 180 | QUnit.module('SummaryReporter', hooks => { 181 | let emitter, reporter; 182 | 183 | hooks.beforeEach(function () { 184 | emitter = new JsReporters.EventEmitter(); 185 | reporter = JsReporters.SummaryReporter.init(emitter); 186 | }); 187 | 188 | test('getSummary() with upward events', async assert => { 189 | playUpwardRun(emitter); 190 | assert.propEqual( 191 | await reporter.getSummary(), 192 | expectedSummary 193 | ); 194 | }); 195 | 196 | test('getSummary() with downward events', async assert => { 197 | playDownwardRun(emitter); 198 | assert.propEqual( 199 | await reporter.getSummary(), 200 | expectedSummary 201 | ); 202 | }); 203 | 204 | test('getSummary(callback)', async assert => { 205 | const done = assert.async(); 206 | playUpwardRun(emitter); 207 | reporter.getSummary((err, summary) => { 208 | assert.strictEqual(err, null); 209 | assert.propEqual(summary, expectedSummary); 210 | done(); 211 | }); 212 | }); 213 | }); 214 | -------------------------------------------------------------------------------- /lib/adapters/JasmineAdapter.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const helpers = require('../helpers.js'); 3 | 4 | /** 5 | * Known limitations: 6 | * 7 | * - Errors in afterAll are ignored. 8 | */ 9 | module.exports = class JasmineAdapter extends EventEmitter { 10 | constructor (jasmine) { 11 | super(); 12 | 13 | // NodeJS or browser 14 | this.env = jasmine.env || jasmine.getEnv(); 15 | 16 | this.suiteChildren = {}; 17 | this.suiteEnds = []; 18 | this.suiteStarts = {}; 19 | this.testStarts = {}; 20 | this.testEnds = {}; 21 | 22 | // See 23 | const reporter = { 24 | jasmineStarted: this.onJasmineStarted.bind(this), 25 | specDone: this.onSpecDone.bind(this), 26 | specStarted: this.onSpecStarted.bind(this), 27 | suiteStarted: this.onSuiteStarted.bind(this), 28 | suiteDone: this.onSuiteDone.bind(this), 29 | jasmineDone: this.onJasmineDone.bind(this) 30 | }; 31 | 32 | if (jasmine.addReporter) { 33 | // For Node.js, use the method from jasmine-npm 34 | jasmine.addReporter(reporter); 35 | } else { 36 | // For browser, use the method from jasmine-core 37 | this.env.addReporter(reporter); 38 | } 39 | } 40 | 41 | createAssertion (expectation) { 42 | return { 43 | passed: expectation.passed, 44 | actual: expectation.actual, 45 | expected: expectation.expected, 46 | message: expectation.message, 47 | stack: expectation.stack !== '' ? expectation.stack : null 48 | }; 49 | } 50 | 51 | createTestEnd (testStart, result) { 52 | const errors = result.failedExpectations.map((expectation) => this.createAssertion(expectation)); 53 | const assertions = errors.concat( 54 | result.passedExpectations.map((expectation) => this.createAssertion(expectation)) 55 | ); 56 | 57 | return { 58 | name: testStart.name, 59 | suiteName: testStart.suiteName, 60 | fullName: testStart.fullName.slice(), 61 | status: (result.status === 'pending') ? 'skipped' : result.status, 62 | // TODO: Jasmine 3.4+ has result.duration, use it. 63 | // Note that result.duration uses 0 instead of null for a 'skipped' test. 64 | runtime: (result.status === 'pending') ? null : (new Date() - this.startTime), 65 | errors, 66 | assertions 67 | }; 68 | } 69 | 70 | /** 71 | * Traverse the Jasmine structured returned by `this.env.topSuite()` 72 | * in order to extract the child-parent relations and full names. 73 | * 74 | */ 75 | processSuite (result, parentNames, parentIds) { 76 | const isGlobalSuite = (result.description === 'Jasmine__TopLevel__Suite'); 77 | 78 | const name = isGlobalSuite ? null : result.description; 79 | const fullName = parentNames.slice(); 80 | 81 | if (!isGlobalSuite) { 82 | fullName.push(name); 83 | } 84 | 85 | parentIds.push(result.id); 86 | this.suiteChildren[result.id] = []; 87 | 88 | result.children.forEach((child) => { 89 | if (child.id.indexOf('suite') === 0) { 90 | this.suiteStarts[child.id] = { 91 | name: child.description, 92 | fullName: [...fullName, child.description] 93 | }; 94 | this.processSuite(child, fullName.slice(), parentIds.slice()); 95 | } else { 96 | this.testStarts[child.id] = { 97 | name: child.description, 98 | suiteName: name, 99 | fullName: [...fullName, child.description] 100 | }; 101 | // Update flat list of test children 102 | parentIds.forEach((id) => { 103 | this.suiteChildren[id].push(child.id); 104 | }); 105 | } 106 | }); 107 | } 108 | 109 | createSuiteEnd (testStart, result) { 110 | const tests = this.suiteChildren[result.id].map((testId) => this.testEnds[testId]); 111 | 112 | const helperData = helpers.aggregateTests(tests); 113 | return { 114 | name: testStart.name, 115 | fullName: testStart.fullName, 116 | // Jasmine has result.status, but does not propagate 'todo' or 'skipped' 117 | status: helperData.status, 118 | runtime: result.duration || helperData.runtime 119 | }; 120 | } 121 | 122 | onJasmineStarted () { 123 | this.processSuite(this.env.topSuite(), [], []); 124 | 125 | let total = 0; 126 | this.env.topSuite().children.forEach(function countChild (child) { 127 | total++; 128 | if (child.id.indexOf('suite') === 0) { 129 | child.children.forEach(countChild); 130 | } 131 | }); 132 | 133 | this.emit('runStart', { 134 | name: null, 135 | testCounts: { 136 | total: total 137 | } 138 | }); 139 | } 140 | 141 | onSuiteStarted (result) { 142 | this.emit('suiteStart', this.suiteStarts[result.id]); 143 | } 144 | 145 | onSpecStarted (result) { 146 | this.startTime = new Date(); 147 | this.emit('testStart', this.testStarts[result.id]); 148 | } 149 | 150 | onSpecDone (result) { 151 | this.testEnds[result.id] = this.createTestEnd(this.testStarts[result.id], result); 152 | this.emit('testEnd', this.testEnds[result.id]); 153 | } 154 | 155 | onSuiteDone (result) { 156 | const suiteEnd = this.createSuiteEnd(this.suiteStarts[result.id], result); 157 | this.suiteEnds.push(suiteEnd); 158 | this.emit('suiteEnd', suiteEnd); 159 | } 160 | 161 | onJasmineDone (doneInfo) { 162 | const topSuite = this.env.topSuite(); 163 | const tests = this.suiteChildren[topSuite.id].map((testId) => this.testEnds[testId]); 164 | const helperData = helpers.aggregateTests([...tests, ...this.suiteEnds]); 165 | this.emit('runEnd', { 166 | name: null, 167 | status: helperData.status, 168 | testCounts: helperData.testCounts, 169 | runtime: helperData.runtime 170 | }); 171 | } 172 | }; 173 | -------------------------------------------------------------------------------- /lib/adapters/MochaAdapter.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const helpers = require('../helpers.js'); 3 | 4 | module.exports = class MochaAdapter extends EventEmitter { 5 | constructor (mocha) { 6 | super(); 7 | 8 | this.errors = null; 9 | this.finalRuntime = 0; 10 | this.finalCounts = { 11 | passed: 0, 12 | failed: 0, 13 | skipped: 0, 14 | todo: 0, 15 | total: 0 16 | }; 17 | 18 | // Mocha will instantiate the given function as a class, even if you only need a callback. 19 | // As such, it can't be an arrow function as those throw TypeError when instantiated. 20 | const self = this; 21 | mocha.reporter(function (runner) { 22 | self.runner = runner; 23 | 24 | runner.on('start', self.onStart.bind(self)); 25 | runner.on('suite', self.onSuite.bind(self)); 26 | runner.on('test', self.onTest.bind(self)); 27 | runner.on('pending', self.onPending.bind(self)); 28 | runner.on('fail', self.onFail.bind(self)); 29 | runner.on('test end', self.onTestEnd.bind(self)); 30 | runner.on('suite end', self.onSuiteEnd.bind(self)); 31 | runner.on('end', self.onEnd.bind(self)); 32 | }); 33 | } 34 | 35 | convertToSuiteStart (mochaSuite) { 36 | return { 37 | name: mochaSuite.title, 38 | fullName: this.titlePath(mochaSuite) 39 | }; 40 | } 41 | 42 | convertToSuiteEnd (mochaSuite) { 43 | const tests = mochaSuite.tests.map(this.convertTest.bind(this)); 44 | const childSuites = mochaSuite.suites.map(this.convertToSuiteEnd.bind(this)); 45 | const helperData = helpers.aggregateTests([...tests, ...childSuites]); 46 | 47 | return { 48 | name: mochaSuite.title, 49 | fullName: this.titlePath(mochaSuite), 50 | status: helperData.status, 51 | runtime: helperData.runtime 52 | }; 53 | } 54 | 55 | convertTest (mochaTest) { 56 | let suiteName; 57 | let fullName; 58 | if (!mochaTest.parent.root) { 59 | suiteName = mochaTest.parent.title; 60 | fullName = this.titlePath(mochaTest.parent); 61 | // Add also the test name. 62 | fullName.push(mochaTest.title); 63 | } else { 64 | suiteName = null; 65 | fullName = [mochaTest.title]; 66 | } 67 | 68 | if (mochaTest.errors !== undefined) { 69 | // If the test has the 'errors' property, this is a "test end". 70 | const errors = mochaTest.errors.map((error) => ({ 71 | passed: false, 72 | actual: error.actual, 73 | expected: error.expected, 74 | message: error.message || error.toString(), 75 | stack: error.stack 76 | })); 77 | // Mocha 8.0 introduced STATE_PENDING 78 | // https://github.com/qunitjs/js-reporters/issues/116 79 | const status = (mochaTest.state === undefined || mochaTest.state === 'pending') ? 'skipped' : mochaTest.state; 80 | const runtime = (mochaTest.duration === undefined) ? null : mochaTest.duration; 81 | 82 | return { 83 | name: mochaTest.title, 84 | suiteName, 85 | fullName, 86 | status, 87 | runtime, 88 | errors, 89 | assertions: errors 90 | }; 91 | } else { 92 | // It is a "test start". 93 | return { 94 | name: mochaTest.title, 95 | suiteName, 96 | fullName 97 | }; 98 | } 99 | } 100 | 101 | titlePath (mochaSuite) { 102 | if (mochaSuite.titlePath) { 103 | // Mocha 4.0+ has Suite#titlePath() 104 | return mochaSuite.titlePath(); 105 | } 106 | 107 | const fullName = []; 108 | if (!mochaSuite.root) { 109 | fullName.push(mochaSuite.title); 110 | } 111 | let parent = mochaSuite.parent; 112 | while (parent && !parent.root) { 113 | fullName.unshift(parent.title); 114 | parent = parent.parent; 115 | } 116 | return fullName; 117 | } 118 | 119 | onStart () { 120 | // total is all tests + all suites 121 | // each suite gets a CRI "test" wrapper 122 | let total = this.runner.suite.total(); 123 | this.runner.suite.suites.forEach(function addSuites (suite) { 124 | total++; 125 | suite.suites.forEach(addSuites); 126 | }); 127 | this.emit('runStart', { 128 | name: null, 129 | testCounts: { 130 | total: total 131 | } 132 | }); 133 | } 134 | 135 | onSuite (mochaSuite) { 136 | if (!mochaSuite.root) { 137 | this.emit('suiteStart', this.convertToSuiteStart(mochaSuite)); 138 | } 139 | } 140 | 141 | onTest (mochaTest) { 142 | this.errors = []; 143 | 144 | this.emit('testStart', this.convertTest(mochaTest)); 145 | } 146 | 147 | /** 148 | * Mocha emits skipped tests here instead of on the "test" event. 149 | */ 150 | onPending (mochaTest) { 151 | this.emit('testStart', this.convertTest(mochaTest)); 152 | } 153 | 154 | onFail (test, error) { 155 | this.errors.push(error); 156 | } 157 | 158 | onTestEnd (mochaTest) { 159 | // Save the errors on Mocha's test object, because when the suite that 160 | // contains this test is emitted on the "suiteEnd" event, it should also 161 | // contain this test with all its details (errors, status, runtime). Runtime 162 | // and status are already attached to the test, but the errors are not. 163 | mochaTest.errors = this.errors; 164 | 165 | const testEnd = this.convertTest(mochaTest); 166 | this.emit('testEnd', testEnd); 167 | this.finalCounts.total++; 168 | this.finalCounts[testEnd.status]++; 169 | this.finalRuntime += testEnd.runtime || 0; 170 | } 171 | 172 | onSuiteEnd (mochaSuite) { 173 | if (!mochaSuite.root) { 174 | const suiteEnd = this.convertToSuiteEnd(mochaSuite); 175 | this.emit('suiteEnd', suiteEnd); 176 | this.finalCounts.total++; 177 | this.finalCounts[suiteEnd.status]++; 178 | this.finalRuntime += suiteEnd.runtime || 0; 179 | } 180 | } 181 | 182 | onEnd (details) { 183 | this.emit('runEnd', { 184 | name: null, 185 | status: this.finalCounts.failed > 0 ? 'failed' : 'passed', 186 | testCounts: this.finalCounts, 187 | runtime: this.finalRuntime 188 | }); 189 | } 190 | }; 191 | -------------------------------------------------------------------------------- /test/unit/tap-reporter.js: -------------------------------------------------------------------------------- 1 | /* eslint-env qunit */ 2 | const { test } = QUnit; 3 | const kleur = require('kleur'); 4 | const sinon = require('sinon'); 5 | const JsReporters = require('../../index.js'); 6 | const data = require('../fixtures/unit.js'); 7 | 8 | QUnit.module('TapReporter', hooks => { 9 | let emitter, sandbox, spy; 10 | 11 | hooks.beforeEach(function () { 12 | emitter = new JsReporters.EventEmitter(); 13 | sandbox = sinon.sandbox.create(); 14 | spy = sandbox.stub(); 15 | // eslint-disable-next-line no-new 16 | new JsReporters.TapReporter(emitter, { 17 | log: spy 18 | }); 19 | }); 20 | 21 | hooks.afterEach(function () { 22 | sandbox.restore(); 23 | }); 24 | 25 | test('output the TAP header', assert => { 26 | emitter.emit('runStart', {}); 27 | 28 | assert.true(spy.calledOnce); 29 | }); 30 | 31 | test('output ok for a passing test', assert => { 32 | const expected = 'ok 1 pass'; 33 | 34 | emitter.emit('testEnd', data.passingTest); 35 | 36 | assert.true(spy.calledWith(expected)); 37 | }); 38 | 39 | test('output ok for a skipped test', assert => { 40 | const expected = kleur.yellow('ok 1 # SKIP skip'); 41 | 42 | emitter.emit('testEnd', data.skippedTest); 43 | 44 | assert.true(spy.calledWith(expected)); 45 | }); 46 | 47 | test('output not ok for a todo test', assert => { 48 | const expected = kleur.cyan('not ok 1 # TODO todo'); 49 | 50 | emitter.emit('testEnd', data.todoTest); 51 | 52 | assert.true(spy.calledWith(expected)); 53 | }); 54 | 55 | test('output not ok for a failing test', assert => { 56 | const expected = kleur.red('not ok 1 fail'); 57 | 58 | emitter.emit('testEnd', data.failingTest); 59 | 60 | assert.true(spy.calledWith(expected)); 61 | }); 62 | 63 | test('output all errors for a failing test', assert => { 64 | emitter.emit('testEnd', data.failingTest); 65 | for (let i = 0; i < data.failingTapData.length; i++) { 66 | assert.true(spy.calledWith(data.failingTapData[i])); 67 | } 68 | }); 69 | 70 | test('output actual assertion value of undefined', assert => { 71 | emitter.emit('testEnd', data.actualUndefinedTest); 72 | assert.true(spy.calledWithMatch(/^ {2}actual {2}: undefined$/m)); 73 | }); 74 | 75 | test('output actual assertion value of Infinity', assert => { 76 | emitter.emit('testEnd', data.actualInfinity); 77 | assert.true(spy.calledWithMatch(/^ {2}actual {2}: Infinity$/m)); 78 | }); 79 | 80 | test('output actual assertion value of "abc"', assert => { 81 | emitter.emit('testEnd', data.actualStringChar); 82 | // No redundant quotes 83 | assert.true(spy.calledWithMatch(/^ {2}actual {2}: abc$/m)); 84 | }); 85 | 86 | test('output actual assertion value of "abc\\n"', assert => { 87 | emitter.emit('testEnd', data.actualStringOneTailLn); 88 | assert.equal(spy.args[1][0], data.actualStringOneTailLnTap); 89 | }); 90 | 91 | test('output actual assertion value of "abc\\n\\n"', assert => { 92 | emitter.emit('testEnd', data.actualStringTwoTailLn); 93 | assert.equal(spy.args[1][0], data.actualStringTwoTailLnTap); 94 | }); 95 | 96 | test('output actual assertion value of "2"', assert => { 97 | emitter.emit('testEnd', data.actualStringNum); 98 | // Quotes required to disambiguate YAML value 99 | assert.true(spy.calledWithMatch(/^ {2}actual {2}: "2"$/m)); 100 | }); 101 | 102 | test('output actual assertion value of "true"', assert => { 103 | emitter.emit('testEnd', data.actualStringBool); 104 | // Quotes required to disambiguate YAML value 105 | assert.true(spy.calledWithMatch(/^ {2}actual {2}: "true"$/m)); 106 | }); 107 | 108 | test('output actual assertion value of 0', assert => { 109 | emitter.emit('testEnd', data.actualZero); 110 | assert.true(spy.calledWithMatch(/^ {2}actual {2}: 0$/m)); 111 | }); 112 | 113 | test('output actual assertion value of []', assert => { 114 | emitter.emit('testEnd', data.actualArray); 115 | assert.equal(spy.args[1][0], data.actualArrayTap); 116 | }); 117 | 118 | test('output actual assertion value of a cyclical structure', assert => { 119 | emitter.emit('testEnd', data.actualCyclical); 120 | assert.equal(spy.args[1][0], data.actualCyclicalTap); 121 | }); 122 | 123 | test('output actual assertion value of a subobject cyclical structure', assert => { 124 | emitter.emit('testEnd', data.actualSubobjectCyclical); 125 | assert.equal(spy.args[1][0], data.actualSubobjectCyclicalTap); 126 | }); 127 | 128 | test('output actual assertion value of an acyclical structure', assert => { 129 | emitter.emit('testEnd', data.actualDuplicateAcyclic); 130 | assert.equal(spy.args[1][0], data.actualDuplicateAcyclicTap); 131 | }); 132 | 133 | test('output expected assertion of undefined', assert => { 134 | emitter.emit('testEnd', data.expectedUndefinedTest); 135 | assert.true(spy.calledWithMatch(/^ {2}expected: undefined$/m)); 136 | }); 137 | 138 | test('output expected assertion of 0', assert => { 139 | emitter.emit('testEnd', data.expectedFalsyTest); 140 | assert.true(spy.calledWithMatch(/^ {2}expected: 0$/m)); 141 | }); 142 | 143 | test('output expected assertion of a circular structure', assert => { 144 | emitter.emit('testEnd', data.expectedCircularTest); 145 | assert.true(spy.calledWithMatch(/^ {2}expected: \{\n {2}"a": "example",\n {2}"cycle": "\[Circular\]"\n\}$/m)); 146 | }); 147 | 148 | test('output the total number of tests', assert => { 149 | const summary = '1..6'; 150 | const passCount = '# pass 3'; 151 | const skipCount = kleur.yellow('# skip 1'); 152 | const todoCount = kleur.cyan('# todo 0'); 153 | const failCount = kleur.red('# fail 2'); 154 | 155 | emitter.emit('runEnd', { 156 | testCounts: { 157 | total: 6, 158 | passed: 3, 159 | failed: 2, 160 | skipped: 1, 161 | todo: 0 162 | } 163 | }); 164 | 165 | assert.true(spy.calledWith(summary)); 166 | assert.true(spy.calledWith(passCount)); 167 | assert.true(spy.calledWith(skipCount)); 168 | assert.true(spy.calledWith(todoCount)); 169 | assert.true(spy.calledWith(failCount)); 170 | }); 171 | }); 172 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # js-reporters 2 | 3 | [![Chat on Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/js-reporters/js-reporters) 4 | [![Build status](https://github.com/qunitjs/js-reporters/actions/workflows/CI.yaml/badge.svg)](https://github.com/qunitjs/js-reporters/actions/workflows/CI.yaml) 5 | [![](https://img.shields.io/npm/dm/js-reporters.svg)](https://www.npmjs.com/package/js-reporters) 6 | [![npm package](https://img.shields.io/npm/v/js-reporters.svg)](https://www.npmjs.com/package/js-reporters) 7 | 8 | The Common Reporter Interface (CRI) for JavaScript Testing Frameworks. 9 | 10 | | Avoid this: | Do this: | 11 | |----------------------------|----------------------------------| 12 | | ![](img/situation-now.png) | ![](img/situation-expected.png) | 13 | 14 | ## Specification 15 | 16 | **See [Common Reporter Interface](spec/cri-draft.adoc) for the latest version of the specification.** 17 | 18 | See also: 19 | 20 | * [example](docs/example.md), illustrates how reporting works in practice. 21 | * [frameworks](docs/frameworks.md), studies various popular testing frameworks. 22 | * **[Integrations](#integrations)**, a list of real-world implementations. 23 | 24 | Help with AsciiDoc (used for the standard document): 25 | 26 | * [AsciiDoc Syntax Quick Reference](https://asciidoctor.org/docs/asciidoc-syntax-quick-reference/) 27 | * [AsciiDoc User Manual](https://asciidoctor.org/docs/user-manual/) 28 | * [AsciiDoc cheatsheet](https://powerman.name/doc/asciidoc) 29 | 30 | ## Background 31 | 32 | In 2014, the [QUnit](https://qunitjs.com/) team started [discussing](https://github.com/qunitjs/qunit/issues/531) the possibility of interoperability between JavaScript testing frameworks such as QUnit, Mocha, Jasmine, Intern, Buster, etc. The "Common Reporter Interface" would be an allow integrations for output formats and communication bridges to be shared between frameworks. This would also benefit high-level consumers of these frameworks such as Karma, BrowserStack, SauceLabs, Testling, by having a standard machine-readable interface. 33 | 34 | Our mission is to deliver: 35 | 36 | - a common JavaScript API, e.g. based on EventEmitter featuring `.on()` and `.off()`. 37 | - a minimum viable set of events with standardized event names and event data. 38 | - a minimum viable set of concepts behind those events to allow consumers to set expectations (e.g. define what "pass", "fail", "skip", "todo", and "pending" mean). 39 | - work with participating testing frameworks to support the Common Reporter Interface. 40 | 41 | Would _you_ be interested in discussing this with us further? Please join in! 42 | 43 | * [Join the Chat room](https://gitter.im/js-reporters/js-reporters) 44 | * [Browse open issues](https://github.com/qunitjs/js-reporters/issues/) 45 | * [Help frameworks and runners implement the spec](#cross-project-coordination) 46 | 47 | ## js-reporters Package 48 | 49 | ### Usage 50 | 51 | Listen to the events and receive the emitted data: 52 | 53 | ```js 54 | // Use automatic discovery of the framework adapter. 55 | const runner = JsReporters.autoRegister(); 56 | 57 | // Listen to standard events, from any testing framework. 58 | runner.on('testEnd', (test) => { 59 | console.log('Test %s has errors:', test.fullName.join(' '), test.errors); 60 | }); 61 | 62 | runner.on('runEnd', (run) => { 63 | const counts = run.testCounts; 64 | 65 | console.log('Testsuite status: %s', run.status); 66 | console.log('Total %d tests: %d passed, %d failed, %d skipped', 67 | counts.total, 68 | counts.passed, 69 | counts.failed, 70 | counts.skipped 71 | ); 72 | console.log('Total duration: %d', run.runtime); 73 | }); 74 | 75 | // Or use one of the built-in reporters. 76 | JsReporters.TapReporter.init(runner); 77 | ``` 78 | 79 | ### Runtime support 80 | 81 | * Internet Explorer 9+ 82 | * Edge 15+ (Legacy) 83 | * Edge 80+ (Chromium-based) 84 | * Safari 9+ 85 | * Firefox 45+ 86 | * Chrome 58+ 87 | * Node.js 10+ 88 | 89 | ### Adapter support 90 | 91 | | Testing framework | Supported | Last checked | Unresolved 92 | |--|--|--|-- 93 | | QUnit | 1.20+ | ✅ `qunit@2.14.1` (Apr 2021) | – 94 | | Jasmine | 2.1+ | ✅ `jasmine@3.7.0` (Apr 2021) | – 95 | | Mocha | 1.18+ | ✅ `mocha@8.3.2` (Apr 2021) | – 96 | 97 | See also [past issues](test/versions/failing-versions.js). 98 | 99 | ### API 100 | 101 | **autoRegister()** 102 | 103 | Automatically detects which testing framework you use and attaches any adapters as needed, and returns a compatible runner object. If no framework is found, it will throw an `Error`. 104 | 105 | ```js 106 | JsReporters.autoRegister(); 107 | ``` 108 | 109 | ## Integrations 110 | 111 | Runners: 112 | 113 | * [QUnit](https://qunitjs.com/), natively since [QUnit 2.2](https://github.com/qunitjs/qunit/releases/2.2.0). 114 | * Jasmine, via [js-reporters JasmineAdapter](lib/adapters/JasmineAdapter.js). 115 | * Mocha, via [js-reporters MochaAdapter](lib/adapters/MochaAdapter.js). 116 | 117 | Reporters: 118 | 119 | * [TAP](lib/reporters/TapReporter), implements the [Test Anything Protocol](https://testanything.org/) for command-line output. 120 | * [browserstack-runner](https://github.com/browserstack/browserstack-runner/blob/0.9.1/lib/_patch/reporter.js), runs JavaScript unit tests remotely in multiple browsers, summarize the results by browser, and fail or pass the continuous integration build accordingly. 121 | * _Add your own, and let us know!_ 122 | 123 | ## Cross-project coordination 124 | 125 | Testing frameworks: 126 | 127 | * [QUnit issue](https://github.com/qunitjs/qunit/issues/531) (Done!) 128 | * [Mocha issue](https://github.com/visionmedia/mocha/issues/1326) (pending…) 129 | * [Jasmine issue](https://github.com/pivotal/jasmine/issues/659) (pending…) 130 | * [Intern issue](https://github.com/theintern/intern/issues/257) (pending…) 131 | * [Vows issue](https://github.com/flatiron/vows/issues/313) (pending…) 132 | * [Buster issue](https://github.com/busterjs/buster/issues/419) (Discontinued.) 133 | * [Nodeunit issue](https://github.com/caolan/nodeunit/issues/276) (Discontinued.) 134 | 135 | Reporters and proxy layers: 136 | 137 | * [BrowserStack](https://github.com/browserstack/browserstack-runner/issues/92) (Done!) 138 | * [Karma](https://github.com/karma-runner/karma/issues/1183) (pending…) 139 | * [grunt-saucelabs](https://github.com/axemclion/grunt-saucelabs/issues/164) (pending…) 140 | * [Testling](https://github.com/substack/testling/issues/93) (pending…) 141 | 142 | ## Credits 143 | 144 | [![Testing Powered By SauceLabs](https://opensource.saucelabs.com/images/opensauce/powered-by-saucelabs-badge-gray.png?sanitize=true "Testing Powered By SauceLabs")](https://saucelabs.com) 145 | -------------------------------------------------------------------------------- /lib/adapters/QUnitAdapter.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const helpers = require('../helpers.js'); 3 | 4 | /** 5 | * Known limitations: 6 | * 7 | * - Due to ordering issues with nested modules on QUnit < 2.2, this adapter 8 | * buffers events and only emits them after the run has completed. 9 | * See 10 | */ 11 | module.exports = class QUnitAdapter extends EventEmitter { 12 | constructor (QUnit) { 13 | super(); 14 | 15 | this.QUnit = QUnit; 16 | this.testEnds = {}; 17 | this.moduleEnds = []; 18 | this.delim = ' > '; 19 | this.totalBegin = null; 20 | 21 | // Ordered lists 22 | this.globalTests = null; 23 | this.globalModules = null; 24 | 25 | QUnit.begin(this.onBegin.bind(this)); 26 | QUnit.log(this.onLog.bind(this)); 27 | QUnit.testDone(this.onTestDone.bind(this)); 28 | QUnit.done(this.onDone.bind(this)); 29 | } 30 | 31 | prepTestEnd (suiteName, parentNames, details) { 32 | const testEnd = this.testEnds[details.testId] = { 33 | name: details.name, 34 | suiteName: suiteName, 35 | fullName: [...parentNames, details.name], 36 | // Placeholders, populated by onTestDone() and onLog() 37 | status: null, 38 | runtime: null, 39 | errors: [], 40 | assertions: [] 41 | }; 42 | return testEnd; 43 | } 44 | 45 | processModule (qunitModule) { 46 | const fullName = qunitModule.name.split(this.delim); 47 | const name = fullName.slice(-1)[0]; 48 | 49 | const childTests = qunitModule.tests.map((details) => { 50 | return this.prepTestEnd(name, fullName, details); 51 | }); 52 | 53 | return { 54 | suiteEnd: { 55 | name, 56 | fullName, 57 | // Placeholders, populated by emitTests() 58 | status: null, 59 | runtime: null 60 | }, 61 | childTests, 62 | childModules: [] 63 | }; 64 | } 65 | 66 | processEverything () { 67 | let modules; 68 | 69 | // Access QUnit internals to get all modules and tests, 70 | // working around missing event data. 71 | 72 | // First, find any global tests. 73 | if (this.QUnit.config.modules.length > 0 && 74 | this.QUnit.config.modules[0].name === '') { 75 | this.globalTests = this.QUnit.config.modules[0].tests.map((details) => this.prepTestEnd(null, [], details)); 76 | modules = this.QUnit.config.modules.slice(1); 77 | } else { 78 | this.globalTests = []; 79 | modules = this.QUnit.config.modules; 80 | } 81 | 82 | // Prepare all suiteEnd leafs 83 | modules = modules.map(this.processModule.bind(this)); 84 | 85 | // For CRI, each module will be represented as a wrapper test 86 | this.totalBegin = Object.keys(this.testEnds).length + modules.length; 87 | 88 | // If a module has a composed name, its name will be the last part of the full name 89 | // and its parent name will be the one right before it. Search for the parent 90 | // module and add the current module to it as a child, among the test leafs. 91 | const globalModules = []; 92 | modules.forEach((mod) => { 93 | if (mod.suiteEnd.fullName.length > 1) { 94 | const parentFullName = mod.suiteEnd.fullName.slice(0, -1); 95 | modules.forEach((otherMod) => { 96 | if (otherMod.suiteEnd.fullName.join(this.delim) === parentFullName.join(this.delim)) { 97 | otherMod.childModules.push(mod); 98 | } 99 | }); 100 | } else { 101 | globalModules.push(mod); 102 | } 103 | }); 104 | this.globalModules = globalModules; 105 | } 106 | 107 | emitTests () { 108 | this.globalTests.forEach((testEnd) => { 109 | this.emit('testStart', helpers.createTestStart(testEnd)); 110 | this.emit('testEnd', testEnd); 111 | }); 112 | 113 | const emitModule = (mod) => { 114 | this.emit('suiteStart', { 115 | name: mod.suiteEnd.name, 116 | fullName: mod.suiteEnd.fullName.slice() 117 | }); 118 | 119 | mod.childTests.forEach((testEnd) => { 120 | this.emit('testStart', helpers.createTestStart(testEnd)); 121 | this.emit('testEnd', testEnd); 122 | }); 123 | mod.childModules.forEach(child => emitModule(child)); 124 | 125 | // This is non-recursive and can be because we emit modules in the original 126 | // depth-first execution order. We fill in the status/runtime placeholders 127 | // for the suiteEnd object of a nested module, and then later a parent module 128 | // follows and sees that child suiteEnd object by reference and can propagate 129 | // and aggregate the information further. 130 | const helperData = helpers.aggregateTests([ 131 | ...mod.childTests, 132 | ...mod.childModules.map(child => child.suiteEnd) 133 | ]); 134 | mod.suiteEnd.status = helperData.status; 135 | mod.suiteEnd.runtime = helperData.runtime; 136 | 137 | this.moduleEnds.push(mod.suiteEnd); 138 | this.emit('suiteEnd', mod.suiteEnd); 139 | }; 140 | this.globalModules.forEach(emitModule); 141 | } 142 | 143 | onBegin (details) { 144 | this.processEverything(); 145 | 146 | this.emit('runStart', { 147 | name: null, 148 | testCounts: { 149 | total: this.totalBegin 150 | } 151 | }); 152 | } 153 | 154 | onLog (details) { 155 | const assertion = { 156 | passed: details.result, 157 | actual: details.actual, 158 | expected: details.expected, 159 | message: details.message, 160 | stack: details.source || null 161 | }; 162 | if (this.testEnds[details.testId]) { 163 | if (!details.result) { 164 | this.testEnds[details.testId].errors.push(assertion); 165 | } 166 | this.testEnds[details.testId].assertions.push(assertion); 167 | } 168 | } 169 | 170 | onTestDone (details) { 171 | const testEnd = this.testEnds[details.testId]; 172 | 173 | // Handle todo status 174 | const netFailure = (details.failed > 0 && !details.todo) || (details.failed === 0 && details.todo); 175 | 176 | if (details.skipped) { 177 | testEnd.status = 'skipped'; 178 | } else if (netFailure) { 179 | testEnd.status = 'failed'; 180 | } else if (details.todo) { 181 | testEnd.status = 'todo'; 182 | } else { 183 | testEnd.status = 'passed'; 184 | } 185 | 186 | // QUnit uses 0 instead of null for runtime of skipped tests. 187 | if (!details.skipped) { 188 | testEnd.runtime = details.runtime; 189 | } else { 190 | testEnd.runtime = null; 191 | } 192 | } 193 | 194 | onDone (details) { 195 | this.emitTests(); 196 | 197 | const allTests = [...this.moduleEnds]; 198 | for (const testId in this.testEnds) { 199 | allTests.push(this.testEnds[testId]); 200 | } 201 | const helperData = helpers.aggregateTests(allTests); 202 | 203 | this.emit('runEnd', { 204 | name: null, 205 | status: helperData.status, 206 | testCounts: helperData.testCounts, 207 | runtime: details.runtime || null 208 | }); 209 | } 210 | }; 211 | -------------------------------------------------------------------------------- /test/integration/adapters.js: -------------------------------------------------------------------------------- 1 | /* eslint-env qunit */ 2 | /* eslint-disable no-unused-expressions */ 3 | 4 | const { test } = QUnit; 5 | const runAdapters = require('./adapters-run.js'); 6 | 7 | function rerequire (file) { 8 | const resolved = require.resolve(file); 9 | delete require.cache[resolved]; 10 | return require(resolved); 11 | } 12 | 13 | /** 14 | * Collect data from an (adapted) runner. 15 | */ 16 | function collectDataFromRunner (collectedData, done, runner) { 17 | runner.on('runStart', (runStart) => { 18 | collectedData.push(['runStart', runStart]); 19 | }); 20 | runner.on('suiteStart', (suiteStart) => { 21 | collectedData.push(['suiteStart', suiteStart]); 22 | }); 23 | runner.on('suiteEnd', (suiteEnd) => { 24 | normalizeSuiteEnd(suiteEnd); 25 | collectedData.push(['suiteEnd', suiteEnd]); 26 | }); 27 | runner.on('testStart', (testStart) => { 28 | collectedData.push(['testStart', testStart]); 29 | }); 30 | runner.on('testEnd', (testEnd) => { 31 | normalizeTestEnd(testEnd); 32 | collectedData.push(['testEnd', testEnd]); 33 | }); 34 | runner.on('runEnd', (runEnd) => { 35 | normalizeRunEnd(runEnd); 36 | collectedData.push(['runEnd', runEnd]); 37 | 38 | // Notify the integration test to continue, and validate the collected data. 39 | done(); 40 | }); 41 | } 42 | 43 | function normalizeTestEnd (test) { 44 | // Replace any actual assertion runtime with hardcoded 42s. 45 | // Preserve absence or other weird values as-is. 46 | if (Number.isFinite(test.runtime)) { 47 | test.runtime = 42; 48 | } 49 | 50 | // Only check the "passed" property. 51 | // Throw away the rest of the actual assertion objects as being framework-specific. 52 | if (test.assertions) { 53 | test.assertions.forEach(assertion => { 54 | Object.keys(assertion).forEach(key => { 55 | if (key !== 'passed') delete assertion[key]; 56 | }); 57 | }); 58 | } 59 | if (test.errors) { 60 | test.errors.forEach(assertion => { 61 | Object.keys(assertion).forEach(key => { 62 | if (key !== 'passed') delete assertion[key]; 63 | }); 64 | }); 65 | } 66 | } 67 | 68 | function normalizeSuiteEnd (suiteEnd) { 69 | if (Number.isFinite(suiteEnd.runtime)) { 70 | suiteEnd.runtime = 42; 71 | } 72 | } 73 | 74 | function normalizeRunEnd (runEnd) { 75 | if (Number.isFinite(runEnd.runtime)) { 76 | runEnd.runtime = 42; 77 | } 78 | } 79 | 80 | function fixExpectedData (adapter, expectedData) { 81 | expectedData.forEach(([eventName, data]) => { 82 | if (eventName === 'testEnd') { 83 | // Don't expect passed assertion for testing frameworks 84 | // that don't record all assertions. 85 | if (adapter === 'Mocha' && data.status === 'passed') { 86 | data.assertions = []; 87 | } 88 | } 89 | if (eventName === 'testEnd' || eventName === 'suiteEnd' || eventName === 'runEnd') { 90 | if (Number.isFinite(data.runtime)) { 91 | data.runtime = 42; 92 | } 93 | } 94 | }); 95 | } 96 | 97 | const integrations = [ 98 | { key: 'main', name: 'Adapters main integration', referenceFile: './reference-data.js' }, 99 | 100 | // This is only implemented by QUnit currently 101 | { key: 'todo', name: 'Adapters todo integration', referenceFile: './reference-data-todo.js' } 102 | ]; 103 | 104 | integrations.forEach(function (integration) { 105 | QUnit.module(integration.name, function () { 106 | Object.keys(runAdapters[integration.key]).forEach(function (adapter) { 107 | QUnit.module(adapter + ' adapter', hooks => { 108 | // Re-require for each adapter because we mutate the expected data. 109 | const expectedData = rerequire(integration.referenceFile); 110 | fixExpectedData(adapter, expectedData); 111 | 112 | const collectedData = []; 113 | 114 | hooks.before(assert => { 115 | const done = assert.async(); 116 | runAdapters[integration.key][adapter]( 117 | collectDataFromRunner.bind(null, collectedData, done) 118 | ); 119 | }); 120 | 121 | // Fist check that, overall, all expected events were emitted and in order. 122 | test('Emitted events names', assert => { 123 | assert.propEqual( 124 | collectedData.map(pair => pair[0]), 125 | expectedData.map(pair => pair[0]), 126 | 'Event names' 127 | ); 128 | }); 129 | 130 | test('Event "testStart" data', assert => { 131 | const actuals = collectedData.filter(pair => pair[0] === 'testStart'); 132 | const expecteds = expectedData.filter(pair => pair[0] === 'testStart'); 133 | assert.propEqual( 134 | actuals.map(expected => expected[1].name), 135 | expecteds.map(pair => pair[1].name), 136 | 'Test names' 137 | ); 138 | expecteds.forEach((expected, i) => { 139 | assert.propEqual( 140 | actuals[i][1], 141 | expected[1], 142 | `Event data for testStart#${i}` 143 | ); 144 | }); 145 | }); 146 | 147 | test('Event "testEnd" data', assert => { 148 | const actuals = collectedData.filter(pair => pair[0] === 'testEnd'); 149 | const expecteds = expectedData.filter(pair => pair[0] === 'testEnd'); 150 | assert.propEqual( 151 | actuals.map(expected => expected[1].name), 152 | expecteds.map(pair => pair[1].name), 153 | 'Test names' 154 | ); 155 | expecteds.forEach((expected, i) => { 156 | assert.propEqual( 157 | actuals[i][1], 158 | expected[1], 159 | `Event data for testEnd#${i}` 160 | ); 161 | }); 162 | }); 163 | 164 | test('Event "suiteStart" data', assert => { 165 | const actuals = collectedData.filter(pair => pair[0] === 'suiteStart'); 166 | const expecteds = expectedData.filter(pair => pair[0] === 'suiteStart'); 167 | assert.propEqual( 168 | actuals.map(expected => expected[1].name), 169 | expecteds.map(pair => pair[1].name), 170 | 'Suite names' 171 | ); 172 | expecteds.forEach((expected, i) => { 173 | assert.propEqual( 174 | actuals[i][1], 175 | expected[1], 176 | `Event data for suiteStart#${i}` 177 | ); 178 | }); 179 | }); 180 | 181 | test('Event "suiteEnd" data', assert => { 182 | const actuals = collectedData.filter(pair => pair[0] === 'suiteEnd'); 183 | const expecteds = expectedData.filter(pair => pair[0] === 'suiteEnd'); 184 | assert.propEqual( 185 | actuals.map(expected => expected[1].name), 186 | expecteds.map(pair => pair[1].name), 187 | 'Suite names' 188 | ); 189 | expecteds.forEach((expected, i) => { 190 | assert.propEqual( 191 | actuals[i][1], 192 | expected[1], 193 | `Event data for suiteEnd#${i}` 194 | ); 195 | }); 196 | }); 197 | 198 | test('Event "runStart" data', assert => { 199 | const actuals = collectedData.filter(pair => pair[0] === 'runStart'); 200 | expectedData.filter(pair => pair[0] === 'runStart').forEach((expected, i) => { 201 | assert.propEqual( 202 | actuals[i][1], 203 | expected[1], 204 | `Event data for runStart#${i}` 205 | ); 206 | }); 207 | }); 208 | 209 | test('Event "runEnd" data', assert => { 210 | const actuals = collectedData.filter(pair => pair[0] === 'runEnd'); 211 | expectedData.filter(pair => pair[0] === 'runEnd').forEach((expected, i) => { 212 | assert.propEqual( 213 | actuals[i][1], 214 | expected[1], 215 | `Event data for runEnd#${i}` 216 | ); 217 | }); 218 | }); 219 | }); 220 | }); 221 | }); 222 | }); 223 | -------------------------------------------------------------------------------- /test/integration/reference-data.js: -------------------------------------------------------------------------------- 1 | const failedAssertion = { 2 | passed: false 3 | }; 4 | 5 | const passedAssertion = { 6 | passed: true 7 | }; 8 | 9 | const globalTestStart = { 10 | name: 'global test', 11 | suiteName: null, 12 | fullName: [ 13 | 'global test' 14 | ] 15 | }; 16 | const globalTestEnd = { 17 | name: 'global test', 18 | suiteName: null, 19 | fullName: [ 20 | 'global test' 21 | ], 22 | status: 'passed', 23 | runtime: 42, 24 | errors: [], 25 | assertions: [ 26 | passedAssertion 27 | ] 28 | }; 29 | 30 | const passingTestStart1 = { 31 | name: 'should pass', 32 | suiteName: 'suite with passing test', 33 | fullName: [ 34 | 'suite with passing test', 35 | 'should pass' 36 | ] 37 | }; 38 | const passingSuiteStart = { 39 | name: 'suite with passing test', 40 | fullName: [ 41 | 'suite with passing test' 42 | ] 43 | }; 44 | const passingTestEnd1 = { 45 | name: 'should pass', 46 | suiteName: 'suite with passing test', 47 | fullName: [ 48 | 'suite with passing test', 49 | 'should pass' 50 | ], 51 | status: 'passed', 52 | runtime: 42, 53 | errors: [], 54 | assertions: [ 55 | passedAssertion 56 | ] 57 | }; 58 | const passingSuiteEnd = { 59 | name: 'suite with passing test', 60 | fullName: [ 61 | 'suite with passing test' 62 | ], 63 | status: 'passed', 64 | runtime: 42 65 | }; 66 | 67 | const skippedTestStart1 = { 68 | name: 'should skip', 69 | suiteName: 'suite with skipped test', 70 | fullName: [ 71 | 'suite with skipped test', 72 | 'should skip' 73 | ] 74 | }; 75 | const skippedSuiteStart = { 76 | name: 'suite with skipped test', 77 | fullName: [ 78 | 'suite with skipped test' 79 | ] 80 | }; 81 | const skippedTestEnd1 = { 82 | name: 'should skip', 83 | suiteName: 'suite with skipped test', 84 | fullName: [ 85 | 'suite with skipped test', 86 | 'should skip' 87 | ], 88 | status: 'skipped', 89 | runtime: null, 90 | errors: [], 91 | assertions: [] 92 | }; 93 | const skippedSuiteEnd = { 94 | name: 'suite with skipped test', 95 | fullName: [ 96 | 'suite with skipped test' 97 | ], 98 | status: 'passed', 99 | runtime: 0 100 | }; 101 | 102 | const failingTestStart1 = { 103 | name: 'should fail', 104 | suiteName: 'suite with failing test', 105 | fullName: [ 106 | 'suite with failing test', 107 | 'should fail' 108 | ] 109 | }; 110 | const failingSuiteStart = { 111 | name: 'suite with failing test', 112 | fullName: [ 113 | 'suite with failing test' 114 | ] 115 | }; 116 | const failingTestEnd1 = { 117 | name: 'should fail', 118 | suiteName: 'suite with failing test', 119 | fullName: [ 120 | 'suite with failing test', 121 | 'should fail' 122 | ], 123 | status: 'failed', 124 | runtime: 42, 125 | errors: [ 126 | failedAssertion 127 | ], 128 | assertions: [ 129 | failedAssertion 130 | ] 131 | }; 132 | const failingSuiteEnd = { 133 | name: 'suite with failing test', 134 | fullName: [ 135 | 'suite with failing test' 136 | ], 137 | status: 'failed', 138 | runtime: 42 139 | }; 140 | 141 | const passingTestStart2 = { 142 | name: 'should pass', 143 | suiteName: 'suite with tests', 144 | fullName: [ 145 | 'suite with tests', 146 | 'should pass' 147 | ] 148 | }; 149 | const passingTestEnd2 = { 150 | name: 'should pass', 151 | suiteName: 'suite with tests', 152 | fullName: [ 153 | 'suite with tests', 154 | 'should pass' 155 | ], 156 | status: 'passed', 157 | runtime: 42, 158 | errors: [], 159 | assertions: [ 160 | passedAssertion 161 | ] 162 | }; 163 | const skippedTestStart2 = { 164 | name: 'should skip', 165 | suiteName: 'suite with tests', 166 | fullName: [ 167 | 'suite with tests', 168 | 'should skip' 169 | ] 170 | }; 171 | const failingTestStart2 = { 172 | name: 'should fail', 173 | suiteName: 'suite with tests', 174 | fullName: [ 175 | 'suite with tests', 176 | 'should fail' 177 | ] 178 | }; 179 | const testSuiteStart = { 180 | name: 'suite with tests', 181 | fullName: [ 182 | 'suite with tests' 183 | ] 184 | }; 185 | const skippedTestEnd2 = { 186 | name: 'should skip', 187 | suiteName: 'suite with tests', 188 | fullName: [ 189 | 'suite with tests', 190 | 'should skip' 191 | ], 192 | status: 'skipped', 193 | runtime: null, 194 | errors: [], 195 | assertions: [] 196 | }; 197 | const failingTestEnd2 = { 198 | name: 'should fail', 199 | suiteName: 'suite with tests', 200 | fullName: [ 201 | 'suite with tests', 202 | 'should fail' 203 | ], 204 | status: 'failed', 205 | runtime: 42, 206 | errors: [ 207 | failedAssertion 208 | ], 209 | assertions: [ 210 | failedAssertion 211 | ] 212 | }; 213 | const testSuiteEnd = { 214 | name: 'suite with tests', 215 | fullName: [ 216 | 'suite with tests' 217 | ], 218 | status: 'failed', 219 | runtime: 84 220 | }; 221 | 222 | const outerTestStart = { 223 | name: 'outer test', 224 | suiteName: 'outer suite', 225 | fullName: [ 226 | 'outer suite', 227 | 'outer test' 228 | ] 229 | }; 230 | const outerTestEnd = { 231 | name: 'outer test', 232 | suiteName: 'outer suite', 233 | fullName: [ 234 | 'outer suite', 235 | 'outer test' 236 | ], 237 | status: 'passed', 238 | runtime: 42, 239 | errors: [], 240 | assertions: [ 241 | passedAssertion 242 | ] 243 | }; 244 | const innerTestStart = { 245 | name: 'inner test', 246 | suiteName: 'inner suite', 247 | fullName: [ 248 | 'outer suite', 249 | 'inner suite', 250 | 'inner test' 251 | ] 252 | }; 253 | const innerSuiteStart = { 254 | name: 'inner suite', 255 | fullName: [ 256 | 'outer suite', 257 | 'inner suite' 258 | ] 259 | }; 260 | const innerTestEnd = { 261 | name: 'inner test', 262 | suiteName: 'inner suite', 263 | fullName: [ 264 | 'outer suite', 265 | 'inner suite', 266 | 'inner test' 267 | ], 268 | status: 'passed', 269 | runtime: 42, 270 | errors: [], 271 | assertions: [ 272 | passedAssertion 273 | ] 274 | }; 275 | const innerSuiteEnd = { 276 | name: 'inner suite', 277 | fullName: [ 278 | 'outer suite', 279 | 'inner suite' 280 | ], 281 | status: 'passed', 282 | runtime: 42 283 | }; 284 | 285 | const outerSuiteStart = { 286 | name: 'outer suite', 287 | fullName: [ 288 | 'outer suite' 289 | ] 290 | }; 291 | const outerSuiteEnd = { 292 | name: 'outer suite', 293 | fullName: [ 294 | 'outer suite' 295 | ], 296 | status: 'passed', 297 | runtime: 84 298 | }; 299 | 300 | const runStart = { 301 | name: null, 302 | testCounts: { 303 | total: 15 304 | } 305 | }; 306 | const runEnd = { 307 | name: null, 308 | status: 'failed', 309 | testCounts: { 310 | passed: 9, 311 | failed: 4, 312 | skipped: 2, 313 | todo: 0, 314 | total: 15 315 | }, 316 | runtime: 294 317 | }; 318 | 319 | module.exports = [ 320 | ['runStart', runStart], 321 | 322 | ['testStart', globalTestStart], 323 | ['testEnd', globalTestEnd], 324 | 325 | ['suiteStart', passingSuiteStart], 326 | ['testStart', passingTestStart1], 327 | ['testEnd', passingTestEnd1], 328 | ['suiteEnd', passingSuiteEnd], 329 | 330 | ['suiteStart', skippedSuiteStart], 331 | ['testStart', skippedTestStart1], 332 | ['testEnd', skippedTestEnd1], 333 | ['suiteEnd', skippedSuiteEnd], 334 | 335 | ['suiteStart', failingSuiteStart], 336 | ['testStart', failingTestStart1], 337 | ['testEnd', failingTestEnd1], 338 | ['suiteEnd', failingSuiteEnd], 339 | 340 | ['suiteStart', testSuiteStart], 341 | ['testStart', passingTestStart2], 342 | ['testEnd', passingTestEnd2], 343 | ['testStart', skippedTestStart2], 344 | ['testEnd', skippedTestEnd2], 345 | ['testStart', failingTestStart2], 346 | ['testEnd', failingTestEnd2], 347 | ['suiteEnd', testSuiteEnd], 348 | 349 | ['suiteStart', outerSuiteStart], 350 | ['testStart', outerTestStart], 351 | ['testEnd', outerTestEnd], 352 | ['suiteStart', innerSuiteStart], 353 | ['testStart', innerTestStart], 354 | ['testEnd', innerTestEnd], 355 | ['suiteEnd', innerSuiteEnd], 356 | ['suiteEnd', outerSuiteEnd], 357 | 358 | ['runEnd', runEnd] 359 | ]; 360 | -------------------------------------------------------------------------------- /lib/reporters/TapReporter.js: -------------------------------------------------------------------------------- 1 | const kleur = require('kleur'); 2 | const hasOwn = Object.hasOwnProperty; 3 | 4 | /** 5 | * Format a given value into YAML. 6 | * 7 | * YAML is a superset of JSON that supports all the same data 8 | * types and syntax, and more. As such, it is always possible 9 | * to fallback to JSON.stringfify, but we generally avoid 10 | * that to make output easier to read for humans. 11 | * 12 | * Supported data types: 13 | * 14 | * - null 15 | * - boolean 16 | * - number 17 | * - string 18 | * - array 19 | * - object 20 | * 21 | * Anything else (including NaN, Infinity, and undefined) 22 | * must be described in strings, for display purposes. 23 | * 24 | * Note that quotes are optional in YAML strings if the 25 | * strings are "simple", and as such we generally prefer 26 | * that for improved readability. We output strings in 27 | * one of three ways: 28 | * 29 | * - bare unquoted text, for simple one-line strings. 30 | * - JSON (quoted text), for complex one-line strings. 31 | * - YAML Block, for complex multi-line strings. 32 | * 33 | * Objects with cyclical references will be stringifed as 34 | * "[Circular]" as they cannot otherwise be represented. 35 | */ 36 | function prettyYamlValue (value, indent = 4) { 37 | if (value === undefined) { 38 | // Not supported in JSON/YAML, turn into string 39 | // and let the below output it as bare string. 40 | value = String(value); 41 | } 42 | 43 | // Support IE 9-11: Use isFinite instead of ES6 Number.isFinite 44 | if (typeof value === 'number' && !isFinite(value)) { 45 | // Turn NaN and Infinity into simple strings. 46 | // Paranoia: Don't return directly just in case there's 47 | // a way to add special characters here. 48 | value = String(value); 49 | } 50 | 51 | if (typeof value === 'number') { 52 | // Simple numbers 53 | return JSON.stringify(value); 54 | } 55 | 56 | if (typeof value === 'string') { 57 | // If any of these match, then we can't output it 58 | // as bare unquoted text, because that would either 59 | // cause data loss or invalid YAML syntax. 60 | // 61 | // - Quotes, escapes, line breaks, or JSON-like stuff. 62 | const rSpecialJson = /['"\\/[{}\]\r\n]/; 63 | // - Characters that are special at the start of a YAML value 64 | const rSpecialYaml = /[-?:,[\]{}#&*!|=>'"%@`]/; 65 | // - Leading or trailing whitespace. 66 | const rUntrimmed = /(^\s|\s$)/; 67 | // - Ambiguous as YAML number, e.g. '2', '-1.2', '.2', or '2_000' 68 | const rNumerical = /^[\d._-]+$/; 69 | // - Ambiguous as YAML bool. 70 | // Use case-insensitive match, although technically only 71 | // fully-lower, fully-upper, or uppercase-first would be ambiguous. 72 | // e.g. true/True/TRUE, but not tRUe. 73 | const rBool = /^(true|false|y|n|yes|no|on|off)$/i; 74 | 75 | // Is this a complex string? 76 | if ( 77 | value === '' || 78 | rSpecialJson.test(value) || 79 | rSpecialYaml.test(value[0]) || 80 | rUntrimmed.test(value) || 81 | rNumerical.test(value) || 82 | rBool.test(value) 83 | ) { 84 | if (!/\n/.test(value)) { 85 | // Complex one-line string, use JSON (quoted string) 86 | return JSON.stringify(value); 87 | } 88 | 89 | // See also 90 | // Support IE 9-11: Avoid ES6 String#repeat 91 | const prefix = (new Array(indent + 1)).join(' '); 92 | 93 | const trailingLinebreakMatch = value.match(/\n+$/); 94 | const trailingLinebreaks = trailingLinebreakMatch ? trailingLinebreakMatch[0].length : 0; 95 | 96 | if (trailingLinebreaks === 1) { 97 | // Use the most straight-forward "Block" string in YAML 98 | // without any "Chomping" indicators. 99 | const lines = value 100 | // Ignore the last new line, since we'll get that one for free 101 | // with the straight-forward Block syntax. 102 | .replace(/\n$/, '') 103 | .split('\n') 104 | .map(line => prefix + line); 105 | return '|\n' + lines.join('\n'); 106 | } else { 107 | // This has either no trailing new lines, or more than 1. 108 | // Use |+ so that YAML parsers will preserve it exactly. 109 | const lines = value 110 | .split('\n') 111 | .map(line => prefix + line); 112 | return '|+\n' + lines.join('\n'); 113 | } 114 | } else { 115 | // Simple string, use bare unquoted text 116 | return value; 117 | } 118 | } 119 | 120 | // Handle null, boolean, array, and object 121 | return JSON.stringify(decycledShallowClone(value), null, 2); 122 | } 123 | 124 | /** 125 | * Creates a shallow clone of an object where cycles have 126 | * been replaced with "[Circular]". 127 | */ 128 | function decycledShallowClone (object, ancestors = []) { 129 | if (ancestors.indexOf(object) !== -1) { 130 | return '[Circular]'; 131 | } 132 | 133 | let clone; 134 | 135 | const type = Object.prototype.toString 136 | .call(object) 137 | .replace(/^\[.+\s(.+?)]$/, '$1') 138 | .toLowerCase(); 139 | 140 | switch (type) { 141 | case 'array': 142 | ancestors.push(object); 143 | clone = object.map(function (element) { 144 | return decycledShallowClone(element, ancestors); 145 | }); 146 | ancestors.pop(); 147 | break; 148 | case 'object': 149 | ancestors.push(object); 150 | clone = {}; 151 | Object.keys(object).forEach(function (key) { 152 | clone[key] = decycledShallowClone(object[key], ancestors); 153 | }); 154 | ancestors.pop(); 155 | break; 156 | default: 157 | clone = object; 158 | } 159 | 160 | return clone; 161 | } 162 | 163 | module.exports = class TapReporter { 164 | constructor (runner, options = {}) { 165 | // Cache references to console methods to ensure we can report failures 166 | // from tests tests that mock the console object itself. 167 | // https://github.com/qunitjs/js-reporters/issues/125 168 | this.log = options.log || console.log.bind(console); 169 | 170 | this.testCount = 0; 171 | 172 | runner.on('runStart', this.onRunStart.bind(this)); 173 | runner.on('testEnd', this.onTestEnd.bind(this)); 174 | runner.on('runEnd', this.onRunEnd.bind(this)); 175 | } 176 | 177 | static init (runner) { 178 | return new TapReporter(runner); 179 | } 180 | 181 | onRunStart (globalSuite) { 182 | this.log('TAP version 13'); 183 | } 184 | 185 | onTestEnd (test) { 186 | this.testCount = this.testCount + 1; 187 | 188 | if (test.status === 'passed') { 189 | this.log(`ok ${this.testCount} ${test.fullName.join(' > ')}`); 190 | } else if (test.status === 'skipped') { 191 | this.log(kleur.yellow(`ok ${this.testCount} # SKIP ${test.fullName.join(' > ')}`)); 192 | } else if (test.status === 'todo') { 193 | this.log(kleur.cyan(`not ok ${this.testCount} # TODO ${test.fullName.join(' > ')}`)); 194 | test.errors.forEach((error) => this.logError(error, 'todo')); 195 | } else { 196 | this.log(kleur.red(`not ok ${this.testCount} ${test.fullName.join(' > ')}`)); 197 | test.errors.forEach((error) => this.logError(error)); 198 | } 199 | } 200 | 201 | onRunEnd (globalSuite) { 202 | this.log(`1..${globalSuite.testCounts.total}`); 203 | this.log(`# pass ${globalSuite.testCounts.passed}`); 204 | this.log(kleur.yellow(`# skip ${globalSuite.testCounts.skipped}`)); 205 | this.log(kleur.cyan(`# todo ${globalSuite.testCounts.todo}`)); 206 | this.log(kleur.red(`# fail ${globalSuite.testCounts.failed}`)); 207 | } 208 | 209 | logError (error, severity) { 210 | let out = ' ---'; 211 | out += `\n message: ${prettyYamlValue(error.message || 'failed')}`; 212 | out += `\n severity: ${prettyYamlValue(severity || 'failed')}`; 213 | 214 | if (hasOwn.call(error, 'actual')) { 215 | out += `\n actual : ${prettyYamlValue(error.actual)}`; 216 | } 217 | 218 | if (hasOwn.call(error, 'expected')) { 219 | out += `\n expected: ${prettyYamlValue(error.expected)}`; 220 | } 221 | 222 | if (error.stack) { 223 | // Since stacks aren't user generated, take a bit of liberty by 224 | // adding a trailing new line to allow a straight-forward YAML Blocks. 225 | out += `\n stack: ${prettyYamlValue(error.stack + '\n')}`; 226 | } 227 | 228 | out += '\n ...'; 229 | this.log(out); 230 | } 231 | }; 232 | -------------------------------------------------------------------------------- /test/fixtures/unit.js: -------------------------------------------------------------------------------- 1 | function mockStack (error) { 2 | error.stack = ` at Object. (/dev/null/test/unit/data.js:6:5) 3 | at require (internal/modules/cjs/helpers.js:22:18) 4 | at /dev/null/node_modules/mocha/lib/mocha.js:220:27 5 | at startup (internal/bootstrap/node.js:283:19)`; 6 | return error; 7 | } 8 | 9 | function copyErrors (testEnd) { 10 | testEnd.assertions = testEnd.errors; 11 | return testEnd; 12 | } 13 | 14 | /** 15 | * Creates an object that has a cyclical reference. 16 | */ 17 | function createCyclical () { 18 | const cyclical = { a: 'example' }; 19 | cyclical.cycle = cyclical; 20 | return cyclical; 21 | } 22 | 23 | /** 24 | * Creates an object that has a cyclical reference in a subobject. 25 | */ 26 | function createSubobjectCyclical () { 27 | const cyclical = { a: 'example', sub: {} }; 28 | cyclical.sub.cycle = cyclical; 29 | return cyclical; 30 | } 31 | 32 | /** 33 | * Creates an object that references another object more 34 | * than once in an acyclical way. 35 | */ 36 | function createDuplicateAcyclical () { 37 | const duplicate = { 38 | example: 'value' 39 | }; 40 | return { 41 | a: duplicate, 42 | b: duplicate, 43 | c: 'unique' 44 | }; 45 | } 46 | 47 | module.exports = { 48 | passingTestStart: { 49 | name: 'pass', 50 | suiteName: null, 51 | fullName: ['pass'] 52 | }, 53 | passingTest: { 54 | name: 'pass', 55 | suiteName: null, 56 | fullName: ['pass'], 57 | status: 'passed', 58 | runtime: 0, 59 | errors: [], 60 | assertions: [] 61 | }, 62 | failingTest: copyErrors({ 63 | name: 'fail', 64 | fullName: ['fail'], 65 | status: 'failed', 66 | runtime: 0, 67 | errors: [ 68 | mockStack(new Error('first error')), 69 | mockStack(new Error('second error')) 70 | ], 71 | assertions: null 72 | }), 73 | failingTapData: [ 74 | ` --- 75 | message: first error 76 | severity: failed 77 | stack: | 78 | at Object. (/dev/null/test/unit/data.js:6:5) 79 | at require (internal/modules/cjs/helpers.js:22:18) 80 | at /dev/null/node_modules/mocha/lib/mocha.js:220:27 81 | at startup (internal/bootstrap/node.js:283:19) 82 | ...`, 83 | ` --- 84 | message: second error 85 | severity: failed 86 | stack: | 87 | at Object. (/dev/null/test/unit/data.js:6:5) 88 | at require (internal/modules/cjs/helpers.js:22:18) 89 | at /dev/null/node_modules/mocha/lib/mocha.js:220:27 90 | at startup (internal/bootstrap/node.js:283:19) 91 | ...` 92 | ], 93 | actualUndefinedTest: copyErrors({ 94 | name: 'Failing', 95 | suiteName: null, 96 | fullName: ['Failing'], 97 | status: 'failed', 98 | runtime: 0, 99 | errors: [{ 100 | passed: false, 101 | actual: undefined, 102 | expected: 'expected' 103 | }], 104 | assertions: null 105 | }), 106 | actualInfinity: copyErrors({ 107 | name: 'Failing', 108 | suiteName: null, 109 | fullName: ['Failing'], 110 | status: 'failed', 111 | runtime: 0, 112 | errors: [{ 113 | passed: false, 114 | actual: Infinity, 115 | expected: 'expected' 116 | }], 117 | assertions: null 118 | }), 119 | actualStringChar: copyErrors({ 120 | name: 'Failing', 121 | suiteName: null, 122 | fullName: ['Failing'], 123 | status: 'failed', 124 | runtime: 0, 125 | errors: [{ 126 | passed: false, 127 | actual: 'abc', 128 | expected: 'expected' 129 | }], 130 | assertions: null 131 | }), 132 | actualStringNum: copyErrors({ 133 | name: 'Failing', 134 | suiteName: null, 135 | fullName: ['Failing'], 136 | status: 'failed', 137 | runtime: 0, 138 | errors: [{ 139 | passed: false, 140 | actual: '2', 141 | expected: 'expected' 142 | }], 143 | assertions: null 144 | }), 145 | actualStringBool: copyErrors({ 146 | name: 'Failing', 147 | suiteName: null, 148 | fullName: ['Failing'], 149 | status: 'failed', 150 | runtime: 0, 151 | errors: [{ 152 | passed: false, 153 | actual: 'true', 154 | expected: 'expected' 155 | }], 156 | assertions: null 157 | }), 158 | actualStringOneTailLn: copyErrors({ 159 | name: 'Failing', 160 | suiteName: null, 161 | fullName: ['Failing'], 162 | status: 'failed', 163 | runtime: 0, 164 | errors: [{ 165 | passed: false, 166 | actual: 'abc\n', 167 | expected: 'expected' 168 | }], 169 | assertions: null 170 | }), 171 | actualStringOneTailLnTap: ` --- 172 | message: failed 173 | severity: failed 174 | actual : | 175 | abc 176 | expected: expected 177 | ...`, 178 | actualStringTwoTailLn: copyErrors({ 179 | name: 'Failing', 180 | suiteName: null, 181 | fullName: ['Failing'], 182 | status: 'failed', 183 | runtime: 0, 184 | errors: [{ 185 | passed: false, 186 | actual: 'abc\n\n', 187 | expected: 'expected' 188 | }], 189 | assertions: null 190 | }), 191 | actualStringTwoTailLnTap: ` --- 192 | message: failed 193 | severity: failed 194 | actual : |+ 195 | abc 196 | 197 | 198 | expected: expected 199 | ...`, 200 | actualZero: copyErrors({ 201 | name: 'Failing', 202 | suiteName: null, 203 | fullName: ['Failing'], 204 | status: 'failed', 205 | runtime: 0, 206 | errors: [{ 207 | passed: false, 208 | actual: 0, 209 | expected: 'expected' 210 | }], 211 | assertions: null 212 | }), 213 | actualArray: copyErrors({ 214 | name: 'Failing', 215 | suiteName: null, 216 | fullName: ['Failing'], 217 | status: 'failed', 218 | runtime: 0, 219 | errors: [{ 220 | passed: false, 221 | actual: [], 222 | expected: 'expected' 223 | }], 224 | assertions: null 225 | }), 226 | actualArrayTap: ` --- 227 | message: failed 228 | severity: failed 229 | actual : [] 230 | expected: expected 231 | ...`, 232 | actualCyclical: copyErrors({ 233 | name: 'Failing', 234 | suiteName: undefined, 235 | fullName: ['Failing'], 236 | status: 'failed', 237 | runtime: 0, 238 | errors: [{ 239 | passed: false, 240 | actual: createCyclical(), 241 | expected: 'expected' 242 | }], 243 | assertions: null 244 | }), 245 | actualCyclicalTap: ` --- 246 | message: failed 247 | severity: failed 248 | actual : { 249 | "a": "example", 250 | "cycle": "[Circular]" 251 | } 252 | expected: expected 253 | ...`, 254 | actualSubobjectCyclical: copyErrors({ 255 | name: 'Failing', 256 | suiteName: undefined, 257 | fullName: ['Failing'], 258 | status: 'failed', 259 | runtime: 0, 260 | errors: [{ 261 | passed: false, 262 | actual: createSubobjectCyclical(), 263 | expected: 'expected' 264 | }], 265 | assertions: null 266 | }), 267 | actualSubobjectCyclicalTap: ` --- 268 | message: failed 269 | severity: failed 270 | actual : { 271 | "a": "example", 272 | "sub": { 273 | "cycle": "[Circular]" 274 | } 275 | } 276 | expected: expected 277 | ...`, 278 | actualDuplicateAcyclic: copyErrors({ 279 | name: 'Failing', 280 | suiteName: undefined, 281 | fullName: ['Failing'], 282 | status: 'failed', 283 | runtime: 0, 284 | errors: [{ 285 | passed: false, 286 | actual: createDuplicateAcyclical(), 287 | expected: 'expected' 288 | }], 289 | assertions: null 290 | }), 291 | actualDuplicateAcyclicTap: ` --- 292 | message: failed 293 | severity: failed 294 | actual : { 295 | "a": { 296 | "example": "value" 297 | }, 298 | "b": { 299 | "example": "value" 300 | }, 301 | "c": "unique" 302 | } 303 | expected: expected 304 | ...`, 305 | expectedUndefinedTest: copyErrors({ 306 | name: 'fail', 307 | suiteName: null, 308 | fullName: [], 309 | status: 'failed', 310 | runtime: 0, 311 | errors: [{ 312 | passed: false, 313 | actual: 'actual', 314 | expected: undefined 315 | }], 316 | assertions: null 317 | }), 318 | expectedFalsyTest: copyErrors({ 319 | name: 'fail', 320 | suiteName: null, 321 | fullName: [], 322 | status: 'failed', 323 | runtime: 0, 324 | errors: [{ 325 | passed: false, 326 | actual: 'actual', 327 | expected: 0 328 | }], 329 | assertions: null 330 | }), 331 | expectedCircularTest: copyErrors({ 332 | name: 'fail', 333 | suiteName: undefined, 334 | fullName: [], 335 | status: 'failed', 336 | runtime: 0, 337 | errors: [{ 338 | passed: false, 339 | actual: 'actual', 340 | expected: createCyclical() 341 | }], 342 | assertions: null 343 | }), 344 | skippedTest: { 345 | name: 'skip', 346 | suiteName: null, 347 | fullName: ['skip'], 348 | status: 'skipped', 349 | runtime: 0, 350 | errors: [], 351 | assertions: [] 352 | }, 353 | todoTest: { 354 | name: 'todo', 355 | suiteName: null, 356 | fullName: ['todo'], 357 | status: 'todo', 358 | runtime: 0, 359 | errors: [], 360 | assertions: [] 361 | } 362 | }; 363 | -------------------------------------------------------------------------------- /docs/frameworks.md: -------------------------------------------------------------------------------- 1 | # Frameworks flow 2 | 3 | This document studies the differencess between various JavaScript testing frameworks. 4 | 5 | ## Mocha 6 | 7 | [Mocha](https://github.com/mochajs/mocha) is a testing framework without builtin assertions, this is why it is not checking tests 8 | for at least one assertion, so in our Mocha specific [test fixture](../test/fixtures/integration/mocha.js) 9 | we can have empty tests. 10 | 11 | Tests are grouped in `suites`, which can also be `nested`. Tests can be placed also outside of a suite, we call them *global tests*. 12 | 13 | Internally Mocha wraps everything in a suite, we call it *global suite*, so the global tests will become the tests of the aformentioned suite, as also all other top level suites will become its direct child suites, implicitly all other suites will become its more deeper child suites, in a recursive structure. 14 | 15 | Test particularities: 16 | * skipped tests start is not emitted by Mocha on its event *test*, but their end is emitted on *test end* 17 | * skipped tests do not have the `duration` property (i.e runtime) at all 18 | * failed tests have only one error, even if the tests contain multiple assertions, Mocha stops on the first failed assertion 19 | * the error of a failed test is only passed as parameter on Mocha's *fail* event, the `err` property is not availabe on the test object passed on "test end" event, it would be availabe only if you use Mocha's builtin reporters, because this property is added by their [base reporter](https://github.com/mochajs/mocha/blob/e939d8e4379a622e28064ca3a75f3e1bda7e225b/lib/reporters/base.js#L279) 20 | 21 | Suite particularities: 22 | * the start and end of a suite, even the global one, are emitted only if the suite contains at least a test or a child suite (i.e nested suites) that contains a test 23 | 24 | One interesting aspect is the execution of nested suites. Practically, when a suite in encountered that contains also suites and tests, its tests are always executed before the suites, no matter if the tests were declared after their siblings suites. 25 | 26 | Lets take an example, to see quite all the idea explained above: 27 | 28 | ```js 29 | describe('a', function() { 30 | describe('b', function() { 31 | it('bb', function () {}); 32 | }); 33 | 34 | describe('c', function() { 35 | describe('ca', function() { 36 | it('cca', function() {}); 37 | }); 38 | 39 | it('cc', function() {}); 40 | }); 41 | 42 | describe('d', function () { 43 | }); 44 | 45 | it('aa', function() {}); 46 | }); 47 | ``` 48 | Execution flow: 49 | * global suite starts 50 | * suite *a* starts 51 | * test *aa* starts 52 | * test *aa* ends 53 | * suite *b* starts 54 | * test *bb* starts 55 | * test *bb* ends 56 | * suite *b* ends 57 | * suite *c* starts 58 | * test *cc* starts 59 | * test *cc* ends 60 | * suite *ca* starts 61 | * test *cca* starts 62 | * test *cca* ends 63 | * suite *ca* ends 64 | * suite *c* ends 65 | * suite *a* ends 66 | * global suite ends 67 | 68 | This is the execution of the above test fixture, as you can see the `d suite` is not executed. 69 | 70 | Mocha has an open [issue](https://github.com/mochajs/mocha/issues/902) for random test execution. 71 | 72 | ## QUnit 73 | 74 | [QUnit](https://qunitjs.com/) is a testing framework with builtin assertion, so it is checking tests for at least one assertion, if it does not find one, the test will fail with an error thrown by QUnit itself. 75 | 76 | Tests are grouped in modules, which can be also nested since [QUnit 1.20](https://github.com/qunitjs/qunit/releases/tag/1.20.0). Tests can be placed outside a module, we call them *global tests*. 77 | 78 | Internally, QUnit has an implicit global module to hold any global tests. Note that user-defined modules are siblings of the global one, not nested within it. To emit a global suite on our *runStart/runEnd* events we must access QUnit internals, *QUnit.config.modules* which is a linear array that will contain all modules, even the nested ones. 79 | 80 | An interesting fact of *QUnit.config.modules* is that it will not contain the implicit *global module* unless it has at least one test, but it will contain all user-defined modules, even if they do not have a test. 81 | 82 | Test particularities: 83 | * skipped tests have a numeric value for their runtime. 84 | 85 | Module particularities: 86 | * the start and end of a module, even the global one, are emitted only if the suite itself contains at least one test. 87 | * nested modules have a concatenated name, from the outer most suite to the inner most. 88 | 89 | The execution is done in the source order, but QUnit has a more flat style for nested modules, it emits the start of a module, emits its tests, then the module ends and starts another, even if the modules were nested, there is not a sort of recursion between the modules. 90 | 91 | **In contrast with the source order execution, the QUnit default reporter is always displaying only the suites in the source order**, tests are displayed together with their parent module which breaks the source order, this applies also for random tests execution (check out below example). 92 | 93 | Example: 94 | 95 | ```js 96 | module('a', function() { 97 | module('b', function() { 98 | test('bb', function(assert) { 99 | assert.ok(true); 100 | }); 101 | }); 102 | 103 | module('c', function() { 104 | module('ca', function() { 105 | test('cca', function(assert) { 106 | assert.ok(true); 107 | }); 108 | }); 109 | 110 | test('cc', function(assert) { 111 | assert.ok(true); 112 | }); 113 | }); 114 | 115 | module('d', function() { 116 | 117 | }); 118 | 119 | test('aa', function(assert) { 120 | assert.ok(true); 121 | }); 122 | }); 123 | ``` 124 | Execution flow: 125 | * module *a > b* starts 126 | * test *bb* starts 127 | * test *bb* ends 128 | * module *a > b* ends 129 | * module *a > c > ca* starts 130 | * test *cca* starts 131 | * test *cca* ends 132 | * module *a > c > ca* ends 133 | * module *a > c* starts 134 | * test *cc* starts 135 | * test *cc* starts 136 | * module *a > c* ends 137 | * module *a* starts 138 | * test *aa* starts 139 | * test *aa* ends 140 | * module *a* ends 141 | 142 | Reporter output: 143 | * a: aa (1) 144 | * a > b: bb (1) 145 | * a > c: cc (1) 146 | * a > c > ca: cca (1) 147 | 148 | The *QUnit.config.modules* will contain 5 modules: 149 | 0. module *a* 150 | 1. module *a > b* 151 | 2. module *a > c* 152 | 3. module *a > c > a* 153 | 4. module *a > d* 154 | 155 | **The above execution flow is the default one**, QUnit has also 2 options that randomizes tests execution: 156 | 1. the [reorder](https://api.qunitjs.com/config/QUnit.config/) option that on a rerun, runs firstly the failed tests, it is activated by default 157 | 2. the [seed](https://api.qunitjs.com/config/QUnit.config/) option that randomizes tests execution, it is disabled by default 158 | 159 | **The QUnit.config.modules will always contain the suites in the same order!** 160 | 161 | ## Jasmine 162 | 163 | [Jasmine](https://jasmine.github.io/) is another testing framework with builtin assertions. Tests will pass, even if they have no assertions. 164 | 165 | Tests are grouped in suites, which can be nested. Tests can be placed also outside a suite, then they will belong to Jasmine's global suite. 166 | 167 | To obtain information about the relationships between tests and suites can be achieved only through Jasmines's `topSuite`, because the objects emitted on Jasmine specific events contain only plain data about the test/suite in cause. 168 | 169 | Tests and suites objects contain always a unique id assigned by Jasmine itself. 170 | 171 | Test particularities: 172 | * tests have no `runtime` prop 173 | * failed tests can contain multiple errors, i.e *failedExpectations* 174 | * *beforeAll*, *beforeEach*, *afterEach* hook's errors will result in tests errors 175 | 176 | Suite particularities: 177 | * the global suite start and end is not emitted 178 | * the start and end of a suite is emitted even if it does not contain any tests or other suites 179 | * suites have a *failedExpectations* prop which can contain only errors happend in the `afterAll` hook 180 | 181 | Jasmine test execution is done exactly in the source order. 182 | 183 | Example: 184 | ```js 185 | describe('a', function() { 186 | describe('b', function() { 187 | it('bb', function() { 188 | expect(true).toBeTruthy(); 189 | }); 190 | }); 191 | 192 | describe('c', function() { 193 | describe('ca', function() { 194 | it('cca', function() { 195 | expect(true).toBeTruthy(); 196 | }); 197 | }); 198 | 199 | it('cc', function() { 200 | expect(true).toBeTruthy(); 201 | }); 202 | }); 203 | 204 | describe('d', function() { 205 | }); 206 | 207 | it('aa', function() { 208 | expect(true).toBeTruthy(); 209 | }); 210 | }); 211 | ``` 212 | 213 | Execution flow: 214 | * Suite *a* starts 215 | * Suite *b* starts 216 | * Test *bb* starts 217 | * Test *bb* ends 218 | * Suite *b* ends 219 | * Suite *c* starts 220 | * Suite *ca* starts 221 | * Test *cca* starts 222 | * Test *cca* ends 223 | * Suite *ca* ends 224 | * Test *cc* starts 225 | * Test *cc* ends 226 | * Suite *c* ends 227 | * Suite *d* starts 228 | * Suite *d* ends 229 | * Test *aa* starts 230 | * Test *aa* ends 231 | * Suite *a* ends 232 | 233 | **Jasmine has also an option for randomizing tests execution**, the default reporter will alwasys show the tests in the order they were executed. 234 | 235 | -------------------------------------------------------------------------------- /spec/cri-draft.adoc: -------------------------------------------------------------------------------- 1 | = Common Reporter Interface - Working Draft 2 | :sectanchors: 3 | :sectlinks: 4 | :sectnums: 5 | :toc: macro 6 | :toclevels: 2 7 | :toc-title: 8 | :note-caption: :paperclip: 9 | :tip-caption: :bulb: 10 | :warning-caption: :warning: 11 | 12 | Participate:: 13 | https://github.com/qunitjs/js-reporters[GitHub qunitjs/js-reporters] (https://github.com/qunitjs/js-reporters/issues/new[new issue], https://github.com/qunitjs/js-reporters/issues[open issues]) + 14 | https://gitter.im/js-reporters/js-reporters[Chat room on Gitter] 15 | 16 | Last updated:: 17 | 20 Februrary 2021 18 | 19 | Abstract:: 20 | This specification defines JavaScript APIs for reporting progress and results of executing software tests. 21 | 22 | Goal:: 23 | This specification is meant to be implemented by testing frameworks, reporters, and middleware adapters. 24 | 25 | toc::[] 26 | 27 | == Terminology 28 | 29 | Testing framework:: 30 | A testing framework is a program that helps define, organize, load, or execute software tests through assertions. (https://en.wikipedia.org/wiki/Test_automation[Learn more]) 31 | 32 | Adapters:: 33 | A program that implements the <> on behalf of a testing framework, for example to support testing frameworks that don't yet implement the CRI standard, or to support reporting events from a remotely-executed test run. 34 | 35 | Producer:: 36 | Any JavaScript program that implements the <> of the CRI standard and emit its events, typically this is a testing framework, or an adapter for one. 37 | 38 | Assertion:: 39 | An assertion is a logical preposition required to evaluate to true. Assertions must be part of a "Test". (link:https://en.wikipedia.org/wiki/Assertion_(software_development)[Learn more]) 40 | 41 | Passing assertion:: 42 | An assertion that has evaluated to boolean true. 43 | 44 | Failed assertion:: 45 | An assertion that has evaluated to boolean false. 46 | 47 | [[test]] Test:: 48 | A test is a named group containing zero or more assertions. + 49 | + 50 | It is recommended that all tests contain assertions, but this is not required. For example, a testing framework that only records failed assertions (such as a testing framework that is decoupled from an assertion library and uses exceptions to discover failures), might not distinguish between a test with passing assertions and a test with no assertions. If a testing framework is generally aware of assertions and if it considers absence of those an error, then it should ensure the test or test [[run]] is marked as failing. For example, by implicitly producing a failed assertion. + 51 | + 52 | In QUnit, a test may be defined by calling `QUnit.test()`. + 53 | In Mocha and Jasmine BDD, a test is known as a "spec", defined by calling `it()`. + 54 | (https://en.wikipedia.org/wiki/Test_case[Learn more]) + 55 | 56 | Skipped test:: 57 | A <> that was not actually run. Testing frameworks may have ways of selecting, partially loading, filtering, or otherwise skipping tests. These implementation choices may mean that some tests are not considered part of the <>, and thus entirely left out of the information exposed to reporters. Presence of one skipped test does not imply that all skipped tests will be reported in this way. + 58 | + 59 | See also the `SKIP` directive of the https://testanything.org/tap-version-13-specification.html#directives[TAP specification]. 60 | 61 | Todo test:: 62 | A <> that is expected to have one or more failing assertions. + 63 | + 64 | See also the `TODO` directive of the https://testanything.org/tap-version-13-specification.html#directives[TAP specification]. 65 | 66 | [[suite]] Suite:: 67 | A suite is a named group representing zero or more tests, and zero or more other suites. A suite that is part of another suite may also be called a "child suite". A suite that holds one or more child suites may also be called an "ancestor suite". + 68 | (https://en.wikipedia.org/wiki/Test_case[Learn more]) + 69 | + 70 | In QUnit, a suite is known as a "module", defined by calling `QUnit.module()`. + 71 | In Mocha and Jasmine BDD, a suite is defined by calling `describe()`. + 72 | In JUnit and other xUnit-derivatives, tests are first grouped in a `TestCase` which are then further grouped under a ``. In the CRI standard, both of these are considered a suite. 73 | 74 | [[run]] Run:: 75 | A run is a single top-level group representing all tests and suites that a producer is planning to report events about. 76 | 77 | Reporter:: 78 | A JavaScript program that consumes information from a <>. For example, to render an HTML graphical user interface, to write command-line output in the https://testanything.org/[TAP] format, write results to a https://llg.cubic.org/docs/junit/[JUnit XML] artifact file, or serialize the information and transfer it over a socket to another server or process. 79 | 80 | [TIP] 81 | ===== 82 | The use of "Suite" and "Test" as the main two data structues was decided in https://github.com/qunitjs/js-reporters/issues/12[issue #12], and later revised in https://github.com/qunitjs/js-reporters/issues/126[issue #126]. 83 | ===== 84 | 85 | == Events 86 | 87 | These are the events that a <> should emit from its <>, for consumption by a <>. 88 | 89 | [TIP] 90 | ===== 91 | These events were selected as: 92 | 93 | - common across known testing frameworks (gathered in https://github.com/qunitjs/js-reporters/issues/1#issuecomment-54841874[issue #1]). 94 | - valid JavaScript identifiers, allowing use as variable name and as object literal key without quotes. 95 | - not overlapping with existing events in known testing frameworks, for easy adoption within existing APIs. 96 | ===== 97 | 98 | === Reporting order 99 | 100 | It is recommended, though not required, that events about tests are emitted in **source order**, based on how the tests are defined by a developer in a test file. This means results of tests defined is higher up in a test file should be emitted earlier than those defined lower down in the file. 101 | 102 | Note that execution order may be different from reporting order. If a testing framework uses concurrency or random seeding for its execution, we recommend that events are still consistently emitted in the source order. 103 | 104 | [TIP] 105 | ===== 106 | Read https://github.com/qunitjs/js-reporters/issues/62[issue #62] for the discussion about reporting order. 107 | ===== 108 | 109 | === `runStart` event 110 | 111 | The **runStart** event indicates the beginning of a <>. It must be emitted exactly once, and before any <> or <>. 112 | 113 | Callback parameters: 114 | 115 | * <> **runStart**: The plan for the run. 116 | 117 | [source,javascript] 118 | ---- 119 | producer.on('runStart', (runStart) => { … }); 120 | ---- 121 | 122 | === `runEnd` event 123 | 124 | The **runEnd** event indicates the end of a <>. It must be emitted exactly once, after the last of any <> or <>. 125 | 126 | Callback parameters: 127 | 128 | * <> **runEnd**: Summary of test results from the completed run. 129 | 130 | [source,javascript] 131 | ---- 132 | producer.on('runEnd', (runEnd) => { … }); 133 | ---- 134 | 135 | === `suiteStart` event 136 | 137 | The **suiteStart** event indicates the beginning of a <>. It must eventually be followed by a corresponding <>. 138 | 139 | Callback parameters: 140 | 141 | * <> **suiteStart**: Basic information about a suite. 142 | 143 | [source,javascript] 144 | ---- 145 | producer.on('suiteStart', (suiteStart) => { … }); 146 | ---- 147 | 148 | === `suiteEnd` event 149 | 150 | The **suiteEnd** event indicates the end of a <>. It must be emitted after its corresponding <>. 151 | 152 | Callback parameters: 153 | 154 | * <> **suiteEnd**: Basic information about a completed suite. 155 | 156 | [source,javascript] 157 | ---- 158 | producer.on('suiteEnd', (suiteEnd) => { … }); 159 | ---- 160 | 161 | === `testStart` event 162 | 163 | The **testStart** event indicates the beginning of a <>. It must eventually be followed by a corresponding <>. A producer may emit several <> events before any corresponding <>, for example when there are child tests, or tests that run concurrently. 164 | 165 | Callback parameters: 166 | 167 | * <> **testStart**: Basic information about a test. 168 | 169 | [source,javascript] 170 | ---- 171 | producer.on('testStart', (testStart) => { … }); 172 | ---- 173 | 174 | [TIP] 175 | ===== 176 | If a producer has no real-time information about test execution, it may simply emit `testStart` back-to-back with `testEnd`. 177 | ===== 178 | 179 | === `testEnd` event 180 | 181 | The **testEnd** event indicates the end of a <>. It must be emitted after its corresponding <>. 182 | 183 | Callback parameters: 184 | 185 | * <> **testEnd**: Result of a completed test. 186 | 187 | [source,javascript] 188 | ---- 189 | producer.on('testEnd', (testEnd) => { … }); 190 | ---- 191 | 192 | == Event data 193 | 194 | The following data structures must be implemented as objects that have the specified fields as own properties. The objects are not required to be an instance of any specific class. They may be null-inherited objects, plain objects, or an instance of any public or private class. 195 | 196 | === SuiteStart 197 | 198 | `SuiteStart` object: 199 | 200 | * `string` **name**: Name of the suite. 201 | * `Array` **fullName**: List of one or more strings, containing (in order) the names of any grandancestor suites, the name of the suite. 202 | 203 | === SuiteEnd 204 | 205 | `SuiteEnd` object: 206 | 207 | * `string` **name**: Name of the suite. 208 | * `Array` **fullName**: List of one or more strings, containing (in order) the names of any grandancestor suites, the name of the suite. 209 | * `string` **status**: Aggregate result of all tests, one of: 210 | ** **failed** if at least one test has failed. 211 | ** **passed**, if there were no failed tests, which means there either were no tests, or tests only had passed, skipped, or todo statuses. 212 | * `number|null` **runtime**: Optional duration of the suite in milliseconds. 213 | 214 | === RunStart 215 | 216 | The plan for the <>. 217 | 218 | `RunStart` object: 219 | 220 | * `string|null` **name**: Name of the overall run, or `null` if the producer is unaware of a name. 221 | * `Object` **testCounts**: Aggregate counts about tests. 222 | ** `number|null` **total**: Total number of tests the producer is expecting to emit events for, e.g. if there would be no unexpected failures. It may be `null` if the total is not known ahead of time. 223 | 224 | === RunEnd 225 | 226 | Summary of test results from the completed <>. 227 | 228 | `RunEnd` object: 229 | 230 | * `string|null` **name**: Name of the overall run, or `null` if the producer is unaware of a name. 231 | * `string` **status**: Aggregate result of all tests, one of: 232 | ** **failed** if at least one test has failed. 233 | ** **passed**, if there were no failed tests, which means there either were no tests, or tests only had passed, skipped, or todo statuses. 234 | * `Object` **testCounts**: Aggregate counts about tests. 235 | ** `number` **passed**: Number of passed tests. 236 | ** `number` **failed**: Number of failed tests. 237 | ** `number` **skipped**: Number of skipped tests. 238 | ** `number` **todo**: Number of todo tests. 239 | ** `number` **total**: Total number of tests, the sum of the above properties must equal this one. 240 | * `number|null` **runtime**: Optional duration of the run in milliseconds. This may be the sum of the runtime of each test, but may also be higher or lower. For example, it could be higher if the producer includes time spent outside specific tests, or lower if tests run concurrently and the reporter measures observed wall time rather than a sum. 241 | 242 | === TestStart 243 | 244 | Basic information about a <>. 245 | 246 | `TestStart` object: 247 | 248 | * `string` **name**: Name of the test. 249 | * `string|null` **suiteName**: Name of the suite the test belongs to, or `null` if it has no suite. 250 | * `Array` **fullName**: List of one or more strings, containing (in order) the names of any grandancestor suites, the name of the suite, and the name of the test itself. 251 | 252 | === TestEnd 253 | 254 | Result of a completed <>. This is a superset of <>. 255 | 256 | `TestEnd` object: 257 | 258 | * `string` **name**: Name of the test. 259 | * `string|null` **suiteName**: Name of the suite the test belongs to, or `null` if it has no suite. 260 | * `Array` **fullName**: List of one or more strings, containing (in order) the names of any ancestor suites, the name of the suite, and the name of the test itself. 261 | * `string` **status**: Result of the test, one of: 262 | ** **passed**, if all assertions have passed, or if no assertions were recorded. 263 | ** **failed**, if at least one assertion has failed or if the test is todo and its assertions unexpectedly all passed. 264 | ** **skipped**, if the test was intentionally not run. 265 | ** **todo**, if the test is todo and indeed has at least one failing assertion still. 266 | * `number|null` **runtime**: Optional duration of the run in milliseconds. 267 | * `Array` **errors**: List of failed <> objects. It should contain at least one item for failed tests, and must be empty for other tests. 268 | * `Array` **assertions**: List of failed and any passed <> objects. For a skipped test, this must be empty. 269 | 270 | === Assertion 271 | 272 | The **Assertion** object contains information about a single assertion. 273 | 274 | `Assertion` object: 275 | 276 | * `boolean` **passed**: Set to `true` for a passed assertion, `false` for a failed assertion. 277 | * `Mixed` **actual**: The actual value passed to the assertion, should be similar to `expected` for passed assertions. 278 | * `Mixed` **expected**: The expected value passed to the assertion, should be similar to `actual` for passed assertions. 279 | * `string` **message**: Name of the assertion, or description of what the assertion checked for. 280 | * `string|null` **stack**: Optional stack trace. For a "passed" assertion, the property must be set to `null`. 281 | 282 | Producers may set additional (non-standard) properties on `Assertion` objects. 283 | 284 | [TIP] 285 | ===== 286 | The properties of the Assertion object was decided in https://github.com/qunitjs/js-reporters/issues/79[issue #79], and later revised by https://github.com/qunitjs/js-reporters/issues/105[issue #105]. 287 | ===== 288 | 289 | == Producer API 290 | 291 | The object on which the Producer API is implemented does not need to be exclusive or otherwise limited to the Producer API. Producers are encouraged to implement the API as transparently as possible. 292 | 293 | [TIP] 294 | ===== 295 | For example, a testing framework that provides its main interface through a singleton or global object, could implement the Producer API within that interface. In QUnit, `producer.on()` is implemented as https://api.qunitjs.com/callbacks/QUnit.on/[QUnit.on()]. 296 | 297 | If the testing framework works through instantiation or through an "environment" instance (such as Jasmine), the Producer API could be implemented by such object instead. 298 | ===== 299 | 300 | === producer.on(eventName, callback) 301 | 302 | Register a callback to be called whenever the specified event is emitted, as described under <>. May be called multiple times, to register multiple callbacks for a given event. 303 | 304 | Parameters: 305 | 306 | * `string` **eventName**: Name of any CRI standard event. 307 | * `Function` **callback**: A callback function. 308 | 309 | Return: 310 | 311 | * `Mixed`: May be `undefined`, or any other value. 312 | 313 | [TIP] 314 | ===== 315 | The `on()` method does not need to be exclusive to CRI standard events. The same event emitter may support other events. 316 | 317 | In Node.js, the https://nodejs.org/api/events.html[built-in `events` module] provides an EventEmitter that could serve as the basis for a Producer API implementation. For example: 318 | 319 | [source,javascript] 320 | ---- 321 | const EventEmitter = require('events'); 322 | const producer = new EventEmitter(); 323 | 324 | // producer.emit('runStart', { … }); 325 | // producer.emit('runEnd', { … }); 326 | 327 | module.exports = producer; 328 | ---- 329 | ===== 330 | 331 | == Reporter API 332 | 333 | The Reporter API can be implemented in as a plain object, a class with static a method, or as exported function. 334 | 335 | === reporter.init(producer) 336 | 337 | Attach the reporter to the <>. 338 | 339 | Parameters: 340 | 341 | * <> **producer**: The main interface of the testing framework. 342 | 343 | Return: 344 | 345 | * `undefined`: Void. 346 | 347 | 348 | [cols="5a,5a"] 349 | |=== 350 | | Example: Class-based reporter | Example: Functional reporter 351 | 352 | | 353 | [source,javascript,indent=0] 354 | ---- 355 | class MyReporter { 356 | constructor (producer) { 357 | // producer.on(…, …); 358 | } 359 | 360 | static init (producer) { 361 | new MyReporter(producer); 362 | } 363 | } 364 | 365 | // CommonJS: 366 | module.exports = MyReporter; 367 | 368 | // ES Module: 369 | export default MyReporter; 370 | ---- 371 | | 372 | [source,javascript,indent=0] 373 | ---- 374 | function init (producer) { 375 | // producer.on(…, …); 376 | } 377 | 378 | // CommonJS: 379 | module.exports = { init: init }; 380 | 381 | // ES Module: 382 | export { init }; 383 | ---- 384 | 385 | // bogus line breaks to workaround vertical-align 386 |   + 387 | 388 |   + 389 | 390 |     391 | 392 | // … otherwise broken on GitHub's adoc renderer. 393 | 394 | |=== 395 | 396 | == Acknowledgments 397 | 398 | The editors would like to thank the following people for their contributions to the project: James M. Greene, Jörn Zaefferer, Franziska Carstens, Jiahao Guo, Florentin Simion, Nikhil Shagrithaya, Trent Willis, Kevin Partington, Martin Olsson, jeberger, Timo Tijhof, and Robert Jackson. 399 | 400 | This standard is written by Jörn Zaefferer, Timo Tijhof, Franziska Carstens, and Florentin Simion. 401 | 402 | Copyright JS Reporters. This text is licensed under the link:../LICENSE[MIT license]. 403 | --------------------------------------------------------------------------------