├── .gitignore ├── globals.js ├── package.json ├── LICENSE ├── commands ├── show.js └── hide.js ├── README.md └── assertions └── visualRegression.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /globals.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | visualRegression: { 3 | // Just a placeholder - doesn't actually work yet 4 | enabled: true, 5 | 6 | // Baseline screenshots stored here 7 | baselineFolder: 'nightwatch/screenshots/baseline', 8 | 9 | // Screenshots for the current run stored here 10 | currentFolder: 'nightwatch/screenshots/new', 11 | 12 | // Screenshots failing comparison stored here 13 | errorFolder: 'nightwatch/screenshots/failures', 14 | 15 | // If no selector is provided, default to this one 16 | defaultSelector: 'body', 17 | 18 | // Automatically hide any elements matching these selectors 19 | // Currently requires a call to `hide` 20 | censorSelectors: [ 21 | ], 22 | 23 | // Fail the regression assertion if the images are more than 24 | // this percent different 25 | mismatchTolerance: 0.3 26 | } 27 | }; 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nightwatch-visual-regression", 3 | "version": "1.0.0", 4 | "description": "Capture and compare screenshots within a given tolerance as a nightwatch assertion", 5 | "main": "assertions/visualRegression.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/Rykus0/nightwatch-visual-regression.git" 12 | }, 13 | "keywords": [ 14 | "nightwatch", 15 | "nightwatchjs", 16 | "regression", 17 | "screenshot", 18 | "resemble", 19 | "compare" 20 | ], 21 | "author": "Tom Pietrosanti", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/Rykus0/nightwatch-visual-regression/issues" 25 | }, 26 | "homepage": "https://github.com/Rykus0/nightwatch-visual-regression#readme", 27 | "dependencies": { 28 | "fs-extra": "^0.26.2", 29 | "gm": "^1.21.1", 30 | "node-resemble-js": "0.0.4", 31 | "path": "^0.12.7", 32 | "util": "^0.10.3" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Tom Pietrosanti 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /commands/show.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Make the given element(s) visible 3 | * Use after the `hide` command to restore visibility 4 | * @param {String} selector Selector for element(s) to show 5 | * @param {Function} callback Callback to invoke when finished 6 | * @return {Object} Return 'this' to allow chaining 7 | */ 8 | exports.command = function(selector, callback) { 9 | var options = this.globals.visualRegression 10 | var self = this; 11 | var ancestors = selector; 12 | var oElement; 13 | 14 | // If the selector comes from a section of a page object 15 | // selector will be an array of objects starting from the outermost 16 | // ancestor (section), and ending with the element 17 | // Join their selectors in order 18 | if( typeof ancestors !== 'string' ){ 19 | selector = ''; 20 | 21 | while( oElement = ancestors.shift() ){ 22 | selector += ' ' + oElement.selector; 23 | } 24 | } 25 | 26 | // Merge with global configuration 27 | if( options.censorSelectors ){ 28 | selector += ',' + options.censorSelectors.join(','); 29 | } 30 | 31 | this.execute(function(selector){ 32 | var els = document.querySelectorAll(selector); 33 | var i = els.length; 34 | 35 | while( i-- ){ 36 | els[i].style.visibility = ''; 37 | } 38 | }, [selector], function(){ 39 | if( typeof callback === 'function' ){ 40 | callback.call(this); 41 | } 42 | }); 43 | 44 | return this; 45 | }; -------------------------------------------------------------------------------- /commands/hide.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Make the given element(s) invisible 3 | * Useful for hiding dynamic/sensitive data from screenshots 4 | * @param {String} selector Selector for element(s) to hide 5 | * @param {Function} callback Callback to invoke when finished 6 | * @return {Object} Return 'this' to allow chaining 7 | */ 8 | exports.command = function(selector, callback) { 9 | var options = this.globals.visualRegression 10 | var self = this; 11 | var ancestors = selector; 12 | var oElement; 13 | 14 | // If the selector comes from a section of a page object 15 | // selector will be an array of objects starting from the outermost 16 | // ancestor (section), and ending with the element 17 | // Join their selectors in order 18 | if( typeof ancestors !== 'string' ){ 19 | selector = ''; 20 | 21 | while( oElement = ancestors.shift() ){ 22 | selector += ' ' + oElement.selector; 23 | } 24 | } 25 | 26 | // Merge with global configuration 27 | if( options.censorSelectors ){ 28 | selector += ',' + options.censorSelectors.join(','); 29 | } 30 | 31 | this.execute(function(selector){ 32 | var els = document.querySelectorAll(selector); 33 | var i = els.length; 34 | 35 | while( i-- ){ 36 | els[i].style.visibility = 'hidden'; 37 | } 38 | }, [selector], function(){ 39 | if( typeof callback === 'function' ){ 40 | callback.call(this); 41 | } 42 | }); 43 | 44 | return this; 45 | }; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Visual Regression Testing for Nightwatch 2 | ======================================== 3 | 4 | This is a NightwatchJS custom assertion and commands for capturing and comparing screenshots during testing. 5 | 6 | Node dependencies are listed in `package.json`. 7 | 8 | This project uses the `gm` module, and so requires that graphicsmagick or imagemagick are installed and configured on your system. 9 | 10 | Description 11 | ----------- 12 | 13 | The first time the assertion is run, a baseline image is saved. 14 | 15 | Subsequent runs will compare to these baseline images. If the mismatch percentage is more than the given threshold, the assertion fails. 16 | 17 | When the assertion fails, the visual diff is copied to a separate folder. 18 | 19 | Generated folders and images are organized in this way: 20 | `testPath/testName/browser_version_os/widthxheight/testLabel__selector--label.png` 21 | 22 | 23 | Installation 24 | ------------ 25 | 26 | * Install graphicsMagick (preferred) or imageMagick 27 | * Update your dependencies with those from package.json (or otherwise make sure they are included) and install 28 | * Copy the contents of `assertions` and `commands` to the corresponding folders in your nightwatchJS configuration or otherwise point to them in your nightwatch configuration (nightwatch.json) 29 | * Update your globals file with the configuration from globals.js (see the _Configuration_ section) 30 | 31 | 32 | Configuration 33 | ------------- 34 | 35 | Copy the contenst of `globals.js` into the file specified by `globals_path` in `nightwatch.json` 36 | 37 | See `globals.js` for configuration details. 38 | 39 | 40 | Usage 41 | ----- 42 | 43 | ```` 44 | module.exports = { 45 | 'My Test': function(client){ 46 | var page = client.page['myPage'](); 47 | 48 | page.navigate(); 49 | 50 | // Screenshot cropped to default selector 51 | client.assert.visualRegression(); 52 | 53 | // Screenshot cropped to contents of given selector 54 | client.assert.visualRegression('.my-component'); 55 | 56 | // Use page selector 57 | page.assert.visualRegression('@myForm'); 58 | 59 | page.click('@button'); 60 | 61 | // Add a custom label to discern identical screenshots 62 | page.assert.visualRegression('@myForm', 'afterButtonPress') 63 | 64 | // Hide Sensitive/Dynamic content from screenshot 65 | page.hide('@username', function(){ 66 | client.assert.visualRegression(); 67 | form.show('@username'); 68 | }); 69 | } 70 | }; 71 | ```` 72 | -------------------------------------------------------------------------------- /assertions/visualRegression.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var fse = require('fs-extra'); 3 | var path = require('path'); 4 | var resemble = require('node-resemble-js'); 5 | 6 | /** 7 | * @typedef {Object} Coordinates 8 | * @property {Number} x 9 | * @property {Number} y 10 | */ 11 | 12 | /** 13 | * @typedef {Object} Dimensions 14 | * @property {Number} width 15 | * @property {Number} height 16 | */ 17 | 18 | /** 19 | * Crops the screenshot to the bounding box of the given element 20 | * Note: This uses gm, which requires that you install graphicsMagick or imageMagick 21 | * @see https://www.npmjs.com/package/gm 22 | * @param {String} file Path and filename of the screenshot to crop 23 | * @param {Coordinates} origin Coordinates of upper-left corner of area to crop 24 | * @param {Dimensions} size 2-D Dimensions of the area to crop 25 | * @param {Function} cb Callback invoked when the process is finished 26 | * @return {Object} Returns 'this' to allow chaining 27 | */ 28 | var cropElement = function(file, origin, size, cb) { 29 | var gm = require('gm'); 30 | var self = this; 31 | 32 | // Crop the screenshot to desired element 33 | gm(file) 34 | .crop(size.width, size.height, origin.x, origin.y) 35 | .write(file, function(err){ 36 | // All Operations Finished, trigger callback 37 | if( typeof cb === "function" ){ 38 | cb.call(self); 39 | } 40 | 41 | if (err) { 42 | console.log('Failure cropping screenshot'); 43 | console.log(err); 44 | } 45 | }) 46 | ; 47 | 48 | return this; 49 | }; 50 | 51 | 52 | /** 53 | * Compares a screenshot to a preconfigured baseline 54 | * If the baseline doesn't exist, the current image is copied to the baseline 55 | * and the tests will pass 56 | * @param {String} screenshotFile Path and file of the new screenshot 57 | * @param {Function} callback Callback invoked when processing finishes 58 | * @return {Object} Return 'this' to allow chaining 59 | */ 60 | var compareToBaseline = function(screenshotFile, callback){ 61 | var options = this.globals.visualRegression; 62 | var diffFile = screenshotFile.replace(/(\.[a-zA-Z0-9]+)$/, '.diff$1'); 63 | var baseFilename = path.join(options.baselineFolder, screenshotFile); 64 | var newFilename = path.join(options.currentFolder, screenshotFile); 65 | var diffFilename = path.join(options.currentFolder, diffFile); 66 | var errorFilename = path.join(options.errorFolder, diffFile); 67 | 68 | var fNew; 69 | var fBase; 70 | var fDiff; 71 | 72 | var statBase; 73 | 74 | // fs.exists is deprecated, so we need to check file existance this way... 75 | try { 76 | statBase = fse.statSync(baseFilename); 77 | } catch(e) { 78 | // Baseline doesn't exist, new screenshot is baseline 79 | fse.copySync(newFilename, baseFilename); 80 | } 81 | 82 | try { 83 | fNew = fse.readFileSync(newFilename); 84 | fBase = fse.readFileSync(baseFilename); 85 | 86 | resemble(fNew) 87 | .compareTo(fBase) 88 | .ignoreAntialiasing() 89 | .onComplete(function(data){ 90 | // Write diff file 91 | fse.ensureFileSync(diffFilename); 92 | data.getDiffImage().pack().pipe(fse.createWriteStream(diffFilename)); 93 | 94 | try { 95 | // Remove previous error file, if it exists 96 | fse.removeSync(errorFilename); 97 | } catch(e) { 98 | //nothing 99 | } 100 | 101 | // Save a reference to the relative screenshot path for later use 102 | data.screenshotFile = screenshotFile; 103 | 104 | // Call callback 105 | callback.call(this, data); 106 | } 107 | ); 108 | } catch(e) { 109 | console.log('Failure during screenshot comparisson'); 110 | console.log(e); 111 | } 112 | 113 | return this; 114 | }; 115 | 116 | 117 | /** 118 | * This assertion compares a screenshot to a baseline 119 | * If the mismatch percentage is greater than the predefined tolerance, it fails 120 | * Failures are moved to a separate folder for easier reference 121 | * @param {String} [selector] Optionally crop the screenshot down to this element 122 | * @param {String} [label] Optional label to add to the filename 123 | * @param {String} [msg] Optionally override the assertion output message 124 | * @return {Object} Return 'this' for chaining 125 | */ 126 | exports.assertion = function(selector, label, msg) { 127 | var options = this.api.globals.visualRegression; 128 | var selElement = selector || options.defaultSelector; 129 | var self = this; 130 | 131 | // If the selector comes from a section of a page object 132 | // selector will be an array of objects starting from the outermost 133 | // ancestor (section), and ending with the element 134 | // Join their selectors in order 135 | if( selector && typeof selector !== 'string' ){ 136 | selElement = ''; 137 | 138 | for( var i = 0; i < selector.length; i++ ){ 139 | oElement = selector[i]; 140 | selElement += ' ' + oElement.selector; 141 | } 142 | } 143 | 144 | // Separate the screenshots by client 145 | var filepath = path.join( 146 | this.api.currentTest.module, // Test Name 147 | [ 148 | this.api.capabilities.browserName, // Browser 149 | this.api.capabilities.version, // Browser Version 150 | this.api.capabilities.platform // OS 151 | ].join('_') 152 | ); 153 | 154 | // Generate a filename unique to this test/element combination 155 | var filename = this.api.currentTest.name + // Test Step 156 | (selElement ? '__' + selElement.replace(/\W+/g, '_') : '') + // Element 157 | (label ? '--' + label : '') + // Custom Label 158 | '.png' // Extension 159 | ; 160 | 161 | /** 162 | * Compares a screenshot to a preconfigured baseline 163 | * If the baseline doesn't exist, the current image is copied to the baseline 164 | * and the tests will pass 165 | * @param {String} screenshotFile Path and file of the new screenshot 166 | * @param {Function} callback Callback invoked when processing finishes 167 | * @return {Object} Return 'this' to allow chaining 168 | */ 169 | var compareToBaseline = function(screenshotFile, callback){ 170 | var options = this.globals.visualRegression; 171 | var diffFile = screenshotFile.replace(/(\.[a-zA-Z0-9]+)$/, '.diff$1'); 172 | var baseFilename = path.join(options.baselineFolder, screenshotFile); 173 | var newFilename = path.join(options.currentFolder, screenshotFile); 174 | var diffFilename = path.join(options.currentFolder, diffFile); 175 | var errorFilename = path.join(options.errorFolder, diffFile); 176 | 177 | var fNew; 178 | var fBase; 179 | var fDiff; 180 | 181 | var statBase; 182 | 183 | // fs.exists is deprecated, so we need to check file existance this way... 184 | try { 185 | statBase = fse.statSync(baseFilename); 186 | } catch(e) { 187 | // Baseline doesn't exist, new screenshot is baseline 188 | fse.copySync(newFilename, baseFilename); 189 | } 190 | 191 | try { 192 | fNew = fse.readFileSync(newFilename); 193 | fBase = fse.readFileSync(baseFilename); 194 | 195 | resemble(fNew) 196 | .compareTo(fBase) 197 | .ignoreAntialiasing() 198 | .onComplete(function(data){ 199 | // Write diff file 200 | fse.ensureFileSync(diffFilename); 201 | data.getDiffImage().pack().pipe(fse.createWriteStream(diffFilename)); 202 | 203 | try { 204 | // Remove previous error file, if it exists 205 | fse.removeSync(errorFilename); 206 | } catch(e) { 207 | //nothing 208 | } 209 | 210 | if (!self.pass(self.value(data))) { 211 | // On a failure, save the diff file to the error folder 212 | // Already packed earlier, when diff file written 213 | try { 214 | fse.ensureFileSync(errorFilename); 215 | data.getDiffImage().pipe(fse.createWriteStream(errorFilename)); 216 | } catch(e) { 217 | console.log(e); 218 | } 219 | 220 | self.message = util.format('Visual Regression: Screen differs by %s% (see: %s)', self.value(data), errorFilename); 221 | } 222 | 223 | // Call callback 224 | callback.call(this, data); 225 | } 226 | ); 227 | } catch(e) { 228 | console.log('Failure during screenshot comparisson'); 229 | console.log(e); 230 | } 231 | }; 232 | 233 | this.expected = options.mismatchTolerance; 234 | this.message = msg || util.format('Visual Regression: "%s" change is less than %s%', selElement, this.expected); 235 | 236 | this.pass = function(value) { 237 | // Mismatch percentage is within tolerance 238 | return value < this.expected; 239 | }; 240 | 241 | this.value = function(result) { 242 | // Return mismatch percentage 243 | return result.misMatchPercentage; 244 | }; 245 | 246 | this.command = function(callback) { 247 | // Get handle of active window 248 | // Needed to get window size 249 | this.api.windowHandle(function(response){ 250 | var wHandle = response.value; 251 | 252 | // Get size of active window 253 | // Note we need to drop 'this.api' and use just 'this' 254 | this.windowSize(wHandle, function(response){ 255 | var wSize = response.value; 256 | var wDimensions = wSize.width + 'x' + wSize.height; 257 | var file; 258 | 259 | // Join the final piece of the path (dimensions) to create 260 | // the full file pathname relative to the base folder 261 | file = path.join( filepath, wDimensions, filename ); 262 | 263 | // For capture/crop, work in the 'new' folder 264 | // For compare, only pass the relative portion of the filename 265 | if( selElement ){ 266 | 267 | // This was failing called within the saveScreenshot callback, so moved here 268 | this.getLocationInView(selector||selElement, function(results){ 269 | var origin = results.value; 270 | 271 | this.getElementSize(selector || selElement, function(results){ 272 | // Get width and height of the element 273 | var size = results.value; 274 | 275 | this.saveScreenshot(path.join(options.currentFolder , file), function(){ 276 | // use selector if defined - may be from a page object 277 | // otherwise, use the element selector string 278 | cropElement.call(this, path.join(options.currentFolder , file), origin, size, function(){ 279 | compareToBaseline.call(this, file, callback); 280 | }); 281 | }); 282 | }); 283 | }); 284 | 285 | } else { 286 | this.saveScreenshot.call(this, path.join(options.currentFolder , file), function(){ 287 | compareToBaseline.call(this, file, callback); 288 | }); 289 | } 290 | 291 | }); 292 | }); 293 | }; 294 | 295 | }; 296 | --------------------------------------------------------------------------------