├── .gitignore
├── img
├── diff1-2.png
├── diff1-3.png
├── version1.png
├── version2.png
├── version3.png
└── testPyramid.png
├── sample_page
└── index.html
├── src
├── screenshot.js
├── pixeldiff.js
└── login.js
├── __tests__
├── analytics.test.js
├── desktop.test.js
├── mobile.test.js
├── login.test.js
├── form.test.js
└── seo.test.js
├── package.json
└── README.md
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | yarn.lock
3 | .history
--------------------------------------------------------------------------------
/img/diff1-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ilhan-mstf/puppeteer-investigation/master/img/diff1-2.png
--------------------------------------------------------------------------------
/img/diff1-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ilhan-mstf/puppeteer-investigation/master/img/diff1-3.png
--------------------------------------------------------------------------------
/img/version1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ilhan-mstf/puppeteer-investigation/master/img/version1.png
--------------------------------------------------------------------------------
/img/version2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ilhan-mstf/puppeteer-investigation/master/img/version2.png
--------------------------------------------------------------------------------
/img/version3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ilhan-mstf/puppeteer-investigation/master/img/version3.png
--------------------------------------------------------------------------------
/img/testPyramid.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ilhan-mstf/puppeteer-investigation/master/img/testPyramid.png
--------------------------------------------------------------------------------
/sample_page/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
8 |
9 |
10 | This is a sample title.
11 | This is a sample sub-title.
12 |
13 |
--------------------------------------------------------------------------------
/src/screenshot.js:
--------------------------------------------------------------------------------
1 | const puppeteer = require('puppeteer');
2 |
3 | (async () => {
4 | const browser = await puppeteer.launch();
5 | const page = await browser.newPage();
6 | await page.goto('http://localhost:8000/');
7 | await page.screenshot({path: 'version3.png'});
8 |
9 | await browser.close();
10 | })();
--------------------------------------------------------------------------------
/__tests__/analytics.test.js:
--------------------------------------------------------------------------------
1 | describe('Design is Dead - Analytics', () => {
2 | beforeAll(async () => {
3 | await page.goto('https://designisdead.com/')
4 | })
5 |
6 | it('should return google tag manager', async () => {
7 | const tagManager = await page.evaluate(() => google_tag_manager)
8 | expect(tagManager).toBeDefined()
9 | })
10 | })
--------------------------------------------------------------------------------
/__tests__/desktop.test.js:
--------------------------------------------------------------------------------
1 | describe('Design is Dead - Desktop', () => {
2 | beforeAll(async () => {
3 | await page.setViewport({ width: 1280, height: 768 })
4 | await page.goto('https://designisdead.com/')
5 | })
6 |
7 | it('should not render hamburger menu', async () => {
8 | await page.waitForSelector('.Page-hamburger', {
9 | hidden: true
10 | })
11 | })
12 | })
--------------------------------------------------------------------------------
/__tests__/mobile.test.js:
--------------------------------------------------------------------------------
1 | const devices = require('puppeteer/DeviceDescriptors');
2 | const iPhonex = devices['iPhone X'];
3 |
4 | describe('Design is Dead - Mobile', () => {
5 | beforeAll(async () => {
6 | await page.emulate(iPhonex)
7 | await page.goto('https://designisdead.com/')
8 | })
9 |
10 | it('should render hamburger menu', async () => {
11 | await page.waitForSelector('.Page-hamburger', {
12 | visible: true
13 | })
14 | })
15 | })
--------------------------------------------------------------------------------
/src/pixeldiff.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const PNG = require('pngjs').PNG;
3 | const pixelmatch = require('pixelmatch');
4 |
5 | const img1 = PNG.sync.read(fs.readFileSync('img/version1.png'));
6 | const img2 = PNG.sync.read(fs.readFileSync('img/version3.png'));
7 | const {width, height} = img1;
8 | const diff = new PNG({width, height});
9 |
10 | pixelmatch(img1.data, img2.data, diff.data, width, height, {threshold: 0.9});
11 |
12 | fs.writeFileSync('img/diff1-3.png', PNG.sync.write(diff));
--------------------------------------------------------------------------------
/src/login.js:
--------------------------------------------------------------------------------
1 | require('dotenv').config()
2 | const puppeteer = require('puppeteer')
3 | const screenshot = 'github.png';
4 | (async () => {
5 | const browser = await puppeteer.launch({headless: true})
6 | const page = await browser.newPage()
7 | await page.goto('https://github.com/login')
8 | await page.type('#login_field', process.env.GITHUB_USER)
9 | await page.type('#password', process.env.GITHUB_PWD)
10 | await page.click('[name="commit"]', {waitUntil: 'domcontentloaded'})
11 | await page.screenshot({ path: screenshot })
12 | browser.close()
13 | console.log('See screenshot: ' + screenshot)
14 | })()
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "puppeteer-investigation",
3 | "version": "0.0.1",
4 | "description": "An investigation of front-end testing with puppeteer",
5 | "main": "main.js",
6 | "scripts": {
7 | "test": "jest --maxWorkers=1"
8 | },
9 | "keywords": [
10 | "jest",
11 | "puppeteer",
12 | "test"
13 | ],
14 | "author": "mustafa ilhan",
15 | "license": "MIT",
16 | "devDependencies": {
17 | "jest": "^25.1.0",
18 | "jest-puppeteer": "^4.4.0",
19 | "pixelmatch": "^5.1.0",
20 | "puppeteer": "^2.1.1"
21 | },
22 | "jest": {
23 | "preset": "jest-puppeteer"
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/__tests__/login.test.js:
--------------------------------------------------------------------------------
1 | require('dotenv').config()
2 |
3 | describe('Github - Login', () => {
4 | beforeAll(async () => {
5 | await page.goto('https://github.com/login')
6 | })
7 |
8 | it('should log in and redirect', async () => {
9 | await page.type('#login_field', process.env.GITHUB_USER)
10 | await page.type('#password', process.env.GITHUB_PWD)
11 | await page.click('[name="commit"]', {waitUntil: 'domcontentloaded'})
12 | const uname = await page.$eval('#account-switcher-left > summary > span.css-truncate.css-truncate-target.ml-1', e => e.innerText)
13 |
14 | await expect(uname).toMatch(process.env.GITHUB_USER)
15 | })
16 | })
--------------------------------------------------------------------------------
/__tests__/form.test.js:
--------------------------------------------------------------------------------
1 | describe('Design is Dead - Layout', () => {
2 | beforeAll(async () => {
3 | await page.setViewport({ width: 1280, height: 768 })
4 | await page.goto('https://designisdead.com/')
5 | })
6 |
7 | it('should show error message when fields are empty', async () => {
8 | const contact = await page.waitForSelector(".Navigation-link.Navigation-contact")
9 | await contact.click()
10 | const submit = await page.waitForSelector("#_form_10_submit")
11 | await submit.click()
12 | const className = await page.evaluate(() => document.querySelector('input[name=fullname]').className)
13 |
14 | expect(className).toEqual(expect.stringContaining('_has_error'))
15 | })
16 | })
--------------------------------------------------------------------------------
/__tests__/seo.test.js:
--------------------------------------------------------------------------------
1 | describe('Design is Dead - SEO', () => {
2 | beforeAll(async () => {
3 | await page.goto('https://designisdead.com/')
4 | })
5 |
6 | it('should display "Design is Dead" text on title', async () => {
7 | await expect(page.title()).resolves.toMatch('Design is Dead')
8 | })
9 |
10 | it('should have description meta-tag', async () => {
11 | const descriptionContent = await page.$eval("head > meta[name='description']", element => element.content);
12 |
13 | expect(descriptionContent).toBeDefined();
14 | })
15 |
16 | it('should have a headline', async () => {
17 | const headlines = await page.$$('h1')
18 |
19 | expect(headlines.length).toBe(1)
20 | })
21 | })
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Automated tests with Puppeteer
2 | Software testing is a serious and required task to reach a certain quality. This blog post focuses on how it can be automated with Puppeteer. Before going into the details of Puppeteer, we should go over software testing and clarify the concepts related to it.
3 |
4 | ## Software Testing
5 | "Production-ready software requires testing before it goes into production" [1]. Here, one approach is to follow manual testing and the other one is to automate the whole testing process. "It's obvious that testing all changes manually is time-consuming, repetitive [, not scalable] and tedious. Repetitive is boring, boring leads to mistakes" [1] that we don't want. Therefore, automation is a great alternative for those repetitive tasks. Further, automation of tests can deliver requested development pace and reliability of the software product, especially on a massive scale. In addition to these, ensuring the reliability and stability of the software is also very important for the developers in order not to lose time on bugs created by changing code blocks. Specifically, it becomes very curial when team size increases.
6 |
7 | Test pyramid [2] is a key concept to follow when you want to write automated tests in your software. This concept defines how much tests you should add to your software for each level. There are three layers of this pyramid:
8 | - **Unit Tests**
9 | - Tests that cover isolated pieces of code, e.g. functions, etc.
10 | - **Service Tests**
11 | - Cohn's [2] naming convention about second level is not self-explanatory. This level generally is accepted as **Integration Tests** by the community. Integration Tests cover connected pieces of the application, e.g. database, filesystem, etc.
12 | - **User Interface Tests**
13 | - With the modern front-end frameworks such as React.js, Vue.js, Angular.js user interface tests can be accomplished in the level of unit tests. Therefore, it becomes harder to set the right level for User Interface Testing, it actually spreads to different levels. However, we can put **End-to-End Tests** to the higher level. It is different than to User Interface Testing since it covers whole journey of functionality from user interaction to services, e.g. login flow, purchase flow, etc.
14 |
15 |
18 |
19 | 
20 |
21 | The key points of this pyramid:
22 | - "Write tests with different granularity" [1]
23 | - "The more high-level you get the fewer tests you should have" [1]
24 |
25 | However, this approach may not suit for each case. In your software, for instance, you may have less business logic and more integration. In this case, you may want to add more integration tests to your software. Therefore, it is best to think of how many tests you should write for each granularity. (You can check this post for more discussion: [https://kentcdodds.com/blog/write-tests](https://kentcdodds.com/blog/write-tests))
26 |
27 |
28 |
29 | As this blog post is related to automated tests with Puppeteer, it can be used both end-to-end and user interface tests. Even if end-to-end tests are triggered from user interface, end-to-end tests covers whole flow and it doesn't have to check all possible cases on the UI.
30 |
31 |
34 |
35 | ### When do you really need end-to-end testing?
36 | End-to-end tests are useful to identify problems in user journeys. User journeys are the flows that a user can follow in your application. Therefore, the idea of end-to-end tests is to imitate a user's behavior in certain flows from start to end to ensure that everything works as expected. Running end-to-end tests are slower than to unit tests since they can touch many components of your application or third party services. As a result, we can think of them as expensive in terms of resources that reserved for them. For instance, assume that you want to run end-to-end tests of a web application, to accomplish this you need to run a browser and this browser consumes significant amount of memory. Furthermore, the initialization of the browser and the other components takes a lot of time. Hence, only relying on end-to-end tests is not preferable in terms of development efficiency. Therefore, you may want to write end-to-end tests for high-value interactions of your application such as checkout, login, etc.
37 |
38 | ## What is Puppeteer?
39 | [Puppeteer](https://pptr.dev/) provides a high-level API to control Chrome or Chromium programmatically. It is an open-source Nodejs library. Puppeteer runs headless (i.e. a browser that doesn't have a user interface) by default, but it can be configured to run full (non-headless) Chrome or Chromium.
40 |
41 | With this tool, you can run your web application on a browser and imitate user actions programmatically.
42 |
43 | ### Use cases:
44 | - Generate screenshots and PDFs of pages.
45 | - Crawl an SPA (Single-Page Application) and generate pre-rendered content (i.e. "SSR" (Server-Side Rendering)).
46 | - Automate form submission, UI testing, keyboard input, etc.
47 | - Create an up-to-date, automated testing environment. Run your tests directly in the latest version of Chrome using the latest JavaScript and browser features. Run regression and end-to-end tests.
48 | - Capture a timeline trace of your site to help diagnose performance issues.
49 | - Test Chrome Extensions.
50 | - Check for console logs and exceptions.
51 | - Replicate user activity.
52 |
53 | ### Jest and Puppeteer
54 | Puppeteer API is not designed for testing and it doesn't provide you the whole functionality of a testing framework. Therefore, it can be used with [Jest](https://jestjs.io/) JavaScript testing framework. The samples on this blog post use [jest-puppeteer](https://www.npmjs.com/package/jest-puppeteer) Nodejs library. It provides all required configuration for writing integration tests using Puppeteer with Jest.
55 |
56 | ### Samples
57 | This section provides a couple of examples to give you better insights into Puppeteer's usage. They don't cover all the features of Puppeteer and this blog post doesn't aim to give you detailed information of the Puppeteer API. You can build upon the given examples and explained concepts.
58 |
59 | The shown examples are also available at this [GitHub repository](https://github.com/ilhan-mstf/puppeteer-investigation).
60 |
61 | #### Prerequistes and Installation
62 | You need:
63 | - The recent stable version of node.js
64 | - The recent stable version of yarn or npm
65 |
66 | Examples have these dependencies:
67 |
68 | ```json
69 | "devDependencies": {
70 | "jest": "^25.1.0",
71 | "jest-puppeteer": "^4.4.0",
72 | "pixelmatch": "^5.1.0",
73 | "puppeteer": "^2.1.1"
74 | }
75 | ```
76 |
77 | `puppeteer` library downloads lastest Chrome executable. You can use `puppeteer-core` instead of `puppeteer` if you want to use existing Chrome executable in your system and pass the path of executable as a configuration option.
78 |
79 | You can run the tests with `yarn test` command.
80 |
81 | #### Taking Screenshot
82 | You can take screenshot of your website with different options such as setting viewport or emulated device.
83 |
84 | ```js
85 | const puppeteer = require('puppeteer');
86 |
87 | (async () => {
88 | const browser = await puppeteer.launch();
89 | const page = await browser.newPage();
90 | await page.goto('https://designisdead.com/');
91 | await page.screenshot({path: 'homepage.png'});
92 |
93 | await browser.close();
94 | })();
95 | ```
96 |
97 | #### Comparing Screenshots
98 | You can detect changes visually by taking screenshots of different versions of your application. You can take screenshots with Puppeteer but you need another tool to compare them. This sample uses [`pixelmatch` library](https://www.npmjs.com/package/pixelmatch).
99 |
100 | ```js
101 | const fs = require('fs');
102 | const PNG = require('pngjs').PNG;
103 | const pixelmatch = require('pixelmatch');
104 |
105 | const img1 = PNG.sync.read(fs.readFileSync('version1.png'));
106 | const img2 = PNG.sync.read(fs.readFileSync('version2.png'));
107 | const {width, height} = img1;
108 | const diff = new PNG({width, height});
109 |
110 | pixelmatch(img1.data, img2.data, diff.data, width, height, {threshold: 0.9});
111 |
112 | fs.writeFileSync('diff.png', PNG.sync.write(diff));
113 | ```
114 |
115 |
126 |
127 | |  |  |  |
128 | |:--:|:--:|:--:|
129 | | Version 1 | Version 2 | Diff 1-2 |
130 |
131 | |  |  |  |
132 | |:--:|:--:|:--:|
133 | | Version 1 | Version 3 | Diff 1-3 |
134 |
135 | Be aware that some of image comparision tools find differences by checking the pixel difference, therefore, if the text is changed, they will show it as a change. In other words, if you change your content, you will see it as a difference.
136 |
137 |
145 |
146 | #### Checking integration of third party applications
147 | You may use third party services, scripts, etc. in your application. Therefore, it will be a good idea to check that their integration with your application works as expected.
148 |
149 | ```js
150 | describe('Analytics', () => {
151 | beforeAll(async () => {
152 | await page.goto('https://designisdead.com/')
153 | })
154 |
155 | it('should return google tag manager', async () => {
156 | const tagManager = await page.evaluate(() => google_tag_manager)
157 | expect(tagManager).toBeDefined()
158 | })
159 | })
160 | ```
161 |
162 | #### Mobile and Desktop Layout
163 | Since there is a significant variance in screen sizes, there are a lot of cases that need to be tested here.
164 |
165 | ```js
166 | const devices = require('puppeteer/DeviceDescriptors');
167 | const iPhonex = devices['iPhone X'];
168 |
169 | describe('Mobile', () => {
170 | beforeAll(async () => {
171 | await page.emulate(iPhonex)
172 | await page.goto('https://designisdead.com/')
173 | })
174 |
175 | it('should render hamburger menu', async () => {
176 | await page.waitForSelector('.Page-hamburger', {
177 | visible: true
178 | })
179 | })
180 | })
181 | ```
182 |
183 | ```js
184 | describe('Desktop', () => {
185 | beforeAll(async () => {
186 | await page.setViewport({ width: 1280, height: 768 })
187 | await page.goto('https://designisdead.com/')
188 | })
189 |
190 | it('should not render hamburger menu', async () => {
191 | await page.waitForSelector('.Page-hamburger', {
192 | hidden: true
193 | })
194 | })
195 | })
196 | ```
197 |
198 | #### Seo checks
199 | Since search engines crawl your production website, it may be a good idea to check your pages' SEO performance. However, even if the below example handles this issue on test cases, you may want to generate a score and corresponding report.
200 |
201 | ```js
202 | describe('SEO', () => {
203 | beforeAll(async () => {
204 | await page.goto('https://designisdead.com/')
205 | })
206 |
207 | it('should display "Design is Dead" text on title', async () => {
208 | await expect(page.title()).resolves.toMatch('Design is Dead')
209 | })
210 |
211 | it('should have description meta-tag', async () => {
212 | const descriptionContent = await page.$eval("head > meta[name='description']", element => element.content);
213 |
214 | expect(descriptionContent).toBeDefined();
215 | })
216 |
217 | it('should have a headline', async () => {
218 | const headlines = await page.$$('h1')
219 |
220 | expect(headlines.length).toBe(1)
221 | })
222 | })
223 | ```
224 |
225 | #### Login
226 | One of the significant features of web applications is to log in user's account. It can be count as an example of End-to-End tests since back-end services involve to the process.
227 |
228 | ```js
229 | // put GITHUB_USER and GITHUB_PWD values to .env file
230 | require('dotenv').config()
231 |
232 | describe('Github - Login', () => {
233 | beforeAll(async () => {
234 | await page.goto('https://github.com/login')
235 | })
236 |
237 | it('should log in and redirect', async () => {
238 | await page.type('#login_field', process.env.GITHUB_USER)
239 | await page.type('#password', process.env.GITHUB_PWD)
240 | await page.click('[name="commit"]', {waitUntil: 'domcontentloaded'})
241 | const uname = await page.$eval('#account-switcher-left > summary > span.css-truncate.css-truncate-target.ml-1', e => e.innerText)
242 |
243 | await expect(uname).toMatch(process.env.GITHUB_USER)
244 | })
245 | })
246 | ```
247 |
248 | ## Selenium and other tools to automate browsers
249 | Puppeteer is not the only tool that provides higher level API to manage and automate browsers. [Playwright](https://www.npmjs.com/package/playwright) is an alternative Node library that supports Chromium, Firefox and WebKit. It is developed by the same team built Puppeteer and its API is very similar to Puppeteer. Another options is [Selenium](https://www.selenium.dev/). It supports all the major browsers. Futher, you can use Selenium with Java, Python, Ruby, C#, JavaScript, Perl and PHP.
250 |
251 | ## Conclusion
252 | In this blog, we go over the levels of software testing and discuss the use cases of end-to-end tests. Later, we look into the details of Puppeteer as an end-to-end test tool by giving some examples.
253 |
254 | ## References:
255 | - [1] https://martinfowler.com/articles/practical-test-pyramid.html
256 | - [2] Mike Cohn, Succeeding with Agile
257 |
258 | ## Suggested Readings:
259 | - https://martinfowler.com/bliki/TestPyramid.html
260 | - https://kentcdodds.com/blog/write-tests
261 | - https://www.freecodecamp.org/news/why-end-to-end-testing-is-important-for-your-team-cb7eb0ec1504/
262 | - https://www.symphonious.net/2015/04/30/making-end-to-end-tests-work/
263 | - https://blogs.dropbox.com/tech/2019/05/athena-our-automated-build-health-management-system/
264 | - https://medium.com/coursera-engineering/improving-end-to-end-testing-at-coursera-using-puppeteer-and-jest-5f1bac9cd176
265 | - Puppeteer example scripts: https://github.com/checkly/puppeteer-examples
266 | - Running Puppeteer on serverless: https://github.com/alixaxel/chrome-aws-lambda
267 | - https://medium.com/@ymcatar/visualization-on-steroid-using-headless-browser-to-auto-refresh-google-data-studio-dashboards-c195e68f10b
268 | - https://www.lambdatest.com/blog/why-selenium-automation-testing-in-production-is-pivotal-for-your-next-release/
269 |
270 | ## Acknowledgements
271 | Thanks to Kris Barnhoorn and Manon Erb for sharing their valuable ideas and checking draft version of this blog post.
272 |
273 | -- Mustafa İlhan, 2020, İzmir
274 |
275 |
276 |
--------------------------------------------------------------------------------