} List of anchor hrefs.
78 | */
79 | function collectAllSameOriginAnchorsDeep(sameOrigin = true) {
80 | const allElements = [];
81 |
82 | const findAllElements = function(nodes) {
83 | for (let i = 0, el; el = nodes[i]; ++i) {
84 | allElements.push(el);
85 | // If the element has a shadow root, dig deeper.
86 | if (el.shadowRoot) {
87 | findAllElements(el.shadowRoot.querySelectorAll('*'));
88 | }
89 | }
90 | };
91 |
92 | findAllElements(document.querySelectorAll('*'));
93 |
94 | const filtered = allElements
95 | .filter(el => el.localName === 'a' && el.href) // element is an anchor with an href.
96 | .filter(el => el.href !== location.href) // link doesn't point to page's own URL.
97 | .filter(el => {
98 | if (sameOrigin) {
99 | return new URL(location).origin === new URL(el.href).origin;
100 | }
101 | return true;
102 | })
103 | .map(a => a.href);
104 |
105 | return Array.from(new Set(filtered));
106 | }
107 |
108 | /**
109 | * Crawls a URL by visiting an url, then recursively visiting any child subpages.
110 | * @param {!Browser} browser
111 | * @param {{url: string, title: string, img?: string, children: !Array}} page Current page.
112 | * @param {number=} depth Current subtree depth of crawl.
113 | */
114 | async function crawl(browser, page, depth = 0) {
115 | if (depth > maxDepth) {
116 | return;
117 | }
118 |
119 | // If we've already crawled the URL, we know its children.
120 | if (crawledPages.has(page.url)) {
121 | console.log(`Reusing route: ${page.url}`);
122 | const item = crawledPages.get(page.url);
123 | page.title = item.title;
124 | page.img = item.img;
125 | page.children = item.children;
126 | // Fill in the children with details (if they already exist).
127 | page.children.forEach(c => {
128 | const item = crawledPages.get(c.url);
129 | c.title = item ? item.title : '';
130 | c.img = item ? item.img : null;
131 | });
132 | return;
133 | } else {
134 | console.log(`Loading: ${page.url}`);
135 |
136 | const newPage = await browser.newPage();
137 | await newPage.goto(page.url, {waitUntil: 'networkidle2'});
138 |
139 | let anchors = await newPage.evaluate(collectAllSameOriginAnchorsDeep);
140 | anchors = anchors.filter(a => a !== URL) // link doesn't point to start url of crawl.
141 |
142 | page.title = await newPage.evaluate('document.title');
143 | page.children = anchors.map(url => ({url}));
144 |
145 | if (SCREENSHOTS) {
146 | const path = `./${OUT_DIR}/${slugify(page.url)}.png`;
147 | let imgBuff = await newPage.screenshot({fullPage: false});
148 | imgBuff = await sharp(imgBuff).resize(null, 150).toBuffer(); // resize image to 150 x auto.
149 | util.promisify(fs.writeFile)(path, imgBuff); // async
150 | page.img = `data:img/png;base64,${imgBuff.toString('base64')}`;
151 | }
152 |
153 | crawledPages.set(page.url, page); // cache it.
154 |
155 | await newPage.close();
156 | }
157 |
158 | // Crawl subpages.
159 | for (const childPage of page.children) {
160 | await crawl(browser, childPage, depth + 1);
161 | }
162 | }
163 |
164 | (async() => {
165 |
166 | mkdirSync(OUT_DIR); // create output dir if it doesn't exist.
167 | await del([`${OUT_DIR}/*`]); // cleanup after last run.
168 |
169 | const browser = await puppeteer.launch();
170 | const page = await browser.newPage();
171 | if (VIEWPORT) {
172 | await page.setViewport(VIEWPORT);
173 | }
174 |
175 | const root = {url: URL};
176 | await crawl(browser, root);
177 |
178 | await util.promisify(fs.writeFile)(`./${OUT_DIR}/crawl.json`, JSON.stringify(root, null, ' '));
179 |
180 | await browser.close();
181 |
182 | })();
183 |
184 |
--------------------------------------------------------------------------------
/detect_sound.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Trivially detect if media elements on the page are producing audio on page load.
21 | * Note: approach doesn't work in headless Chrome (which doesn't play sound).
22 | */
23 |
24 | const puppeteer = require('puppeteer');
25 |
26 | const URL = process.env.URL || 'https://www.youtube.com/watch?v=sK1ODp0nDbM';
27 |
28 | (async() => {
29 |
30 | // Note: headless doesn't play audio. Launch headful chrome.
31 | const browser = await puppeteer.launch({headless: false});
32 |
33 | const page = await browser.newPage();
34 | await page.goto(URL, {waitUntil: 'networkidle2'});
35 |
36 | const playingAudio = await page.evaluate(() => {
37 | const mediaEls = Array.from(document.querySelectorAll('audio,video'));
38 | return mediaEls.every(el => el.duration > 0 && !el.paused);
39 | });
40 |
41 | console.log('Playing audio:', playingAudio);
42 |
43 | await browser.close();
44 |
45 | })();
46 |
--------------------------------------------------------------------------------
/element-to-pdf.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Takes a screenshot of the latest tweet in a user's timeline and creates a
21 | * PDF of it. Shows how to use Puppeteer to:
22 | *
23 | * 1. screenshot a DOM element
24 | * 2. craft an HTML page on-the-fly
25 | * 3. produce an image of the element and PDF of the page with the image embedded
26 | *
27 | * Usage:
28 | * node element-to-pdf.js
29 | * USERNAME=ChromiumDev node element-to-pdf.js
30 | *
31 | * --searchable makes "find in page" work:
32 | * node element-to-pdf.js --searchable
33 | *
34 | * Output:
35 | * tweet.png and tweet.pdf
36 | */
37 | const puppeteer = require('puppeteer');
38 |
39 | const username = process.env.USERNAME || 'ebidel';
40 | const searchable = process.argv.includes('--searchable');
41 |
42 | (async() => {
43 |
44 | const browser = await puppeteer.launch();
45 |
46 | const page = await browser.newPage();
47 | await page.setViewport({width: 1200, height: 800, deviceScaleFactor: 2});
48 | await page.goto(`https://twitter.com/${username}`);
49 |
50 | // Can't use elementHandle.click() because it clicks the center of the element
51 | // with the mouse. On tweets like https://twitter.com/ebidel/status/915996563234631680
52 | // there is an embedded card link to another tweet that it clicks.
53 | await page.$eval(`.tweet[data-screen-name="${username}"]`, tweet => tweet.click());
54 | await page.waitForSelector('.tweet.permalink-tweet', {visible: true});
55 |
56 | const overlay = await page.$('.tweet.permalink-tweet');
57 | const screenshot = await overlay.screenshot({path: 'tweet.png'});
58 |
59 | if (searchable) {
60 | await page.evaluate(tweet => {
61 | const width = getComputedStyle(tweet).width;
62 | tweet = tweet.cloneNode(true);
63 | tweet.style.width = width;
64 | document.body.innerHTML = `
65 | ;
66 | ${tweet.outerHTML}
67 |
68 | `;
69 | }, overlay);
70 | } else {
71 | await page.setContent(`
72 |
73 |
74 |
75 |
90 |
91 |
92 | })
93 |
94 |
95 | `);
96 | }
97 |
98 | await page.pdf({path: 'tweet.pdf', printBackground: true});
99 |
100 | await browser.close();
101 |
102 | })();
103 |
--------------------------------------------------------------------------------
/fullscreen.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Launch a URL in full screen.
21 | */
22 |
23 | const puppeteer = require('puppeteer');
24 |
25 | const url = process.env.URL || 'https://news.ycombinator.com/';
26 |
27 | (async() => {
28 |
29 | const browser = await puppeteer.launch({
30 | headless: false,
31 | // See flags at https://peter.sh/experiments/chromium-command-line-switches/.
32 | args: [
33 | '--disable-infobars', // Removes the butter bar.
34 | '--start-maximized',
35 | // '--start-fullscreen',
36 | // '--window-size=1920,1080',
37 | // '--kiosk',
38 | ],
39 | });
40 |
41 | const page = await browser.newPage();
42 | await page.setViewport({width: 1920, height: 1080});
43 | await page.goto(url);
44 | await page.evaluate('document.documentElement.webkitRequestFullscreen()');
45 | await page.waitFor(5000);
46 |
47 | await browser.close();
48 | })();
--------------------------------------------------------------------------------
/google_search_features.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Prints a non-exhaustive list of HTML/JS/CSS features a page is using which
21 | * are not supported by the Google Search bot and "Render as Google"
22 | * (runs Chrome 41). See developers.google.com/search/docs/guides/rendering.
23 | */
24 |
25 | const fs = require('fs');
26 | const puppeteer = require('puppeteer');
27 | const fetch = require('node-fetch');
28 | const chalk = require('chalk');
29 | const caniuseDB = require('caniuse-db/data.json').data;
30 |
31 | const url = process.env.URL || 'https://www.chromestatus.com/features';
32 |
33 | const GOOGLE_SEARCH_CHROME_VERSION = process.env.CHROME_VERSION || 41;
34 |
35 | const BlinkFeatureNameToCaniuseName = {
36 | AddEventListenerPassiveTrue: 'passive-event-listener',
37 | AddEventListenerPassiveFalse: 'passive-event-listener',
38 | PromiseConstructor: 'promises',
39 | PromiseResolve: 'promises',
40 | PromiseReject: 'promises',
41 | V8PromiseChain: 'promises',
42 | DocumentRegisterElement: 'custom-elements',
43 | V0CustomElementsRegisterHTMLCustomTag: 'custom-elements',
44 | V0CustomElementsCreateCustomTagElement: 'custom-elements',
45 | V0CustomElementsRegisterHTMLTypeExtension: 'custom-elements',
46 | V0CustomElementsCreateTypeExtensionElement: 'custom-elements',
47 | CSSSelectorPseudoMatches: 'css-matches-pseudo',
48 | CustomElementRegistryDefine: 'custom-elementsv1',
49 | ElementAttachShadow: 'shadowdomv1',
50 | ElementAttachShadowOpen: 'shadowdomv1',
51 | ElementAttachShadowClosed: 'shadowdomv1',
52 | CSSSelectorPseudoSlotted: 'shadowdomv1',
53 | HTMLSlotElement: 'shadowdomv1',
54 | CSSSelectorPseudoHost: 'shadowdom',
55 | ElementCreateShadowRoot: 'shadowdom',
56 | CSSSelectorPseudoShadow: 'shadowdom',
57 | CSSSelectorPseudoContent: 'shadowdom',
58 | CSSSelectorPseudoHostContext: 'shadowdom',
59 | HTMLShadowElement: 'shadowdom',
60 | HTMLContentElement: 'shadowdom',
61 | LinkRelPreconnect: 'link-rel-preconnect',
62 | LinkRelPreload: 'link-rel-preload',
63 | HTMLImports: 'imports',
64 | HTMLImportsAsyncAttribute: 'imports',
65 | LinkRelModulePreload: 'es6-module',
66 | V8BroadcastChannel_Constructor: 'broadcastchannel',
67 | Fetch: 'fetch',
68 | GlobalCacheStorage: 'cachestorage', // missing: https://github.com/Fyrd/caniuse/issues/3122
69 | OffMainThreadFetch: 'fetch',
70 | IntersectionObserver_Constructor: 'intersectionobserver',
71 | V8Window_RequestIdleCallback_Method: 'requestidlecallback',
72 | NotificationPermission: 'notifications',
73 | UnprefixedPerformanceTimeline: 'user-timing',
74 | V8Element_GetBoundingClientRect_Method: 'getboundingclientrect',
75 | AddEventListenerThirdArgumentIsObject: 'once-event-listener', // TODO: not a perfect match.
76 | // TODO: appears to be no UMA tracking for classes, async/await, spread, and
77 | // other newer js features. Those aren't being caught here.
78 | contain: 'css-containment',
79 | 'tab-size': 'css3-tabsize',
80 | // Explicitly disabled by search https://developers.google.com/search/docs/guides/rendering
81 | UnprefixedIndexedDB: 'indexeddb',
82 | DocumentCreateEventWebGLContextEvent: 'webgl',
83 | CSSGridLayout: 'css-grid',
84 | CSSValueDisplayContents: 'css-display-contents',
85 | CSSPaintFunction: 'css-paint-api',
86 | WorkerStart: 'webworkers',
87 | ServiceWorkerControlledPage: 'serviceworkers',
88 | PrepareModuleScript: 'es6-module',
89 | // CookieGet:
90 | // CookieSet
91 | };
92 |
93 | /**
94 | * Unique items based on obj property.
95 | * @param {!Array} items
96 | * @param {string} propName Property name to filter on.
97 | * @return {!Array} unique array of items
98 | */
99 | function uniqueByProperty(items, propName) {
100 | const posts = Array.from(items.reduce((map, item) => {
101 | return map.set(item[propName], item);
102 | }, new Map()).values());
103 | return posts;
104 | }
105 |
106 | /**
107 | * Sorts array of features by their name
108 | * @param {!Object} a
109 | * @param {!Object} b
110 | */
111 | function sortByName(a, b) {
112 | if (a.name < b.name) {
113 | return -1;
114 | }
115 | if (a.name > b.name) {
116 | return 1;
117 | }
118 | return 0;
119 | }
120 |
121 | function printHeader(usage) {
122 | console.log('');
123 | console.log(`${chalk.bold(chalk.yellow('CAREFUL'))}: using ${usage.FeatureFirstUsed.length} HTML/JS, ${usage.CSSFirstUsed.length} CSS features. Some features are ${chalk.underline('not')} supported by the Google Search crawler.`);
124 | console.log(`The bot runs ${chalk.redBright('Chrome ' + GOOGLE_SEARCH_CHROME_VERSION)}, which may not render your page correctly when it's being indexed.`);
125 | console.log('');
126 | console.log(chalk.dim('More info at https://developers.google.com/search/docs/guides/rendering.'));
127 | console.log('');
128 | console.log(`Features used which are not supported by Google Search:`);
129 | console.log('');
130 | }
131 |
132 | /**
133 | * Returns true if `feature` is supported by the Google Search bot.
134 | * @param {string} feature caniuse.com feature name/id.
135 | * @return {boolean} True if the feature is (likely) supported by Google Search.
136 | */
137 | function supportedByGoogleSearch(feature) {
138 | const data = caniuseDB[feature];
139 | if (!data) {
140 | return null;
141 | }
142 | const support = data.stats.chrome[GOOGLE_SEARCH_CHROME_VERSION];
143 | return support === 'y'; // TODO: consider 'p'. Partial support / polyfill.
144 | }
145 |
146 | /**
147 | * Fetches HTML/JS feature id/names from chromestatus.com.
148 | * @param {!Browser} browser
149 | * @return {!Map} key/val pairs of ids -> feature name
150 | */
151 | async function fetchFeatureToNameMapping() {
152 | const resp = await fetch('https://www.chromestatus.com/data/blink/features');
153 | return new Map(await resp.json());
154 | }
155 |
156 | /**
157 | * Fetches CSS property id/names from chromestatus.com
158 | * @param {!Browser} browser
159 | * @return {!Map} key/val pairs of ids -> feature name
160 | */
161 | async function fetchCSSFeatureToNameMapping(browser) {
162 | const resp = await fetch('https://www.chromestatus.com/data/blink/cssprops');
163 | return new Map(await resp.json());
164 | }
165 |
166 | /**
167 | * Start a trace during load to capture web platform features used by the page.
168 | * @param {!Browser} browser
169 | * @return {!Object}
170 | */
171 | async function collectFeatureTraceEvents(browser) {
172 | const page = await browser.newPage();
173 |
174 | console.log(chalk.cyan(`Trace started.`));
175 |
176 | await page.tracing.start({
177 | categories: [
178 | '-*',
179 | 'disabled-by-default-devtools.timeline', // for TracingStartedInPage
180 | 'disabled-by-default-blink.feature_usage'
181 | ],
182 | });
183 | console.log(chalk.cyan(`Navigating to ${url}`));
184 | await page.goto(url, {waitUntil: 'networkidle2'});
185 | console.log(chalk.cyan(`Waiting for page to be idle...`));
186 | await page.waitFor(5000); // add a little more time in case other features are used.
187 | const trace = JSON.parse(await page.tracing.stop());
188 | console.log(chalk.cyan(`Trace complete.`));
189 |
190 | // Filter out all trace events that aren't 1. blink feature usage
191 | // and 2. from the same process/thread id as our test page's main thread.
192 | const traceStartEvent = findTraceStartEvent(trace.traceEvents);
193 | const events = trace.traceEvents.filter(e => {
194 | return e.cat === 'disabled-by-default-blink.feature_usage' &&
195 | e.pid === traceStartEvent.pid && e.tid === traceStartEvent.tid;
196 | });
197 |
198 | // // Gut check.
199 | // console.assert(events.every((entry, i, arr) => {
200 | // // const nextIdx = Math.min(i + 1, arr.length - 1);
201 | // // return entry.pid === arr[nextIdx].pid && entry.tid === arr[nextIdx].tid;
202 | // return entry.pid === traceStartEvent.pid && entry.tid === traceStartEvent.tid;
203 | // }), 'Trace event is not from the same process/thread id as the page being tested.');
204 |
205 | await page.close();
206 |
207 | return events;
208 | }
209 |
210 | /**
211 | * @param {Array} events
212 | * @return {Object}
213 | */
214 | function findTraceStartEvent(events) {
215 | const startedInBrowserEvt = events.find(e => e.name === 'TracingStartedInBrowser');
216 | if (startedInBrowserEvt && startedInBrowserEvt.args.data && startedInBrowserEvt.args.data.frames) {
217 | const mainFrame = startedInBrowserEvt.args.data.frames.find(frame => !frame.parent);
218 | const pid = mainFrame && mainFrame.processId;
219 | const threadNameEvt = events.find(e => e.pid === pid && e.ph === 'M' &&
220 | e.cat === '__metadata' && e.name === 'thread_name' && e.args.name === 'CrRendererMain');
221 |
222 | const tid = threadNameEvt && threadNameEvt.tid;
223 | if (pid && tid) {
224 | return {
225 | pid,
226 | tid
227 | };
228 | }
229 | }
230 |
231 | // // Support legacy browser versions
232 | const startedInPageEvt = events.find(e => e.name === 'TracingStartedInPage');
233 | if (startedInPageEvt && startedInPageEvt.args && startedInPageEvt.args.data) {
234 | return {
235 | pid: startedInPageEvt.pid,
236 | tid: startedInPageEvt.tid
237 | };
238 | }
239 | }
240 |
241 | /**
242 | * @param {!Object} feature
243 | */
244 | function printFeatureName(feature, url= null) {
245 | const suffix = url ? `: ${url}` : '';
246 | if (feature.css) {
247 | console.log(chalk.grey('-'), `CSS \`${feature.name}\`${suffix}`);
248 | } else {
249 | console.log(chalk.grey('-'), `${feature.name}${suffix}`);
250 | }
251 | }
252 |
253 | (async() => {
254 |
255 | const browser = await puppeteer.launch({
256 | // headless: false,
257 | });
258 |
259 | // Parallelize the separate page loads.
260 | const [featureMap, cssFeatureMap, traceEvents] = await Promise.all([
261 | fetchFeatureToNameMapping(),
262 | fetchCSSFeatureToNameMapping(),
263 | collectFeatureTraceEvents(browser),
264 | ]);
265 |
266 | const usage = traceEvents.reduce((usage, e) => {
267 | if (!(e.name in usage)) {
268 | usage[e.name] = [];
269 | }
270 | const isCSS = e.name === 'CSSFirstUsed';
271 | const id = e.args.feature;
272 | const name = isCSS ? cssFeatureMap.get(id) : featureMap.get(id);
273 | usage[e.name].push({id, name, ts: e.ts, css: isCSS});
274 |
275 | return usage;
276 | }, {});
277 |
278 | // Unique events based on feature property id.
279 | usage.FeatureFirstUsed = uniqueByProperty(usage.FeatureFirstUsed, 'id');
280 | usage.CSSFirstUsed = uniqueByProperty(usage.CSSFirstUsed, 'id');
281 |
282 | printHeader(usage);
283 |
284 | const allFeaturesUsed = Object.entries([...usage.FeatureFirstUsed, ...usage.CSSFirstUsed].sort(sortByName));
285 | for (const [id, feature] of allFeaturesUsed) {
286 | const caniuseName = BlinkFeatureNameToCaniuseName[feature.name];
287 | const supported = supportedByGoogleSearch(caniuseName);
288 | if (caniuseName && !supported) {
289 | const url = chalk.magentaBright(`https://caniuse.com/#feat=${caniuseName}`);
290 | printFeatureName(feature, url);
291 | }
292 | }
293 | console.log('');
294 | console.log('All features used on the page:');
295 | console.log('');
296 | for (const [id, feature] of allFeaturesUsed) {
297 | printFeatureName(feature);
298 | }
299 | console.log('');
300 |
301 | await browser.close();
302 |
303 | })();
304 |
--------------------------------------------------------------------------------
/hash_navigation.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Hash (#) changes aren't considered navigations in Chrome. This makes it
21 | * tricky to test a SPA that use hashes to change views.
22 | *
23 | * This script shows how to observe the view of a SPA changing in Puppeteer
24 | * by injecting code into the page that listens for `hashchange` events.
25 | *
26 | * To run:
27 | * 1. Start a web server in this folder on port 5000.
28 | * 2. node hash_navigation.js
29 | */
30 |
31 | const {URL} = require('url');
32 | const puppeteer = require('puppeteer');
33 |
34 | const url = 'http://localhost:5000/html/spa.html';
35 |
36 | async function printVisibleView(page) {
37 | console.log('Visible panel:', await page.$eval(':target', el => el.textContent));
38 | }
39 |
40 | async function main() {
41 | const browser = await puppeteer.launch();
42 |
43 | const page = await browser.newPage();
44 |
45 | // Catch + "forward" hashchange events from page to node puppeteer.
46 | await page.exposeFunction('onHashChange', url => page.emit('hashchange', url));
47 | await page.evaluateOnNewDocument(() => {
48 | addEventListener('hashchange', e => onHashChange(location.href));
49 | });
50 |
51 | // Listen for hashchange events in node Puppeteer code.
52 | page.on('hashchange', url => console.log('hashchange event:', new URL(url).hash));
53 |
54 | await page.goto(url);
55 | await page.waitForSelector('[data-page="#page1"]'); // wait for view 1 to be "loaded".
56 | await printVisibleView(page);
57 |
58 | try {
59 | // "Navigate" to view 2 in SPA. We don't want to wait for the `load` event,
60 | // so set a small timeout and catch the "navigation timeout".
61 | await page.goto(`${url}#page2`, {timeout: 1});
62 | } catch (err) {
63 | // noop
64 | }
65 |
66 | await page.waitForSelector('[data-page="#page2"]'); // wait for view 2 to be "loaded".
67 | await printVisibleView(page);
68 |
69 | await browser.close();
70 | }
71 |
72 | main();
--------------------------------------------------------------------------------
/html/d3tree.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
28 |
29 |
30 |
200 |
201 |
202 | 
37 | 
12 |
13 | ### [verify_sw_caching.js](./verify_sw_caching.js)
14 |
15 | Verify all the resources you expect are being cached by a service worker for offline.
16 |
17 | 
18 |
19 | ### [google_search_features.js](./google_search_features.js)
20 |
21 | Gut check your page to make sure it renders correctly for Google Search.
22 |
23 | 
24 |
25 | ### [lazyimages_without_scroll_events.js](./lazyimages_without_scroll_events.js)
26 |
27 | Determine if your lazy loaded images will be seen correctly by Google Search.
28 |
29 | 
30 |
31 | ### [speech.js](./speech.js)
32 |
33 | Make your browser talk.
34 |
35 |
36 | 
45 |
46 |
47 | ### [element-to-pdf.js](./element-to-pdf.js)
48 |
49 | Turn a DOM element into a PDF.
50 |
51 | 
52 |
53 | ### [crawlsite.js](./crawlsite.js)
54 |
55 | Discover all the URLs on a site and visualize the subpages.
56 |
57 | 
58 |
59 | ### [side-by-side-pageload.js](./side-by-side-pageload.js)
60 |
61 | Load 2 or more pages side-by-side to visually see the difference in page load. Optional desktop viewport and throttling settings.
62 |
63 | 
64 |
65 | ### License
66 |
67 | [Apache 2.0](./LICENSE) © 2018 Google Inc.
68 |
--------------------------------------------------------------------------------
/cdp/raw_websocket.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Shows how to use raw web sockets to send messages to the page using
21 | * the DevTools protocol.
22 | * See https://chromedevtools.github.io/devtools-protocol/
23 | */
24 |
25 | const {URL} = require('url');
26 | const WebSocket = require('ws');
27 | const puppeteer = require('puppeteer');
28 |
29 | const url = process.env.URL || 'https://example.com';
30 |
31 | (async() => {
32 |
33 | const browser = await puppeteer.launch();
34 | const page = await browser.newPage();
35 |
36 | // // 1. createCDPSession() is the easiest way to work with the raw DTP:
37 | // const client = await page.target().createCDPSession();
38 | // const version = await client.send('Browser.getVersion');
39 | // console.log(version);
40 |
41 | // 2.But you can also use raw websockets...
42 |
43 | // Pull page's ws:// debugging url.
44 | const wsUrl = new URL(browser.wsEndpoint());
45 | const resp = await page.goto(`http://localhost:${wsUrl.port}/json`);
46 | const {webSocketDebuggerUrl} = (await resp.json())[0];
47 |
48 | await page.goto(url); // Navigate to actual page.
49 |
50 | // Connect to page's ws:// endpoint.
51 | const ws = new WebSocket(webSocketDebuggerUrl).on('open', () => {
52 | ws.send(JSON.stringify({
53 | id: 1,
54 | method: 'Runtime.evaluate',
55 | params: {expression: 'document.title'} // return the page title.
56 | }));
57 | }).on('message', async data => {
58 | console.log('Title of page:', JSON.parse(data).result.result.value);
59 | ws.close();
60 | await browser.close();
61 | });
62 |
63 | })();
64 |
--------------------------------------------------------------------------------
/code_coverage.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Shows how to use Puppeteer's code coverage API to measure CSS/JS coverage across
21 | * different points of time during loading. Great for determining if a lazy loading strategy
22 | * is paying off or working correctly.
23 | *
24 | * Install:
25 | * npm i puppeteer chalk cli-table
26 | * Run:
27 | * URL=https://example.com node code_coverage.js
28 | */
29 |
30 | const puppeteer = require('puppeteer');
31 | const chalk = require('chalk');
32 | const Table = require('cli-table');
33 |
34 | const URL = process.env.URL || 'https://www.chromestatus.com/features';
35 |
36 | const EVENTS = [
37 | 'domcontentloaded',
38 | 'load',
39 | // 'networkidle2',
40 | 'networkidle0',
41 | ];
42 |
43 | function formatBytesToKB(bytes) {
44 | if (bytes > 1024) {
45 | const formattedNum = new Intl.NumberFormat('en-US', {maximumFractionDigits: 1}).format(bytes / 1024);
46 | return `${formattedNum}KB`;
47 | }
48 | return `${bytes} bytes`;
49 | }
50 |
51 | class UsageFormatter {
52 | constructor(stats) {
53 | this.stats = stats;
54 | }
55 |
56 | static eventLabel(event) {
57 | // const maxEventLabelLen = EVENTS.reduce((currMax, event) => Math.max(currMax, event.length), 0);
58 | // const eventLabel = event + ' '.repeat(maxEventLabelLen - event.length);
59 | return chalk.magenta(event);
60 | }
61 |
62 | summary(used = this.stats.usedBytes, total = this.stats.totalBytes) {
63 | const percent = Math.round((used / total) * 100);
64 | return `${formatBytesToKB(used)}/${formatBytesToKB(total)} (${percent}%)`;
65 | }
66 |
67 | shortSummary(used, total = this.stats.totalBytes) {
68 | const percent = Math.round((used / total) * 100);
69 | return used ? `${formatBytesToKB(used)} (${percent}%)` : 0;
70 | }
71 |
72 | /**
73 | * Constructors a bar chart for the % usage of each value.
74 | * @param {!{jsUsed: number, cssUsed: number, totalBytes: number}=} stats Usage stats.
75 | * @return {string}
76 | */
77 | barGraph(stats = this.stats) {
78 | // const MAX_TERMINAL_CHARS = process.stdout.columns;
79 | const maxBarWidth = 30;
80 |
81 | const jsSegment = ' '.repeat((stats.jsUsed / stats.totalBytes) * maxBarWidth);
82 | const cssSegment = ' '.repeat((stats.cssUsed / stats.totalBytes) * maxBarWidth);
83 | const unusedSegment = ' '.repeat(maxBarWidth - jsSegment.length - cssSegment.length);
84 |
85 | return chalk.bgRedBright(jsSegment) + chalk.bgBlueBright(cssSegment) +
86 | chalk.bgBlackBright(unusedSegment);
87 | }
88 | }
89 |
90 | const stats = new Map();
91 |
92 | /**
93 | * @param {!Object} coverage
94 | * @param {string} type Either "css" or "js" to indicate which type of coverage.
95 | * @param {string} eventType The page event when the coverage was captured.
96 | */
97 | function addUsage(coverage, type, eventType) {
98 | for (const entry of coverage) {
99 | if (!stats.has(entry.url)) {
100 | stats.set(entry.url, []);
101 | }
102 |
103 | const urlStats = stats.get(entry.url);
104 |
105 | let eventStats = urlStats.find(item => item.eventType === eventType);
106 | if (!eventStats) {
107 | eventStats = {
108 | cssUsed: 0,
109 | jsUsed: 0,
110 | get usedBytes() { return this.cssUsed + this.jsUsed; },
111 | totalBytes: 0,
112 | get percentUsed() {
113 | return this.totalBytes ? Math.round(this.usedBytes / this.totalBytes * 100) : 0;
114 | },
115 | eventType,
116 | url: entry.url,
117 | };
118 | urlStats.push(eventStats);
119 | }
120 |
121 | eventStats.totalBytes += entry.text.length;
122 |
123 | for (const range of entry.ranges) {
124 | eventStats[`${type}Used`] += range.end - range.start - 1;
125 | }
126 | }
127 | }
128 |
129 | async function collectCoverage() {
130 | const browser = await puppeteer.launch({headless: true});
131 |
132 | // Do separate load for each event. See
133 | // https://github.com/GoogleChrome/puppeteer/issues/1887
134 | const collectPromises = EVENTS.map(async event => {
135 | console.log(`Collecting coverage @ ${UsageFormatter.eventLabel(event)}...`);
136 |
137 | const page = await browser.newPage();
138 |
139 | // page.on('response', async response => {
140 | // console.log(response.request().url(), (await response.text()).length);
141 | // });
142 |
143 | await Promise.all([
144 | page.coverage.startJSCoverage(),
145 | page.coverage.startCSSCoverage()
146 | ]);
147 |
148 | await page.goto(URL, {waitUntil: event});
149 | // await page.waitForNavigation({waitUntil: event});
150 |
151 | const [jsCoverage, cssCoverage] = await Promise.all([
152 | page.coverage.stopJSCoverage(),
153 | page.coverage.stopCSSCoverage()
154 | ]);
155 |
156 | addUsage(cssCoverage, 'css', event);
157 | addUsage(jsCoverage, 'js', event);
158 |
159 | await page.close();
160 | });
161 |
162 | await Promise.all(collectPromises);
163 |
164 | return browser.close();
165 | }
166 |
167 | (async() => {
168 |
169 | await collectCoverage();
170 |
171 | for (const [url, vals] of stats) {
172 | console.log('\n' + chalk.cyan(url));
173 |
174 | const table = new Table({
175 | // chars: {mid: '', 'left-mid': '', 'mid-mid': '', 'right-mid': ''},
176 | head: [
177 | 'Event',
178 | `${chalk.bgRedBright(' JS ')} ${chalk.bgBlueBright(' CSS ')} % used`,
179 | 'JS used',
180 | 'CSS used',
181 | 'Total bytes used'
182 | ],
183 | // style : {compact : true, 'padding-left' : 0}
184 | style: {head: ['white'], border: ['grey']}
185 | // colWidths: [20, 20]
186 | });
187 |
188 | EVENTS.forEach(event => {
189 | const usageForEvent = vals.filter(val => val.eventType === event);
190 |
191 | if (usageForEvent.length) {
192 | for (const stats of usageForEvent) {
193 | // totalBytes += stats.totalBytes;
194 | // totalUsedBytes += stats.usedBytes;
195 |
196 | const formatter = new UsageFormatter(stats);
197 | table.push([
198 | UsageFormatter.eventLabel(stats.eventType),
199 | formatter.barGraph(),
200 | formatter.shortSummary(stats.jsUsed), // !== 0 ? `${formatBytesToKB(stats.jsUsed)}KB` : 0,
201 | formatter.shortSummary(stats.cssUsed),
202 | formatter.summary()
203 | ]);
204 | }
205 | } else {
206 | table.push([UsageFormatter.eventLabel(event), 'no usage found', '-', '-', '-']);
207 | }
208 | });
209 |
210 | console.log(table.toString());
211 | }
212 |
213 | // Print total usage for each event.
214 | // console.log('\n');
215 | EVENTS.forEach(event => {
216 | let totalBytes = 0;
217 | let totalUsedBytes = 0;
218 |
219 | const metrics = Array.from(stats.values());
220 | const statsForEvent = metrics.map(eventStatsForUrl => {
221 | const statsForEvent = eventStatsForUrl.filter(stat => stat.eventType === event)[0];
222 | // TODO: need to sum max totalBytes. Currently ignores stats if event didn't
223 | // have an entry. IOW, all total numerators should be max totalBytes seen for that event.
224 | if (statsForEvent) {
225 | totalBytes += statsForEvent.totalBytes;
226 | totalUsedBytes += statsForEvent.usedBytes;
227 | }
228 | });
229 |
230 | const percentUsed = Math.round(totalUsedBytes / totalBytes * 100);
231 |
232 | console.log(`Total used @ ${chalk.magenta(event)}: ${formatBytesToKB(totalUsedBytes)}/${formatBytesToKB(totalBytes)} (${percentUsed}%)`);
233 | });
234 |
235 | })();
236 |
--------------------------------------------------------------------------------
/connect.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * @fileoverview
21 | *
22 | * Demonstrates how to use puppeteer.connect() to re-connect to instance of
23 | * Chrome that's already running with remote debugging enabled.
24 | *
25 | * These first time you run this script, it will launch Chrome and print
26 | * the remote debugging websocket url.
27 | *
28 | * node connect.js
29 | *
30 | * The second time you run the script, pass the websocket URL as an env variable.
31 | * Puppeteer will reconnect that to the browser instance running instead of
32 | * launching a new browser.
33 | *
34 | * wsURL=ws://127.0.0.1:9222/devtools/browser/72775377-7f73-4436 node connect.js
35 | */
36 |
37 | const path = require('path');
38 | const puppeteer = require('puppeteer');
39 |
40 | const browserWSEndpoint = process.env.wsURL || null;
41 |
42 | (async() => {
43 |
44 | if (!browserWSEndpoint) {
45 | const browser = await puppeteer.launch({
46 | handleSIGINT: false, // so Chrome doesn't exit when we quit Node.
47 | headless: false // to see what's happening
48 | });
49 |
50 | console.log('1. Quit this script (cmd/ctrl+C).');
51 | console.log('2. Chrome will still be running.');
52 | console.log('4. Re-return the script with:');
53 | console.log(` wsURL=${browser.wsEndpoint()} node ${path.basename(__filename)}`);
54 | console.log('5. Puppeteer will reconnect to the existing Chrome instead of launching a new browser.');
55 |
56 | return;
57 | }
58 |
59 | console.log('Reconnecting to existing Chrome....');
60 | const browser = await puppeteer.connect({browserWSEndpoint});
61 | const page = await browser.newPage();
62 | await page.goto('https://example.com');
63 |
64 | console.log(`Page title:`, await page.title());
65 |
66 | await browser.close();
67 | })();
68 |
--------------------------------------------------------------------------------
/crawlsite.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Google Inc. All rights reserved.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | *
16 | * @author ebidel@ (Eric Bidelman)
17 | */
18 |
19 | /**
20 | * Discovers all the pages in site or single page app (SPA) and creates
21 | * a tree of the result in ./output/