├── .github ├── ISSUE_TEMPLATE │ ├── Bug_report.md │ ├── Feature_request.md │ └── Support_question.md ├── PULL_REQUEST_TEMPLATE.md └── renovate.json ├── .gitignore ├── .node-version ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── Readme.md ├── doc ├── alternatives.md ├── an-almost-static-stack-optimization.md ├── anatomy-of-js-static-website-generator.md ├── behind-the-scenes.md ├── emotion-site-optimization.md ├── images │ ├── emotion-0-filmstrip.png │ ├── emotion-0-waterfall-full.png │ ├── emotion-0-waterfall.png │ ├── emotion-1-filmstrip.png │ ├── emotion-1-waterfall.png │ ├── emotion-2-filmstrip.png │ ├── emotion-2-waterfall.png │ ├── emotion-3-filmstrip.png │ ├── emotion-3-waterfall.png │ ├── round-0-surge-info.png │ ├── round-0-surge.png │ ├── round-1-firebase.png │ ├── round-3-firebase.png │ ├── round-4-cloudflare.png │ ├── round-4-firebase.png │ ├── round-5-firebase.png │ ├── round-6-firebase.png │ ├── round-7-firebase.png │ ├── round-8-firebase.png │ └── round-9-firebase.png ├── load-performance-optimizations.md ├── recipes.md └── who-uses-it │ ├── blacklane.png │ ├── cloud.gov.au.png │ └── reformma.png ├── index.js ├── package.json ├── run.js ├── src ├── puppeteer_utils.js └── tracker.js ├── tests ├── __snapshots__ │ └── defaultOptions.test.js.snap ├── defaultOptions.test.js ├── examples │ ├── cra │ │ ├── index.html │ │ └── static │ │ │ └── js │ │ │ ├── 0.35040230.chunk.js │ │ │ └── main.42105999.js │ ├── many-pages │ │ └── index.html │ ├── one-page │ │ └── index.html │ ├── other │ │ ├── 404.html │ │ ├── ajax-request.html │ │ ├── css │ │ │ ├── bg.png │ │ │ ├── big.css │ │ │ └── small.css │ │ ├── fix-insert-rule.html │ │ ├── form-elements.html │ │ ├── history-push-more.html │ │ ├── history-push.html │ │ ├── index.html │ │ ├── js │ │ │ ├── main.js │ │ │ └── test.json │ │ ├── link-to-file.html │ │ ├── localhost-links-different-port.html │ │ ├── remove-blobs.html │ │ ├── snap-save-state.html │ │ ├── svg.html │ │ ├── third-party-resource.html │ │ ├── with-big-css.html │ │ ├── with-image.html │ │ ├── with-script-error.html │ │ ├── with-script.html │ │ └── with-small-css.html │ ├── partial │ │ ├── index.html │ │ └── index.js │ └── processed │ │ └── 200.html ├── helper.js └── run.test.js ├── vendor └── preload_polyfill.min.js └── yarn.lock /.github/ISSUE_TEMPLATE/Bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐛 Bug Report 3 | about: If something isn't working as expected 🤔. 4 | 5 | --- 6 | 7 | ## Bug Report 8 | 9 | **Current Behavior** 10 | A clear and concise description of the behavior. 11 | 12 | **Reproducible demo** 13 | Link to GitHub repository or codesandbox with a demo of the bug behavior. 14 | 15 | 26 | 27 | **Expected behavior/code** 28 | A clear and concise description of what you expected to happen (or code). 29 | 30 | **Possible Solution** 31 | 32 | 33 | **Additional context/Screenshots** 34 | Add any other context about the problem here. If applicable, add screenshots to help explain. 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🚀 Feature Request 3 | about: I have a suggestion (and may want to implement it 🙂)! 4 | 5 | --- 6 | 7 | ## Feature Request 8 | 9 | **Is your feature request related to a problem? Please describe.** 10 | A clear and concise description of what the problem is. Ex. I have an issue when [...] 11 | 12 | **Describe the solution you'd like** 13 | A clear and concise description of what you want to happen. Add any considered drawbacks. 14 | 15 | **Describe alternatives you've considered** 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | **Teachability, Documentation, Adoption, Migration Strategy** 19 | If you can, explain how users will be able to use this and possibly write out a version the docs. 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Support_question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🤗 Support Question 3 | about: If you have a question 💬, please check out StackOverflow! 4 | 5 | --- 6 | 7 | --------------^ Click "Preview" for a nicer view! 8 | We primarily use GitHub as an issue tracker; for usage and support questions, please check out these resources below. Thanks! 😁. 9 | 10 | --- 11 | 12 | * StackOverflow: https://stackoverflow.com/search?q=react-snap using the tag `react-snap` 13 | * Also have a look at the readme and doc folder: 14 | https://github.com/stereobooster/react-snap/blob/master/README.md 15 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 24 | 25 | ### Description 26 | Please describe your pull request. 27 | 28 | 💔Thank you! -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | yarn-error.log 3 | -------------------------------------------------------------------------------- /.node-version: -------------------------------------------------------------------------------- 1 | 8.6.0 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "8" 4 | - "10" 5 | cache: yarn 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 1.23.0 2 | 3 | - Potential fix for #301 illegal operation on a directory (#317) 4 | - Disable ServiceWorker (#315) 5 | 6 | # 1.22.1 7 | 8 | - Fix bug introduced in 1.22.0 (#312 by @kjkta) 9 | 10 | # 1.22.0 11 | 12 | - Fix CRA2 compatibility (#264) 13 | 14 | # 1.21.0 15 | 16 | - Support in browser redirect (#292, #294, #296) 17 | - Update dependency minimalcss to v0.8.1 (#300) 18 | - Update dependency html-minifier to v3.5.21 19 | - Update dependency puppeteerto ^v1.8.0 20 | - Update dependency express to v4.16.4 21 | 22 | # 1.20.0 23 | 24 | - Added ability to save screenshots as jpeg. [#288](https://github.com/stereobooster/react-snap/pull/288) by @tsantef 25 | - Fixed unclear message when there is no ``. [#273](https://github.com/stereobooster/react-snap/pull/273) by @onionhammer 26 | - Fixed broken SVG links. [#279](https://github.com/stereobooster/react-snap/pull/279) by @derappelt 27 | 28 | # 1.19.0 29 | 30 | - Fix `fixWebpackChunksIssue` for `react-scripts@2.0.0-next.a671462c` and later. [#252](https://github.com/stereobooster/react-snap/pull/252) 31 | - Make sure that `vendors` chunk for `react-scripts@2.0.0-next.a671462c` and later is present in preload links. Push preload links before first style if `inlineCss` is true. [#254](https://github.com/stereobooster/react-snap/pull/254) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 stereobooster 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | [![Stand With Ukraine](https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner2-direct.svg)](https://vshymanskyy.github.io/StandWithUkraine) 2 | 3 | # react-snap [![Build Status](https://travis-ci.org/stereobooster/react-snap.svg?branch=master)](https://travis-ci.org/stereobooster/react-snap) [![npm](https://img.shields.io/npm/v/react-snap.svg)](https://www.npmjs.com/package/react-snap) ![npm](https://img.shields.io/npm/dt/react-snap.svg) [![Twitter Follow](https://img.shields.io/twitter/url/http/shields.io.svg?style=social&label=Follow)](https://twitter.com/stereobooster) 4 | 5 | Pre-renders a web app into static HTML. Uses [Headless Chrome](https://github.com/GoogleChrome/puppeteer) to crawl all available links starting from the root. Heavily inspired by [prep](https://github.com/graphcool/prep) and [react-snapshot](https://github.com/geelen/react-snapshot), but written from scratch. Uses best practices to get the best loading performance. 6 | 7 | ## 😍 Features 8 | 9 | - Enables **SEO** (Google, DuckDuckGo...) and **SMO** (Twitter, Facebook...) for SPAs. 10 | - **Works out-of-the-box** with [create-react-app](https://github.com/facebookincubator/create-react-app) - no code-changes required. 11 | - Uses a **real browser** behind the scenes, so there are no issues with unsupported HTML5 features, like WebGL or Blobs. 12 | - Does a lot of **load performance optimization**. [Here are details](doc/load-performance-optimizations.md), if you are curious. 13 | - **Does not depend on React**. The name is inspired by `react-snapshot` but works with any technology (e.g., Vue). 14 | - npm package does not have a compilation step, so **you can fork** it, change what you need, and install it with a GitHub URL. 15 | 16 | **Zero configuration** is the main feature. You do not need to worry about how it works or how to configure it. But if you are curious, [here are details](doc/behind-the-scenes.md). 17 | 18 | ## Basic usage with create-react-app 19 | 20 | Install: 21 | 22 | ```sh 23 | yarn add --dev react-snap 24 | ``` 25 | 26 | Change `package.json`: 27 | 28 | ```json 29 | "scripts": { 30 | "postbuild": "react-snap" 31 | } 32 | ``` 33 | 34 | Change `src/index.js` (for React 16+): 35 | 36 | ```js 37 | import { hydrate, render } from "react-dom"; 38 | 39 | const rootElement = document.getElementById("root"); 40 | if (rootElement.hasChildNodes()) { 41 | hydrate(, rootElement); 42 | } else { 43 | render(, rootElement); 44 | } 45 | ``` 46 | 47 | That's it! 48 | 49 | ## Basic usage with Preact 50 | 51 | To do [hydration in Preact you need to use this trick](https://github.com/developit/preact/issues/1060#issuecomment-389987994): 52 | 53 | ```js 54 | const rootElement = document.getElementById("root"); 55 | if (rootElement.hasChildNodes()) { 56 | preact.render(, rootElement, rootElement.firstElementChild); 57 | } else { 58 | preact.render(, rootElement); 59 | } 60 | ``` 61 | 62 | ## Basic usage with Vue.js 63 | 64 | Install: 65 | 66 | ```sh 67 | yarn add --dev react-snap 68 | ``` 69 | 70 | Change `package.json`: 71 | 72 | ```json 73 | "scripts": { 74 | "postbuild": "react-snap" 75 | }, 76 | "reactSnap": { 77 | "source": "dist", 78 | "minifyHtml": { 79 | "collapseWhitespace": false, 80 | "removeComments": false 81 | } 82 | } 83 | ``` 84 | 85 | Or use `preserveWhitespace: false` in `vue-loader`. 86 | 87 | `source` - output folder of webpack or any other bundler of your choice 88 | 89 | Read more about `minifyHtml` caveats in [#142](https://github.com/stereobooster/react-snap/issues/142). 90 | 91 | Example: [Switch from prerender-spa-plugin to react-snap](https://github.com/stereobooster/prerender-spa-plugin/commit/ee73d39b862bc905b44a04c6eaa58e6730957819) 92 | 93 | ### Caveats 94 | 95 | Only works with routing strategies using the HTML5 history API. No hash(bang) URLs. 96 | 97 | Vue uses the `data-server-rendered` attribute on the root element to mark SSR generated markup. When this attribute is present, the VDOM rehydrates instead of rendering everything from scratch, which can result in a flash. 98 | 99 | This is a small hack to fix rehydration problem: 100 | 101 | ```js 102 | window.snapSaveState = () => { 103 | document.querySelector("#app").setAttribute("data-server-rendered", "true"); 104 | }; 105 | ``` 106 | 107 | `window.snapSaveState` is a callback to save the state of the application at the end of rendering. It can be used for Redux or async components. In this example, it is repurposed to alter the DOM, this is why I call it a "hack." Maybe in future versions of `react-snap`, I will come up with better abstractions or automate this process. 108 | 109 | ### Vue 1.x 110 | 111 | Make sure to use [`replace: false`](https://v1.vuejs.org/api/#replace) for root components 112 | 113 | ## ✨ Examples 114 | 115 | - [Emotion website load performance optimization](doc/emotion-site-optimization.md) 116 | - [Load performance optimization](doc/an-almost-static-stack-optimization.md) 117 | - [recipes](doc/recipes.md) 118 | - [stereobooster/an-almost-static-stack](https://github.com/stereobooster/an-almost-static-stack) 119 | 120 | ## ⚙️ Customization 121 | 122 | If you need to pass some options for `react-snap`, you can do this in your `package.json` like this: 123 | 124 | ```json 125 | "reactSnap": { 126 | "inlineCss": true 127 | } 128 | ``` 129 | 130 | Not all options are documented yet, but you can check `defaultOptions` in `index.js`. 131 | 132 | ### inlineCss 133 | 134 | Experimental feature - requires improvements. 135 | 136 | `react-snap` can inline critical CSS with the help of [minimalcss](https://github.com/peterbe/minimalcss) and full CSS will be loaded in a non-blocking manner with the help of [loadCss](https://www.npmjs.com/package/fg-loadcss). 137 | 138 | Use `inlineCss: true` to enable this feature. 139 | 140 | TODO: as soon as this feature is stable, it should be enabled by default. 141 | 142 | ## ⚠️ Caveats 143 | 144 | ### Async components 145 | 146 | Also known as [code splitting](https://webpack.js.org/guides/code-splitting/), [dynamic import](https://github.com/tc39/proposal-dynamic-import) (TC39 proposal), "chunks" (which are loaded on demand), "layers", "rollups", or "fragments". See: [Guide To JavaScript Async Components](https://github.com/stereobooster/guide-to-async-components) 147 | 148 | An async component (in React) is a technique (typically implemented as a higher-order component) for loading components on demand with the dynamic `import` operator. There are a lot of solutions in this field. Here are some examples: 149 | 150 | - [`react.lazy`](https://reactjs.org/docs/code-splitting.html#reactlazy) 151 | - [`loadable-components`](https://github.com/smooth-code/loadable-components) 152 | - [`react-loadable`](https://github.com/thejameskyle/react-loadable) 153 | - [`react-async-component`](https://github.com/ctrlplusb/react-async-component) 154 | 155 | It is not a problem to render async components with `react-snap`, the tricky part happens when a prerendered React application boots and async components are not loaded yet, so React draws the "loading" state of a component, and later when the component is loaded, React draws the actual component. As a result, the user sees a flash: 156 | 157 | ``` 158 | 100% /----| |---- 159 | / | | 160 | / | | 161 | / | | 162 | / |____| 163 | visual progress / 164 | / 165 | 0% -------------/ 166 | ``` 167 | 168 | Usually a _code splitting_ library provides an API to handle it during SSR, but as long as "real" SSR is not used in react-snap - the issue surfaces, and there is no simple way to fix it. 169 | 170 | 1. Use [react-prerendered-component](https://github.com/theKashey/react-prerendered-component). This library holds onto the prerendered HTML until the dynamically imported code is ready. 171 | 172 | ```js 173 | import loadable from "@loadable/component"; 174 | import { PrerenderedComponent } from "react-prerendered-component"; 175 | 176 | const prerenderedLoadable = dynamicImport => { 177 | const LoadableComponent = loadable(dynamicImport); 178 | return React.memo(props => ( 179 | // you can use the `.preload()` method from react-loadable or react-imported-component` 180 | 181 | 182 | 183 | )); 184 | }; 185 | 186 | const MyComponent = prerenderedLoadable(() => import("./MyComponent")); 187 | ``` 188 | 189 | `MyComponent` will use prerendered HTML to prevent the page content from flashing (it will find the required piece of HTML using an `id` attribute generated by `PrerenderedComponent` and inject it using `dangerouslySetInnerHTML`). 190 | 191 | 2. The same approach will work with `React.lazy`, but `React.lazy` doesn't provide a prefetch method (`load` or `preload`), so you need to implement it yourself (this can be a fragile solution). 192 | 193 | ```js 194 | const prefetchMap = new WeakMap(); 195 | const prefetchLazy = LazyComponent => { 196 | if (!prefetchMap.has(LazyComponent)) { 197 | prefetchMap.set(LazyComponent, LazyComponent._ctor()); 198 | } 199 | return prefetchMap.get(LazyComponent); 200 | }; 201 | 202 | const prerenderedLazy = dynamicImport => { 203 | const LazyComponent = React.lazy(dynamicImport); 204 | return React.memo(props => ( 205 | 206 | 207 | 208 | )); 209 | }; 210 | 211 | const MyComponent = prerenderedLazy(() => import("./MyComponent")); 212 | ``` 213 | 214 | 3. use `loadable-components` 2.2.3 (current is >5). The old version of `loadable-components` can solve this issue for a "snapshot" setup: 215 | 216 | ```js 217 | import { loadComponents, getState } from "loadable-components"; 218 | window.snapSaveState = () => getState(); 219 | 220 | loadComponents() 221 | .then(() => hydrate(AppWithRouter, rootElement)) 222 | .catch(() => render(AppWithRouter, rootElement)); 223 | ``` 224 | 225 | If you don't use babel plugin, [don't forget to provide modules](https://github.com/smooth-code/loadable-components/issues/114): 226 | 227 | ```js 228 | const NotFoundPage = loadable(() => import("src/pages/NotFoundPage"), { 229 | modules: ["NotFoundPage"] 230 | }); 231 | ``` 232 | 233 | > `loadable-components` were deprecated in favour of `@loadable/component`, but `@loadable/component` dropped `getState`. So if you want to use `loadable-components` you can use old version (`2.2.3` latest version at the moment of writing) or you can wait until `React` will implement proper handling of this case with asynchronous rendering and `React.lazy`. 234 | 235 | ### Redux 236 | 237 | See: [Redux Server Rendering Section](https://redux.js.org/docs/recipes/ServerRendering.html#the-client-side) 238 | 239 | ```js 240 | // Grab the state from a global variable injected into the server-generated HTML 241 | const preloadedState = window.__PRELOADED_STATE__; 242 | 243 | // Allow the passed state to be garbage-collected 244 | delete window.__PRELOADED_STATE__; 245 | 246 | // Create Redux store with initial state 247 | const store = createStore(counterApp, preloadedState || initialState); 248 | 249 | // Tell react-snap how to save Redux state 250 | window.snapSaveState = () => ({ 251 | __PRELOADED_STATE__: store.getState() 252 | }); 253 | ``` 254 | 255 | **Caution**: as of now, only basic "JSON" data types are supported: e.g. `Date`, `Set`, `Map`, and `NaN` **won't** be handled correctly ([#54](https://github.com/stereobooster/react-snap/issues/54)). 256 | 257 | ### Third-party requests: Google Analytics, Mapbox, etc. 258 | 259 | You can block all third-party requests with the following config: 260 | 261 | ```json 262 | "skipThirdPartyRequests": true 263 | ``` 264 | 265 | ### AJAX 266 | 267 | `react-snap` can capture all AJAX requests. It will store `json` requests in the domain in `window.snapStore[]`, where `` is the path of the request. 268 | 269 | Use `"cacheAjaxRequests": true` to enable this feature. 270 | 271 | This feature can conflict with the browser cache. See [#197](https://github.com/stereobooster/react-snap/issues/197#issuecomment-397893434) for details. You may want to disable cache in this case: `"puppeteer": { "cache": false }`. 272 | 273 | ### Service Workers 274 | 275 | By default, `create-react-app` uses `index.html` as a fallback: 276 | 277 | ```json 278 | navigateFallback: publicUrl + '/index.html', 279 | ``` 280 | 281 | You need to change this to an un-prerendered version of `index.html` - `200.html`, otherwise you will see `index.html` flash on other pages (if you have any). See [Configure sw-precache without ejecting](https://github.com/stereobooster/react-snap/blob/master/doc/recipes.md#configure-sw-precache-without-ejecting) for more information. 282 | 283 | ### Containers and other restricted environments 284 | 285 | Puppeteer (Headless Chrome) may fail due to sandboxing issues. To get around this, 286 | you may use: 287 | 288 | ```json 289 | "puppeteerArgs": ["--no-sandbox", "--disable-setuid-sandbox"] 290 | ``` 291 | 292 | Read more about [puppeteer troubleshooting](https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md). 293 | 294 | `"inlineCss": true` sometimes causes problems in containers. 295 | 296 | #### Docker + Alpine 297 | 298 | To run `react-snap` inside `docker` with Alpine, you might want to use a custom Chromium executable. See [#93](https://github.com/stereobooster/react-snap/issues/93#issuecomment-354994505) and [#132](https://github.com/stereobooster/react-snap/issues/132#issuecomment-362333702). 299 | 300 | #### Heroku 301 | 302 | ``` 303 | heroku buildpacks:add https://github.com/jontewks/puppeteer-heroku-buildpack.git 304 | heroku buildpacks:add heroku/nodejs 305 | heroku buildpacks:add https://github.com/heroku/heroku-buildpack-static.git 306 | ``` 307 | 308 | See this [PR](https://github.com/stereobooster/an-almost-static-stack/pull/7/files). At the moment of writing, Heroku doesn't support HTTP/2. 309 | 310 | ### Semantic UI 311 | 312 | [Semantic UI](https://semantic-ui.com/) is defined over class substrings that contain spaces 313 | (e.g., "three column"). Sorting the class names, therefore, breaks the styling. To get around this, 314 | use the following configuration: 315 | 316 | ```json 317 | "minifyHtml": { "sortClassName": false } 318 | ``` 319 | 320 | From version `1.17.0`, `sortClassName` is `false` by default. 321 | 322 | ### JSS 323 | 324 | > Once JS on the client is loaded, components initialized and your JSS styles are regenerated, it's a good time to remove server-side generated style tag in order to avoid side-effects 325 | > 326 | > https://github.com/cssinjs/jss/blob/master/docs/ssr.md 327 | 328 | This basically means that JSS doesn't support `rehydration`. See [#99](https://github.com/stereobooster/react-snap/issues/99) for a possible solutions. 329 | 330 | ### `react-router` v3 331 | 332 | See [#135](https://github.com/stereobooster/react-snap/issues/135). 333 | 334 | ### userAgent 335 | 336 | You can use `navigator.userAgent == "ReactSnap"` to do some checks in the app code while snapping—for example, if you use an absolute path for your API AJAX request. While crawling, however, you should request a specific host. 337 | 338 | Example code: 339 | 340 | ```js 341 | const BASE_URL = 342 | process.env.NODE_ENV == "production" && navigator.userAgent != "ReactSnap" 343 | ? "/" 344 | : "http://xxx.yy/rest-api"; 345 | ``` 346 | 347 | ## Alternatives 348 | 349 | See [alternatives](doc/alternatives.md). 350 | 351 | ## Who uses it 352 | 353 | | [![cloud.gov.au](doc/who-uses-it/cloud.gov.au.png)](https://github.com/govau/cloud.gov.au/blob/0187dd78d8f1751923631d3ff16e0fbe4a82bcc6/www/ui/package.json#L29) | [![blacklane](doc/who-uses-it/blacklane.png)](http://m.blacklane.com/) | [![reformma](doc/who-uses-it/reformma.png)](http://reformma.com) | 354 | | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------ | 355 | 356 | 357 | ## Contributing 358 | 359 | ### Report a bug 360 | 361 | Please provide a reproducible demo of a bug and steps to reproduce it. Thanks! 362 | 363 | ### Share on the web 364 | 365 | Tweet it, like it, share it, star it. Thank you. 366 | 367 | ### Code 368 | 369 | You can also contribute to [minimalcss](https://github.com/peterbe/minimalcss), which is a big part of `react-snap`. Also, give it some stars. 370 | -------------------------------------------------------------------------------- /doc/alternatives.md: -------------------------------------------------------------------------------- 1 | # Alternatives 2 | 3 | ## Prerendering, snapshotting 4 | 5 | | | react-snap | [prerender-spa-plugin][prerender-spa-plugin] | [react-snapshot][react-snapshot] | [prep][prep] | [snapshotify][snapshotify] | 6 | |-------------------------------|------------|----------------------------------------------|----------------------------------|--------------|----------------------------| 7 | | State | supported | supported | unsupported | unsupported | experimental | 8 | | DOM implementation | puppeteer | phantomjs-prebuilt | jsdom | nightmare | puppeteer | 9 | | Doesn't depend on Webpack | + | - | + | + | + | 10 | | Doesn't depend on React | + | + | - | + | + | 11 | | Load performance optimisation | + | - | - | - | + | 12 | | Zero-configuration | + | - | + | - | + | 13 | | Redux | + | - | + | - | - | 14 | | Async components | + | - | - | - | + | 15 | | Webpack code splitting | + | [+][code-splitting] | - | - | + | 16 | | `CSSStyleSheet.insertRule` | + | - | - | - | + | 17 | | blob urls | + | ? | - | - | - | 18 | | All browser features | + | - | - | ? | + | 19 | 20 | [prerender-spa-plugin]: https://github.com/chrisvfritz/prerender-spa-plugin 21 | [react-snapshot]: https://github.com/geelen/react-snapshot 22 | [prep]: https://github.com/graphcool/prep 23 | [snapshotify]: https://github.com/errorception/snapshotify 24 | [code-splitting]: https://github.com/chrisvfritz/prerender-spa-plugin#code-splitting 25 | 26 | - Load performance optimisation - something beyond rendering HTML, like critical CSS 27 | - Zero-configuration - provides sensible defaults 28 | - **Redux** - can save state at the end of rendering 29 | - **async components** - can save state of async components to prevent flash on the client side 30 | - `insertRule` - Works with CSS-in-JS solutions which use `CSSStyleSheet.insertRule` 31 | - **blob urls** - removes blob urls from generated HTML 32 | 33 | ### Less popular options 34 | 35 | - [presite](https://github.com/egoist/presite), taki (jsdom, chromy (chrome-remote-interface, chrome-launcher)) 36 | - [prerenderer](https://github.com/tribex/prerenderer), experimental, jsdom, cheerio, chrome-remote-interface 37 | - [prerender-chrome-headless](https://github.com/en-japan-air/prerender-chrome-headless), puppeteer 38 | - [chrome-render](https://github.com/gwuhaolin/chrome-render), chrome-pool (chrome-remote-interface, chrome-runner) 39 | - [react-prerender](https://github.com/Robert-W/react-prerender), react, cheerio 40 | - [simple-react-prerender](https://github.com/beac0n/simple-react-prerender), react, jsdom, mock-browser, isomorphic-fetch 41 | - [vue-prerender](https://github.com/eldarc/vue-prerender), experimental, vue, puppeteer 42 | - [prerender-seo](https://github.com/posrix/prerender-seo), phantomjs-prebuilt 43 | - [puppeteer-prerender](https://github.com/fenivana/puppeteer-prerender), puppeteer, request 44 | - [puppeteer-prerenderer](https://github.com/GoodeUser/puppeteer-prerenderer), puppeteer 45 | - [pre-render](https://github.com/kriasoft/pre-render), chrome-remote-interface, chrome-launcher, lighthouse-logger 46 | - [prerender-plugin](https://github.com/mubaidr/prerender-plugin), webpack, puppeteer 47 | - [webpack-static-site-generator](https://github.com/esalter-va/webpack-static-site-generator), nightmare, xvfb 48 | - [junctions-static](https://github.com/jamesknelson/junctions/tree/master/packages/junctions-static), jsdom, react 49 | 50 | ## SEO-only server prerenderers 51 | 52 | - [rendertron](https://github.com/GoogleChrome/rendertron), chrome-remote-interface, chrome-launcher 53 | - [prerender](https://github.com/prerender/prerender), chrome-remote-interface 54 | - [puppetron](https://github.com/cheeaun/puppetron), puppeteer 55 | - [pupperender](https://github.com/LasaleFamine/pupperender), puppeteer 56 | - [spiderable-middleware](https://github.com/VeliovGroup/spiderable-middleware), request 57 | 58 | ## Other 59 | 60 | - [React on Rails](https://github.com/shakacode/react_on_rails), SSR of React on top of Ruby on Rails 61 | - [usus](https://github.com/gajus/usus), chrome-remote-interface, chrome-launcher 62 | - [hypernova](https://github.com/airbnb/hypernova) 63 | - [static-site-generator-webpack-plugin](https://github.com/markdalgleish/static-site-generator-webpack-plugin), webpack 64 | - [Sitepack: A toolkit for building lazymorphic applications](https://github.com/sitepack/sitepack) 65 | - [chromeless](https://github.com/graphcool/chromeless), chrome-launcher, chrome-remote-interface 66 | 67 | ## Software as a service 68 | 69 | - https://www.prerender.cloud/ 70 | - https://www.roast.io/docs/config/ssr 71 | 72 | ## React static site generators 73 | 74 | - [gatsby](https://github.com/gatsbyjs/gatsby) 75 | - [phenomic](https://github.com/phenomic/phenomic) 76 | - [react-static](https://github.com/nozzle/react-static) 77 | - [nextein](https://github.com/elmasse/nextein) 78 | 79 | ## SSR 80 | 81 | - [razzle](https://github.com/jaredpalmer/razzle), react 82 | - [next.js](https://github.com/zeit/next.js/), react 83 | - [nuxtjs](https://nuxtjs.org/), vue 84 | - [Create React App Universal CLI](https://github.com/antonybudianto/cra-universal), react 85 | 86 | ## WebComponents 87 | 88 | - [talk: WebComponents SSR](https://youtu.be/yT-EsESAmgA) 89 | - [skatejs/ssr](https://github.com/skatejs/ssr) 90 | - [rendertron#web-components](https://github.com/GoogleChrome/rendertron#web-components) 91 | - [shadydom](https://github.com/webcomponents/shadydom) 92 | 93 | ## Headless browsers 94 | 95 | - Puppeteer vs chrome-launcher - [WIP: use chrome-launcher for handling the browser](https://github.com/GoogleChrome/puppeteer/pull/23) 96 | - [Chrome DevTools Protocol Viewer](https://chromedevtools.github.io/devtools-protocol/) 97 | - [chrome-remote-interface](https://github.com/cyrus-and/chrome-remote-interface) 98 | - [RemoteDebug Protocol Compatibility Tables](http://compatibility.remotedebug.org/) 99 | - [Puppeteer for Firefox](https://github.com/autonome/puppeteer-fx) 100 | - [Awesome chrome-devtools](https://github.com/ChromeDevTools/awesome-chrome-devtools) 101 | - [HeadlessBrowsers](https://github.com/dhamaniasad/HeadlessBrowsers) 102 | 103 | -------------------------------------------------------------------------------- /doc/an-almost-static-stack-optimization.md: -------------------------------------------------------------------------------- 1 | Experiment: how much faster you can make [An Almost Static Stack](https://github.com/superhighfives/an-almost-static-stack). 2 | 3 | Let's set expectations upfront. I do not suggest to switch to one technology over another. I use real-life measurement metrics and minimal example. The purpose is to show that you can get a lot of performance improvement, with a small investment of your time, it is not hard at all. You do not need Ph.D. to do this. 4 | 5 | Setup: 6 | - All measurements were done using [webpagetest](https://www.webpagetest.org) with following settings: "From: Dulles, VA - Moto G4 - Chrome - 3G". 7 | - Most measurements were done against firebase (except round 0 and round 4.1), but still, there is variation in Time To the First Byte from 1.3s to 1.7s. This is because of network fluctuations, it is out of experiment scope. Apply correlation accordingly. 8 | - Code for experiments are stored in [stereobooster/an-almost-static-stack#experiment](https://github.com/stereobooster/an-almost-static-stack/tree/experiment) 9 | - Each step is stored in the separate tag. To test it, do the following: 10 | 11 | ```sh 12 | git clone https://github.com/stereobooster/an-almost-static-stack.git 13 | cd an-almost-static-stack 14 | git checkout round-N 15 | yarn install 16 | yarn build or yarn deploy 17 | ``` 18 | 19 | ## Round 0 20 | 21 | Let's measure what we have out of the box. Original setup uses Surge ([demo](https://yadg.surge.sh)). 22 | 23 | ``` 24 | git checkout round-0 25 | ``` 26 | 27 | ![round-0-surge-info.png](images/round-0-surge-info.png) 28 | 29 | ![round-0-surge.png](images/round-0-surge.png) 30 | 31 | It performs not that well: 32 | 33 | - First Byte Time: D, 1.966s 34 | - Keep-alive: F, Disabled 35 | - HTTP1 36 | - Compression: gzip (not brotli) 37 | 38 | ## First round: hosting 39 | 40 | First improvement would be to choose better hosting or use CDN in front of it. So you can use anything from the following range (I'm listing popular and free or cheap options): 41 | 42 | | | Expiration headers | Easy to deploy | Free plan | Custom Headers | HTTP push | 43 | |-----------------------|--------------------|----------------|-----------|------------------|------------------| 44 | | Surge + Cloudflare | ± [1][1] | + | + | - [5][5] | - | 45 | | Now + Cloudflare | ± [2][2], [6][6] | ± [3][3] | + | ± [2][2], [6][6] | ± [2][2], [6][6] | 46 | | S3 + Cloudflare | + | ± 4 | + | - [7][7] | - | 47 | | S3 + Cloudfront | + | - | - | ± [8][8] | - | 48 | | Firebase | + | + | + | + | - | 49 | | Firebase + Cloudflare | + | + | + | + | + | 50 | 51 | [1]: https://surge.sh/help/using-lucid-caching-automatically 52 | [2]: https://zeit.co/api 53 | [3]: https://github.com/zeit/now-cli/issues/992 54 | [5]: https://github.com/sintaxi/surge/issues/165 55 | [6]: https://github.com/zeit/serve/issues/289 56 | [7]: https://github.com/aws/aws-cli/issues/818#issuecomment-69972662 57 | [8]: https://medium.com/@tom.cook/edge-lambda-cloudfront-custom-headers-3d134a2c18a2 58 | 59 | 4: There is AWS CLI, but to set headers you need to use a script 60 | 61 | **Solution**: I will use Firebase because this is the easiest option for me, but you can choose any option. Just make sure it has HTTP2 support and ability to set expiration headers. 62 | 63 | 🔖 [round-1](https://github.com/stereobooster/an-almost-static-stack/compare/round-0...stereobooster:round-1?expand=1) 64 | 65 | ![round-1-firebase.png](images/round-1-firebase.png) 66 | 67 | ## Second round: PWA 68 | 69 | Let's test with Lighthouse. Use Lighthouse checkbox in webpagetest interface. 70 | 71 | **Performance**: 79 72 | 73 | **Lighthouse PWA Score**: 45 74 | 75 | - Does not register a service worker 76 | - Does not respond with a 200 when offline 77 | - Does not redirect HTTP traffic to HTTPS 78 | - User will not be prompted to Install the Web App. Failures: No manifest was fetched, Site does not register a service worker, Service worker does not successfully serve the manifest's start_url. 79 | - Is not configured for a custom splash screen. Failures: No manifest was fetched. 80 | - Address bar does not match brand colors. Failures: No manifest was fetched, No `` tag found. 81 | 82 | This is because the project was created with an old version of c-r-a. A new version comes with service worker out of the box. 83 | 84 | **Solution**: Let's add it back by copying from the fresh project. I also added icons and appcache-nanny. Important: use `Cache-Control:max-age=0` for `service-worker.js`. 85 | 86 | 87 | 🔖 [round-2](https://github.com/stereobooster/an-almost-static-stack/compare/round-1...stereobooster:round-2?expand=1) 88 | 89 | **Lighthouse PWA Score**: 91 90 | 91 | - Does not redirect HTTP traffic to HTTPS 92 | 93 | ## Third round: render-blocking stylesheets 94 | 95 | Next problem: Lighthouse complains about render-blocking stylesheets. 96 | 97 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 98 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 99 | | 3.566s | 1.731s | 2.558s | 2558 | 3.814s | 3.566s | 3 | 90 KB | 5.593s | 10 | 211 KB | $---- | 100 | 101 | **Solution**: switch from `react-snapshot` to `react-snap` and use `inlineCss` feature. 102 | 103 | `inlineCss` - will either inline critical CSS and load all the rest in a non-blocking manner or will inline all CSS directly in HTML. 104 | 105 | **Caution**: inlineCss is an experimental feature. Test carefully if you are using it. 106 | 107 | 🔖 [round-3](https://github.com/stereobooster/an-almost-static-stack/compare/round-2...stereobooster:round-3?expand=1) 108 | 109 | **Start Render** reduced by 0.5s 110 | 111 | ![round-3-firebase.png](images/round-3-firebase.png) 112 | 113 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 114 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 115 | | 3.766s | 1.723s | 2.032s | 2032 | 3.979s | 3.766s | 2 | 89 KB | 5.704s | 9 | 210 KB | $---- | 116 | 117 | ## Round 4: Link headers 118 | 119 | Next optimization is pretty trivial and does not require code modification, but it requires the support of custom headers from your hosting. You will need to be able to set `Link` header. 120 | 121 | `react-snap` can generate Link headers in [superstatic](https://github.com/firebase/superstatic) format, like this: 122 | 123 | ```json 124 | { 125 | "source":"about", 126 | "headers":[{ 127 | "key":"Link", 128 | "value":";rel=preload;as=script,;rel=preload;as=script" 129 | }] 130 | } 131 | ``` 132 | 133 | **Solution**: I wrote a small script, to make use of this data and pass it to firebase. 134 | 135 | 🔖 [round-4](https://github.com/stereobooster/an-almost-static-stack/compare/round-3...stereobooster:round-4?expand=1) 136 | 137 | **First Interactive (beta)** reduced by 0.6s 138 | 139 | ~~**Caution**: there is a bug in firebase-cli which prevents setting header for root path. So I tested `/about` page.~~. Fixed in `v1.4.1`. 140 | 141 | ~~**Caution 2**: Link headers contain `service-worker.js`, which won't be used by all browsers. I will need to fix this issue. On the other side, all main browsers have this feature in technical preview, so it will be resolved soon.~~ Fixed in `v1.6.0`. 142 | 143 | ![round-4-firebase.png](images/round-4-firebase.png) 144 | 145 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 146 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 147 | | 2.908s | 1.559s | 1.883s | 1883 | 3.113s | 2.908s | 3 | 109 KB | 4.647s | 10 | 217 KB | $---- | 148 | 149 | ## Round 4: HTTP2 server push 150 | 151 | This strategy the same as above, but I will proxy all requests through Cloudflare and will use [Cloudflare feature, that will convert `Link` headers, to HTTP2 server push](https://blog.cloudflare.com/announcing-support-for-http-2-server-push-2/). 152 | 153 | ![round-4-cloudflare.png](images/round-4-cloudflare.png) 154 | 155 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 156 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 157 | | 3.005s | 1.734s | 2.121s | 2121 | 3.226s | 3.005s | 3 | 88 KB | 5.498s | 10 | 187 KB | $---- | 158 | 159 | **Conclusion**: not worth it, almost no changes. 160 | 161 | ## Round 5: minify JS, by removing runtime of CSS-in-JS 162 | 163 | Not saying you should do it, but I want to show you what price you are paying by using CSS-in-JS solution. I know that styled-components have nice development experience. 164 | 165 | **Solution**: replace `styled-components` with `CSS modules`. To do this I will use `c-r-a` fork with CSS modules support. 166 | 167 | 🔖 [round-5](https://github.com/stereobooster/an-almost-static-stack/compare/round-4...stereobooster:round-5?expand=1) 168 | 169 | ``` 170 | File sizes after gzip: 171 | 172 | 67 KB (-19.62 KB) build/static/js/main.2d68d422.js 173 | 511 B build/static/css/main-cssmodules.361f0795.css 174 | 191 B build/static/css/main.9ddb714c.css 175 | ``` 176 | 177 | **First Interactive (beta)** reduced by 0.2s 178 | 179 | ![round-5-firebase.png](images/round-5-firebase.png) 180 | 181 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 182 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 183 | | 2.407s | 1.301s | 1.661s | 1661 | 2.654s | 2.407s | 5 | 90 KB | 4.054s | 13 | 179 KB | $---- | 184 | 185 | ## Round 6: minify JS, by replacing react with preact 186 | 187 | Another risky move, but I want to show you what price you are paying by using React. I know that React is safer. 188 | 189 | **Solution**: use script which will replace `react` with `preact`, using `preact-compat`. 190 | 191 | 🔖 [round-6](https://github.com/stereobooster/an-almost-static-stack/compare/round-5...stereobooster:round-6?expand=1) 192 | 193 | ``` 194 | File sizes after gzip: 195 | 196 | 32.17 KB (-34.83 KB) build/static/js/main.617b519f.js 197 | 511 B build/static/css/main-cssmodules.361f0795.css 198 | 191 B build/static/css/main.9ddb714c.css 199 | ``` 200 | 201 | ![round-6-firebase.png](images/round-6-firebase.png) 202 | 203 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 204 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 205 | | 2.027s | 1.333s | 1.628s | 1628 | 2.265s | 2.027s | 3 | 54 KB | 3.684s | 11 | 108 KB | $---- | 206 | 207 | **First Interactive (beta)** reduced by 0.4s 208 | 209 | **Conclusion**: Preact is incompatible with some React 16 features. 210 | 211 | ## Round 7: load JS in a non-blocking manner 212 | 213 | **Solution**: use `asyncScriptTags` feature of `react-snap` 214 | 215 | 🔖 [round-7](https://github.com/stereobooster/an-almost-static-stack/compare/round-6...stereobooster:round-7?expand=1) 216 | 217 | ![round-7-firebase.png](images/round-7-firebase.png) 218 | 219 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 220 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 221 | | 2.012s | 1.325s | 1.685s | 1685 | 2.239s | 2.012s | 3 | 54 KB | 3.554s | 11 | 108 KB | $---- | 222 | 223 | **Caution**: this will work if you have a mostly static website or website with progressive enhancement in mind. 224 | 225 | **Conclusion**: not worth it, almost no changes. 226 | 227 | ## Round 8: server-side only React 228 | 229 | [Inspired by Netflix](https://twitter.com/NetflixUIE/status/923374215041912833). 230 | 231 | **Solution**: use `removeScriptTags` feature of `react-snap` 232 | 233 | 🔖 [round-8](https://github.com/stereobooster/an-almost-static-stack/compare/round-7...stereobooster:round-8?expand=1) 234 | 235 | **Lighthouse PWA Score**: 45 236 | 237 | ![round-8-firebase.png](images/round-8-firebase.png) 238 | 239 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 240 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 241 | | 1.161s | 1.326s | 1.665s | 1665 | 1.474s | 1.161s | 1 | 2 KB | 2.699s | 3 | 14 KB | $---- | 242 | 243 | 244 | ## Round 9: vanilla create-react-app 245 | 246 | This is for comparison - original create-react-app without any additional optimizations. 247 | 248 | 🔖 [round-9](https://github.com/stereobooster/an-almost-static-stack/compare/round-8...stereobooster:round-9?expand=1) 249 | 250 | **Lighthouse PWA Score**: 82 251 | 252 | ![round-9-firebase.png](images/round-9-firebase.png) 253 | 254 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 255 | |-----------|------------|--------------|-------------|--------------------------|--------|----------|----------|--------|----------|----------|-------| 256 | | 2.613s | 1.330s | 3.178s | 3178 | 2.951s | 2.613s | 4 | 68 KB | 4.216s | 9 | 84 KB | $---- | 257 | 258 | 259 | ## Not covered subjects 260 | 261 | I haven't explored how to optimize fetch requests (AJAX), `async components`, images, fonts, Redux or Apollo. Maybe will cover in future. 262 | 263 | -------------------------------------------------------------------------------- /doc/anatomy-of-js-static-website-generator.md: -------------------------------------------------------------------------------- 1 | # Anatomy of JavaScript static website generator 2 | 3 | First of all, let's assume we have JavaScript application itself which is able to run on the client. Now we want to prerender our application to make the first paint faster, to make it crawlable by search engine bots and by social network bots. 4 | 5 | ## DOM 6 | 7 | To prerender JavaScript a application we need either a virtual DOM like in React or a Node.js DOM implementation like JSDOM or a headless browser like a puppeteer. 8 | 9 | ### Virtual DOM with something like `renderToString` 10 | 11 | **Examples**: 12 | 13 | - `react-static` uses React's `renderToString` and `renderToStaticMarkup` 14 | 15 | **Cons**: 16 | 17 | - Works only with the chosen library, for example React in case of `react-static` 18 | - Works only with components with SSR support, some components don't have it 19 | 20 | **Pros**: 21 | 22 | - It is possible to use caching, for example [`react-component-caching`](https://github.com/rookLab/react-component-caching) 23 | 24 | ### Node.js DOM library 25 | 26 | **Examples**: 27 | 28 | - `react-snapshot` uses React's `renderToString` and JSDOM 29 | - `prerender-loader` uses JSDOM 30 | 31 | **Cons**: 32 | 33 | - Doesn't support some modern DOM features, for example, `Blob` 34 | - Components which have SSR-specific logic most likely will not work as expected, for example, `react-ideal-image` on the server renders another way than on the client 35 | - Need to write specific logic to exclude some client-side-only logic, for example, cookie consent or Google Analytics 36 | 37 | ### Headles 38 | 39 | **Examples**: 40 | 41 | - `react-snap` uses `puppeteer` 42 | 43 | **Cons**: 44 | 45 | - Components which have SSR-specific logic most likely will not work as expected, for example, `react-ideal-image` on the server renders another way than on the client 46 | - Need to write specific logic to exclude some client-side-only logic, for example, cookie consent or Google Analytics 47 | 48 | ## Routes 49 | 50 | Because we have a client-side application, most likely we deal with client-side routing, something like `react-router`. Next question is how our prerenderer will know what pages exist in the application. Possible options are manually list all pages, use some kind of programmatic generator or crawl the pages the same way as search engine bots do. 51 | 52 | ### Manually list routes 53 | 54 | **Examples**: 55 | 56 | - [`prerender-loader`](https://github.com/GoogleChromeLabs/prerender-loader/issues/6) 57 | - [`prerender-spa-plugin`](https://github.com/chrisvfritz/prerender-spa-plugin) 58 | 59 | **Cons**: 60 | 61 | - we need to create and update routes manually 62 | 63 | ### Programmatic generator 64 | 65 | **Examples**: 66 | 67 | - `gatsby` (`createPages` in `gatsby-node.js`) 68 | - `react-static` (`getRoutes` in `static.config.js`) 69 | - `react-snap` - we can require it as lib and pass array of pages to the generator function 70 | 71 | **Cons**: 72 | 73 | - we need to create some config for the pages 74 | 75 | ### Crawl 76 | 77 | **Examples**: 78 | 79 | - `react-snap` 80 | - `react-snapshot` 81 | 82 | **Cons**: 83 | 84 | - In some cases, if some routes cannot be discovered by a crawler we need to add those manually 85 | - We may not want to render all the routes then we need to ignore those routes 86 | 87 | ## Data layer 88 | 89 | If prerenderer is data layer agnostic or not? 90 | 91 | ### Data layer agnostic 92 | 93 | **Examples**: 94 | 95 | - `react-snap` 96 | - `react-static` 97 | 98 | **Pros**: 99 | 100 | - we can use anything we are used to 101 | 102 | ### Not data layer agnostic 103 | 104 | **Examples**: 105 | 106 | - `gatsby` 107 | 108 | **Cons**: 109 | 110 | - We need to use a framework specific data layer, for example, in the case of Gatsby it is Graphql 111 | 112 | ## Data generator 113 | 114 | ### Without data generator 115 | 116 | **Examples**: 117 | 118 | - `react-snap` 119 | 120 | ### With data generator 121 | 122 | **Examples**: 123 | 124 | - `react-static` (`getData` in `static.config.js`) 125 | - `gatsby` (`createNodes` in `gatsby-node.js`) 126 | 127 | ## Data rehydration 128 | 129 | Now, once we have our application prerendered, the next question is how to properly rehydrate it. The main trick here is to recreate exactly the same state of the application as it was at the moment of HTML rendering, so rehydration would reuse existing markup as much as possible (ideally all of it). To do this we need to pass serialized state of the application. 130 | 131 | ### Redux 132 | 133 | **Examples**: 134 | 135 | - `react-snap` ([Redux example](https://github.com/stereobooster/react-snap#redux)) 136 | 137 | ### Special data acсessor 138 | 139 | **Examples**: 140 | 141 | - `react-static` (`RouteProps`, `SiteProps`) 142 | - `next.js` (`getInitialProps`) 143 | 144 | ### Cache AJAX requests 145 | 146 | **Examples**: 147 | 148 | - `react-snap` ([Cache AJAX requests example](https://github.com/stereobooster/react-snap#ajax)) 149 | 150 | ## Webpack 151 | 152 | Some of prerenderers are Webpack agnostic, some are implemented as Webpack plugin, others have it built-in. 153 | 154 | ### Webpack agnostic 155 | 156 | **Examples**: 157 | 158 | - `react-snap` 159 | 160 | **Pros**: 161 | - can be used with other bundlers, for example Parcel 162 | - no need to change webpack config to use it, e.g. it is possible to use with create-react-app without ejecting 163 | 164 | ### Webpack plugin 165 | 166 | **Examples**: 167 | 168 | - `prerender-loader` 169 | - `prerender-spa-plugin` 170 | 171 | ### Webpack built-in 172 | 173 | **Examples**: 174 | 175 | - `next.js` 176 | - `react-static` 177 | - `gatsby` 178 | 179 | **Cons**: 180 | 181 | - would require rewriting existing code unless you initially started with it 182 | 183 | ## Data source 184 | 185 | Not necessarily part of the prerenderer but can be. 186 | 187 | ### Markdown, Front Matter, Git 188 | 189 | Approach pioneered by Jekyll. 190 | 191 | **Examples**: 192 | 193 | - `react-static` 194 | - `gatsby` 195 | 196 | ### Graphql as an interface to the file system 197 | 198 | Approach pioneered by Gatsby. 199 | 200 | **Examples**: 201 | 202 | - `gatsby` 203 | 204 | ### Other 205 | 206 | It can be anything like JSON files, JSON APIs etc 207 | -------------------------------------------------------------------------------- /doc/behind-the-scenes.md: -------------------------------------------------------------------------------- 1 | # Behind the scenes 2 | 3 | 1. copies index.html as 200.html 4 | 2. starts local web server with your application (by default uses `/build` as a root) 5 | 3. visits `/` or any other pages listed in `include` configuration 6 | 4. find all links on the page with the same domain, add them to queue 7 | 5. If there is more than one page in the queue it also adds `/404.html` to the queue 8 | 6. renders the page with the help of puppeteer 9 | 7. waits till there are no active network requests for more than 0.5 second 10 | 8. removes webpack chunks, if needed 11 | 9. removes styles with blob URLs, if needed 12 | 10. recreates text for style tags for CSS-in-JS solutions, if needed 13 | 11. inlines critical CSS, if configured 14 | 12. collects assets for http2 push manifest, if configured 15 | 13. minifies HTML and saves it to the disk 16 | 14. if `route` ends with `.html` it will be used as is, otherwise `route/index.html` is used 17 | 18 | ## Other features 19 | 20 | - `react-snap` works concurrently, by default it uses 4 tabs in the browser. Can be configured with `concurrency` option. 21 | -------------------------------------------------------------------------------- /doc/emotion-site-optimization.md: -------------------------------------------------------------------------------- 1 | Experiment: how much faster you can make [emotion website](https://emotion.sh/). 2 | 3 | Let's set expectations upfront. I do not suggest to switch to one technology over another. I use real-life measurement metrics and minimal example. The purpose is to show that you can get a lot of performance improvement, with a small investment of your time, it is not hard at all. You do not need Ph.D. to do this. 4 | 5 | Setup: 6 | - All measurements were done using [webpagetest](https://www.webpagetest.org) with following settings: "From: Dulles, VA - Moto G4 - Chrome - 3G". 7 | - Most measurements were done against **firebase**, but still, there is variation in Time To the First Byte from 1.3s to 1.7s. This is because of network fluctuations, it is out of experiment scope. Apply correlation accordingly. 8 | - Code for experiments are stored in [stereobooster/emotion#react-snap](https://github.com/stereobooster/emotion/tree/react-snap) 9 | - Each step is stored in the separate tag. To test it, do the following: 10 | 11 | ```sh 12 | git clone https://github.com/stereobooster/emotion.git 13 | cd emotion 14 | git checkout round-N 15 | yarn install 16 | yarn build:site 17 | ``` 18 | 19 | ## Round 0 20 | 21 | ``` 22 | git checkout round-0 23 | ``` 24 | 25 | ![emotion-0-filmstrip.png](images/emotion-0-filmstrip.png) 26 | 27 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 28 | |-----------|------------|--------------|-------------|--------------------------|---------|----------|----------|---------|----------|----------|-------| 29 | | 59.639s | 1.350s | 9.525s | 9676 | 14.866s | 59.639s | 6 | 9,084 KB | 61.694s | 8 | 9,113 KB | $$$$$ | 30 | 31 | ![emotion-0-waterfall.png](images/emotion-0-waterfall.png) 32 | 33 | ![emotion-0-waterfall-full.png](images/emotion-0-waterfall-full.png) 34 | 35 | ## Round 1 36 | 37 | ``` 38 | git checkout round-1 39 | ``` 40 | 41 | Add `react-snap`. No configurations! 42 | 43 | ![emotion-1-filmstrip.png](images/emotion-1-filmstrip.png) 44 | 45 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 46 | |-----------|------------|--------------|-------------|--------------------------|---------|----------|----------|---------|----------|----------|-------| 47 | | 49.870s | 1.473s | 4.072s | 4384 | 15.375s | 49.870s | 6 | 9,144 KB | 51.388s | 8 | 9,174 KB | $$$$$ | 48 | 49 | ![emotion-1-waterfall.png](images/emotion-1-waterfall.png) 50 | 51 | ## Round 2 52 | 53 | ``` 54 | git checkout round-2 55 | ``` 56 | 57 | Use `"inlineCss": true`. 58 | 59 | ![emotion-2-filmstrip.png](images/emotion-2-filmstrip.png) 60 | 61 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 62 | |-----------|------------|--------------|-------------|--------------------------|---------|----------|----------|---------|----------|----------|-------| 63 | | 53.369s | 1.331s | 1.777s | **2208** | 14.237s | 53.369s | 6 | 9,210 KB | 54.875s | 8 | 9,242 KB | $$$$$ | 64 | 65 | ![emotion-2-waterfall.png](images/emotion-2-waterfall.png) 66 | 67 | ## Round 3 68 | 69 | 70 | ``` 71 | git checkout round-3 72 | ``` 73 | 74 | Use `Link` headers. 75 | 76 | ![emotion-3-filmstrip.png](images/emotion-3-filmstrip.png) 77 | 78 | | Load Time | First Byte | Start Render | Speed Index | First Interactive (beta) | Time | Requests | Bytes In | Time | Requests | Bytes In | Cost | 79 | |-----------|------------|--------------|-------------|--------------------------|---------|----------|----------|---------|----------|----------|-------| 80 | | 54.145s | 1.472s | 1.886s | 2603 | **13.238s** | 54.145s | 6 | 9,020 KB | 56.609s | 10 | 9,918 KB | $$$$$ | 81 | 82 | ![emotion-3-waterfall.png](images/emotion-3-waterfall.png) 83 | 84 | 85 | ## Next steps 86 | 87 | - replace gif with mp4 88 | - postpone download of gif until needed (play button) 89 | - optimize font loading strategy 90 | - optimize image and/or use a smaller size 91 | 92 | -------------------------------------------------------------------------------- /doc/images/emotion-0-filmstrip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-0-filmstrip.png -------------------------------------------------------------------------------- /doc/images/emotion-0-waterfall-full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-0-waterfall-full.png -------------------------------------------------------------------------------- /doc/images/emotion-0-waterfall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-0-waterfall.png -------------------------------------------------------------------------------- /doc/images/emotion-1-filmstrip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-1-filmstrip.png -------------------------------------------------------------------------------- /doc/images/emotion-1-waterfall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-1-waterfall.png -------------------------------------------------------------------------------- /doc/images/emotion-2-filmstrip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-2-filmstrip.png -------------------------------------------------------------------------------- /doc/images/emotion-2-waterfall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-2-waterfall.png -------------------------------------------------------------------------------- /doc/images/emotion-3-filmstrip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-3-filmstrip.png -------------------------------------------------------------------------------- /doc/images/emotion-3-waterfall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/emotion-3-waterfall.png -------------------------------------------------------------------------------- /doc/images/round-0-surge-info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-0-surge-info.png -------------------------------------------------------------------------------- /doc/images/round-0-surge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-0-surge.png -------------------------------------------------------------------------------- /doc/images/round-1-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-1-firebase.png -------------------------------------------------------------------------------- /doc/images/round-3-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-3-firebase.png -------------------------------------------------------------------------------- /doc/images/round-4-cloudflare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-4-cloudflare.png -------------------------------------------------------------------------------- /doc/images/round-4-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-4-firebase.png -------------------------------------------------------------------------------- /doc/images/round-5-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-5-firebase.png -------------------------------------------------------------------------------- /doc/images/round-6-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-6-firebase.png -------------------------------------------------------------------------------- /doc/images/round-7-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-7-firebase.png -------------------------------------------------------------------------------- /doc/images/round-8-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-8-firebase.png -------------------------------------------------------------------------------- /doc/images/round-9-firebase.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/images/round-9-firebase.png -------------------------------------------------------------------------------- /doc/load-performance-optimizations.md: -------------------------------------------------------------------------------- 1 | # Load performance optimizations 2 | 3 | ## Works out of the box 4 | 5 | ### Prerendered HTML 6 | 7 | `react-snap` prerenders HTML, so the browser can start to render content or download required resources ASAP. 8 | 9 | ### preconnect for third-party resources 10 | 11 | `react-snap` tracks all third-party connections during rendering and will place appropriate preconnect links in the `header`. 12 | 13 | ### If you are using code splitting feature of Webpack 14 | 15 | `react-snap` will remove chunk scripts from the HTML and instead will place preload links in the `header`. 16 | 17 | ### If you are using CSS-in-JS solution 18 | 19 | `react-snap` will prerender all styles and save in the HTML. 20 | 21 | ## Requires configuration 22 | 23 | This is a brief overview of what described in Readme and [recipes](recipes.md). 24 | 25 | ### inlineCss 26 | 27 | With this configuration enabled `react-snap` will inline critical CSS and stylesheet links will be loaded in a nonblocking manner with the help of [loadCss](https://www.npmjs.com/package/fg-loadcss). 28 | 29 | ### cacheAjaxRequests 30 | 31 | If you are doing AJAX requests (to the same domain), `react-snap` can cache this data in the window. Think of it as a poor man's Redux rehydration. 32 | 33 | ### http2PushManifest 34 | 35 | `react-snap` can record all resources (scripts, styles, images) required for the page and write down this data to the JSON file. You can use this JSON file to generate HTTP2 server pushes or Link headers. 36 | 37 | ### If you are using Redux 38 | 39 | Use `window.snapSaveState` callback to store Redux state, so it can be used to rehydrate on the client side. 40 | 41 | ### If you are using loadable-components 42 | 43 | Use `window.snapSaveState` callback to store `loadable-components` state, so it can be used to rehydrate on the client side. 44 | 45 | ### If you are using Apollo 46 | 47 | Use `window.snapSaveState` callback to store `Apollo` state, so it can be used to rehydrate on the client side. 48 | 49 | **Caution**: I didn't test this one. If you use it please let me know. 50 | 51 | ```js 52 | // Grab the state from a global variable injected into the server-generated HTML 53 | const preloadedState = window.__APOLLO_STORE__ 54 | 55 | // Allow the passed state to be garbage-collected 56 | delete window.__APOLLO_STORE__ 57 | 58 | const client = new ApolloClient({ 59 | initialState: preloadedState, 60 | }); 61 | 62 | // Tell react-snap how to save state 63 | window.snapSaveState = () => ({ 64 | "__APOLLO_STORE__": client.store.getState() 65 | }); 66 | ``` 67 | -------------------------------------------------------------------------------- /doc/recipes.md: -------------------------------------------------------------------------------- 1 | # create-react-app recipes 2 | 3 | 4 | 5 | - [General](#general) 6 | * [Prerender website without ejecting](#prerender-website-without-ejecting) 7 | * [Preact without ejecting](#preact-without-ejecting) 8 | * [Split in chunks](#split-in-chunks) 9 | * [Configure sw-precache without ejecting](#configure-sw-precache-without-ejecting) 10 | * [Use sw-precache with Google Analytics](#use-sw-precache-with-google-analytics) 11 | * [Add Appcache](#add-appcache) 12 | * [Meta tags](#meta-tags) 13 | * [The Perfect 404](#the-perfect-404) 14 | - [Hosting on AWS S3 + cloudflare.com](#hosting-on-aws-s3--cloudflarecom) 15 | * [Setup Cloudflare](#setup-cloudflare) 16 | * [Deployment](#deployment) 17 | * [Caveats](#caveats) 18 | - [react-snap specific](#react-snap-specific) 19 | * [Usage with Google Analytics](#usage-with-google-analytics) 20 | * [Use to render screenshots](#use-to-render-screenshots) 21 | 22 | 23 | 24 | ## General 25 | 26 | ### Prerender website without ejecting 27 | 28 | Use [react-snap](https://github.com/stereobooster/react-snap/blob/master/Readme.md#basic-usage-with-create-react-app) ;) 29 | 30 | ### Preact without ejecting 31 | 32 | Full example is [here](https://github.com/stereobooster/an-almost-static-stack/blob/react-snap/scripts/build-preact.js). 33 | 34 | ```sh 35 | yarn add preact preact-compat 36 | ```` 37 | 38 | `scripts/build-preact.js`: 39 | 40 | ```js 41 | process.env.NODE_ENV = "production" 42 | 43 | const config = require("react-scripts/config/webpack.config.prod") 44 | 45 | config.resolve.alias["react"] = "preact-compat" 46 | config.resolve.alias["react-dom"] = "preact-compat" 47 | 48 | require("react-scripts/scripts/build") 49 | ``` 50 | 51 | ### Split in chunks 52 | 53 | With webpack 2+ you can use dynamic `import` to split bundles in chunks. See articles: 54 | - http://thejameskyle.com/react-loadable.html 55 | - https://serverless-stack.com/chapters/code-splitting-in-create-react-app.html 56 | 57 | ### Configure sw-precache without ejecting 58 | 59 | Full example is [here](https://github.com/stereobooster/an-almost-static-stack/blob/react-snap/scripts/sw-precache-config.js). 60 | 61 | Tip: See [material design offline states](https://material.io/guidelines/patterns/offline-states.html) for UI advices on offline applications. Also see section about [snackbars & toasts](https://material.io/guidelines/components/snackbars-toasts.html). 62 | 63 | `package.json`: 64 | 65 | ```json 66 | "scripts": { 67 | "generate-sw": "sw-precache --root=build --config scripts/sw-precache-config.js && uglifyjs build/service-worker.js -o build/service-worker.js", 68 | "build-snap": "react-scripts build && react-snap && yarn run generate-sw" 69 | } 70 | ``` 71 | 72 | `scripts/sw-precache-config.js`: 73 | 74 | ```js 75 | module.exports = { 76 | // a directory should be the same as "reactSnap.destination", 77 | // which default value is `build` 78 | staticFileGlobs: [ 79 | "build/static/css/*.css", 80 | "build/static/js/*.js", 81 | "build/shell.html", 82 | "build/index.html" 83 | ], 84 | stripPrefix: "build", 85 | publicPath: ".", 86 | // there is "reactSnap.include": ["/shell.html"] in package.json 87 | navigateFallback: "/shell.html", 88 | // Ignores URLs starting from /__ (useful for Firebase): 89 | // https://github.com/facebookincubator/create-react-app/issues/2237#issuecomment-302693219 90 | navigateFallbackWhitelist: [/^(?!\/__).*/], 91 | // By default, a cache-busting query parameter is appended to requests 92 | // used to populate the caches, to ensure the responses are fresh. 93 | // If a URL is already hashed by Webpack, then there is no concern 94 | // about it being stale, and the cache-busting can be skipped. 95 | dontCacheBustUrlsMatching: /\.\w{8}\./, 96 | // configuration specific to this experiment 97 | runtimeCaching: [ 98 | { 99 | urlPattern: /api/, 100 | handler: "fastest" 101 | } 102 | ] 103 | }; 104 | ``` 105 | 106 | You can use `200.html` instead of `shell.html` if you use `react-snap` and do not have separate `shell.html`. This is important because `react-snap` will prerender `index.html` and when user will be offline their will see a flash of `index.html` on navigation. 107 | 108 | ### Use sw-precache with Google Analytics 109 | 110 | See this article https://developers.google.com/web/updates/2016/07/offline-google-analytics 111 | 112 | ### Add Appcache 113 | 114 | Full example is [here](https://github.com/stereobooster/an-almost-static-stack/blob/react-snap/scripts/generate-appcache.js) 115 | 116 | [Webkit promises to add Service Worker support](https://webkit.org/status/#specification-service-workers) meantime we can use Appcache. 117 | 118 | Tip: you can prompt user to "install your site as web app", like [this](https://www.npmjs.com/package/angular-add-to-home-screen). 119 | 120 | Tip 2: you may want something like [localForage](https://localforage.github.io/localForage/) to save data on client side 121 | 122 | ```sh 123 | yarn add appcache-nanny 124 | ``` 125 | 126 | copy [`appcache-loader.html`](https://github.com/gr2m/appcache-nanny/blob/master/appcache-loader.html) to `public/`. 127 | 128 | `scripts/generate-appcache.js`: 129 | 130 | ```js 131 | const SW_PRECACHE_CONFIG = './sw-precache-config' 132 | const OUT_FILE = '../build/manifest.appcache' 133 | 134 | const glob = require('globby') 135 | const { staticFileGlobs, stripPrefix, navigateFallback } = require(SW_PRECACHE_CONFIG) 136 | const fs = require('fs') 137 | const path = require('path') 138 | 139 | glob(staticFileGlobs).then(files => { 140 | // filter out directories 141 | files = files.filter(file => fs.statSync(file).isFile()) 142 | // strip out prefix 143 | files = files.map(file => file.replace(stripPrefix, '')) 144 | 145 | const index = files.indexOf(navigateFallback); 146 | if (index > -1) { 147 | files.splice(index, 1); 148 | } 149 | 150 | const out = [ 151 | 'CACHE MANIFEST', 152 | `# version ${ new Date().getTime() }`, 153 | '', 154 | 'CACHE:', 155 | ...files, 156 | '', 157 | 'NETWORK:', 158 | '*', 159 | 'http://*', 160 | 'https://*', 161 | '', 162 | 'FALLBACK:', 163 | `/ ${navigateFallback}` 164 | ].join('\n') 165 | 166 | fs.writeFileSync(path.join(__dirname, OUT_FILE), out) 167 | console.log(`Wrote ${OUT_FILE} with ${files.length} resources.`) 168 | }) 169 | ``` 170 | 171 | `registerServiceWorker.js`: 172 | 173 | ```js 174 | import appCacheNanny from "appcache-nanny"; 175 | 176 | export default function register() { 177 | if (process.env.NODE_ENV !== 'production') return false; 178 | if ('serviceWorker' in navigator) { 179 | window.addEventListener('load', () => { 180 | const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`; 181 | navigator.serviceWorker 182 | .register(swUrl) 183 | .then(registration => { 184 | registration.onupdatefound = () => { 185 | const installingWorker = registration.installing; 186 | installingWorker.onstatechange = () => { 187 | if (installingWorker.state === 'installed') { 188 | if (navigator.serviceWorker.controller) { 189 | // At this point, the old content will have been purged and 190 | // the fresh content will have been added to the cache. 191 | // It's the perfect time to display a "New content is 192 | // available; please refresh." message in your web app. 193 | console.log('New content is available; please refresh.'); 194 | } else { 195 | // At this point, everything has been precached. 196 | // It's the perfect time to display a 197 | // "Content is cached for offline use." message. 198 | console.log('Content is cached for offline use.'); 199 | } 200 | } 201 | }; 202 | }; 203 | }) 204 | .catch(error => { 205 | console.error('Error during service worker registration:', error); 206 | }); 207 | }); 208 | } else if (window.applicationCache) { 209 | appCacheNanny.start(); 210 | appCacheNanny.on('updateready', () => { 211 | console.log('New content is available; please refresh.'); 212 | }); 213 | appCacheNanny.on('cached', () => { 214 | console.log('Content is cached for offline use.'); 215 | }); 216 | } 217 | return true; 218 | } 219 | 220 | export function unregister() { 221 | if ('serviceWorker' in navigator) { 222 | navigator.serviceWorker.ready.then(registration => { 223 | registration.unregister(); 224 | }); 225 | } else if (window.applicationCache) { 226 | appCacheNanny.stop(); 227 | } 228 | } 229 | ``` 230 | 231 | ### Meta tags 232 | 233 | Full example is [here](https://github.com/stereobooster/an-almost-static-stack/blob/react-snap/src/components/Seo.js). 234 | 235 | Tip: If you do not have images for social media, you can use screenshots of your website. See [Use to render screenshots](#use-to-render-screenshots) section. 236 | 237 | ```sh 238 | yarn add react-helmet 239 | ``` 240 | 241 | ```js 242 | import React from 'react' 243 | import Helmet from 'react-helmet' 244 | import { basePath } from './Config.js'; 245 | 246 | const locales = { 247 | "en": "en_US" 248 | } 249 | 250 | const Meta = (data) => { 251 | const lang = data.lang || "en" 252 | const title = data.title 253 | const description = data.description 254 | const image = data.image !== undefined && `${basePath}${data.image}` 255 | const canonical = data.canonical !== undefined && `${basePath}${data.canonical}` 256 | const type = data.type === undefined ? "article" : "website" 257 | const width = data.image && (data.width || 1200) 258 | const height = data.image && (data.height || 630) 259 | 260 | return ( 261 | 262 | 263 | { title } 264 | 265 | { canonical ? : null } 266 | { image ? : null } 267 | { image ? : null } 268 | 269 | 270 | 271 | { description ? : null } 272 | { canonical ? : null } 273 | 274 | 275 | { image ? : null } 276 | { width ? : null } 277 | { height ? : null } 278 | 279 | 280 | {/* change type of twitter if there is no image? */} 281 | 282 | 283 | { description ? : null } 284 | { image ? : null } 285 | 286 | { canonical ? : null } 287 | { canonical ? : null } 288 | 289 | ) 290 | } 291 | 292 | export default Meta 293 | ``` 294 | 295 | ### The Perfect 404 296 | 297 | See [The Perfect 404](https://alistapart.com/article/perfect404) 298 | 299 | ## Hosting on AWS S3 + cloudflare.com 300 | 301 | If you have less than 20k requests in a month you can host for free. Plus you can get free SSL from CloudFlare. 302 | 303 | There is [blogpost](https://medium.com/@omgwtfmarc/deploying-create-react-app-to-s3-or-cloudfront-48dae4ce0af) recommended by c-r-a. **Do not follow it**. 304 | 305 | Basic AWS S3 setup described [here](http://docs.aws.amazon.com/AmazonS3/latest/user-guide/static-website-hosting.html). 306 | 307 | ### Setup Cloudflare 308 | 309 | - Set `Browser Cache Expiration` to `Respect Existing Headers` 310 | - Set `Always use HTTPS` to `On` 311 | - `Auto Minify` uncheck all checkboxes 312 | 313 | Some additional bits about CloudFlare: https://github.com/virtualjj/aws-s3-backed-cloudflare-static-website 314 | 315 | ### Deployment 316 | 317 | Full example is [here](https://github.com/stereobooster/an-almost-static-stack/blob/react-snap/scripts/aws-deploy.js). 318 | 319 | Use [s3-sync-aws](https://www.npmjs.com/package/s3-sync-aws) for deployment: 320 | 321 | ```js 322 | import _ from "highland" 323 | import level from "level" 324 | import s3sync from "s3-sync-aws" 325 | import readdirp from "readdirp" 326 | import fs from "fs" 327 | 328 | const db = level(__dirname + "/cache") 329 | 330 | const files = readdirp({ 331 | root: __dirname + "/../build" 332 | , directoryFilter: ["!.git", "!cache", "!.DS_Store"] 333 | }) 334 | 335 | const uploader = s3sync({ 336 | key: process.env.AWS_ACCESS_KEY 337 | , secret: process.env.AWS_SECRET_KEY 338 | , bucket: "" 339 | , concurrency: 16 340 | , headers: { 341 | CacheControl: "max-age=14400" // use longer cache if you can 342 | } 343 | }).on("data", file => console.log(`max-age=14400 ${file.url}`) 344 | 345 | const noCacheUploader = s3sync(db, { 346 | key: process.env.AWS_ACCESS_KEY 347 | , secret: process.env.AWS_SECRET_KEY 348 | , bucket: "" 349 | , concurrency: 16 350 | , headers: { 351 | CacheControl: "max-age=0" 352 | } 353 | }).on("data", file => console.log(`max-age=0 ${file.url}`) 354 | 355 | _(files) 356 | .reject((x) => x.path.indexOf("manifest.json") !== -1 ) 357 | .reject((x) => x.path.indexOf(".html") !== -1 ) 358 | .reject((x) => x.path.indexOf("service-worker.js") !== -1 ) 359 | .reject((x) => x.path.indexOf("manifest.appcache") !== -1 ) 360 | .pipe(uploader) 361 | 362 | _(files) 363 | .filter((x) => (x.path.indexOf(".html") !== -1 || x.path.indexOf("service-worker.js") !== -1 || x.path.indexOf("manifest.json") !== -1 || x.path.indexOf("manifest.appcache") !== -1) ) 364 | .pipe(noCacheUploader) 365 | ``` 366 | 367 | ### Caveats 368 | 369 | - AWS S3 does not support custom HTTP headers, that is why you will not be able to use [HTTP2 push with Cloudflare](https://blog.cloudflare.com/announcing-support-for-http-2-server-push-2/). 370 | - [s3-sync-aws does not remove old files](https://github.com/andreialecu/s3-sync-aws/issues/3). 371 | 372 | ## react-snap specific 373 | 374 | ### Usage with Google Analytics 375 | 376 | First way: 377 | 378 | ```js 379 | import ReactGA from 'react-ga' 380 | const snap = navigator.userAgent !== 'ReactSnap'; 381 | const production = process.env.NODE_ENV === 'production'; 382 | if (production && snap) { ReactGA.initialize('XX-XXXXXXXX-X') } 383 | ``` 384 | 385 | Second way: 386 | 387 | ``` 388 | "skipThirdPartyRequests": true 389 | ``` 390 | 391 | Tip: see [The Google Analytics Setup I Use on Every Site I Build](https://philipwalton.com/articles/the-google-analytics-setup-i-use-on-every-site-i-build/), [ganalytics](https://github.com/lukeed/ganalytics) 392 | 393 | ### Use to render screenshots 394 | 395 | `scripts/screenshots.js`: 396 | 397 | ```js 398 | const { run } = require("react-snap"); 399 | 400 | run({ 401 | destination: "build/screenshots", 402 | saveAs: "png" 403 | }); 404 | ``` 405 | -------------------------------------------------------------------------------- /doc/who-uses-it/blacklane.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/who-uses-it/blacklane.png -------------------------------------------------------------------------------- /doc/who-uses-it/cloud.gov.au.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/who-uses-it/cloud.gov.au.png -------------------------------------------------------------------------------- /doc/who-uses-it/reformma.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/doc/who-uses-it/reformma.png -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const crawl = require("./src/puppeteer_utils.js").crawl; 2 | const http = require("http"); 3 | const express = require("express"); 4 | const serveStatic = require("serve-static"); 5 | const fallback = require("express-history-api-fallback"); 6 | const path = require("path"); 7 | const nativeFs = require("fs"); 8 | const mkdirp = require("mkdirp"); 9 | const minify = require("html-minifier").minify; 10 | const url = require("url"); 11 | const minimalcss = require("minimalcss"); 12 | const CleanCSS = require("clean-css"); 13 | const twentyKb = 20 * 1024; 14 | 15 | const defaultOptions = { 16 | //# stable configurations 17 | port: 45678, 18 | source: "build", 19 | destination: null, 20 | concurrency: 4, 21 | include: ["/"], 22 | userAgent: "ReactSnap", 23 | // 4 params below will be refactored to one: `puppeteer: {}` 24 | // https://github.com/stereobooster/react-snap/issues/120 25 | headless: true, 26 | puppeteer: { 27 | cache: true 28 | }, 29 | puppeteerArgs: [], 30 | puppeteerExecutablePath: undefined, 31 | puppeteerIgnoreHTTPSErrors: false, 32 | publicPath: "/", 33 | minifyCss: {}, 34 | minifyHtml: { 35 | collapseBooleanAttributes: true, 36 | collapseWhitespace: true, 37 | decodeEntities: true, 38 | keepClosingSlash: true, 39 | sortAttributes: true, 40 | sortClassName: false 41 | }, 42 | // mobile first approach 43 | viewport: { 44 | width: 480, 45 | height: 850 46 | }, 47 | sourceMaps: true, 48 | //# workarounds 49 | // using CRA1 for compatibility with previous version will be changed to false in v2 50 | fixWebpackChunksIssue: "CRA1", 51 | removeBlobs: true, 52 | fixInsertRule: true, 53 | skipThirdPartyRequests: false, 54 | cacheAjaxRequests: false, 55 | http2PushManifest: false, 56 | // may use some glob solution in the future, if required 57 | // works when http2PushManifest: true 58 | ignoreForPreload: ["service-worker.js"], 59 | //# unstable configurations 60 | preconnectThirdParty: true, 61 | // Experimental. This config stands for two strategies inline and critical. 62 | // TODO: inline strategy can contain errors, like, confuse relative urls 63 | inlineCss: false, 64 | //# feature creeps to generate screenshots 65 | saveAs: "html", 66 | crawl: true, 67 | waitFor: false, 68 | externalServer: false, 69 | //# even more workarounds 70 | removeStyleTags: false, 71 | preloadImages: false, 72 | // add async true to script tags 73 | asyncScriptTags: false, 74 | //# another feature creep 75 | // tribute to Netflix Server Side Only React https://twitter.com/NetflixUIE/status/923374215041912833 76 | // but this will also remove code which registers service worker 77 | removeScriptTags: false 78 | }; 79 | 80 | /** 81 | * 82 | * @param {{source: ?string, destination: ?string, include: ?Array, sourceMaps: ?boolean, skipThirdPartyRequests: ?boolean }} userOptions 83 | * @return {*} 84 | */ 85 | const defaults = userOptions => { 86 | const options = { 87 | ...defaultOptions, 88 | ...userOptions 89 | }; 90 | options.destination = options.destination || options.source; 91 | 92 | let exit = false; 93 | if (!options.include || !options.include.length) { 94 | console.log("🔥 include option should be an non-empty array"); 95 | exit = true; 96 | } 97 | if (options.preloadResources) { 98 | console.log( 99 | "🔥 preloadResources option deprecated. Use preloadImages or cacheAjaxRequests" 100 | ); 101 | exit = true; 102 | } 103 | if (options.minifyOptions) { 104 | console.log("🔥 minifyOptions option renamed to minifyHtml"); 105 | options.minifyHtml = options.minifyOptions; 106 | } 107 | if (options.asyncJs) { 108 | console.log("🔥 asyncJs option renamed to asyncScriptTags"); 109 | options.asyncScriptTags = options.asyncJs; 110 | } 111 | if (options.fixWebpackChunksIssue === true) { 112 | console.log( 113 | "🔥 fixWebpackChunksIssue - behaviour changed, valid options are CRA1, CRA2, Parcel, false" 114 | ); 115 | options.fixWebpackChunksIssue = "CRA1"; 116 | } 117 | if ( 118 | options.saveAs !== "html" && 119 | options.saveAs !== "png" && 120 | options.saveAs !== "jpeg" 121 | ) { 122 | console.log("🔥 saveAs supported values are html, png, and jpeg"); 123 | exit = true; 124 | } 125 | if (exit) throw new Error(); 126 | if (options.minifyHtml && !options.minifyHtml.minifyCSS) { 127 | options.minifyHtml.minifyCSS = options.minifyCss; 128 | } 129 | 130 | if (!options.publicPath.startsWith("/")) { 131 | options.publicPath = `/${options.publicPath}`; 132 | } 133 | options.publicPath = options.publicPath.replace(/\/$/, ""); 134 | 135 | options.include = options.include.map( 136 | include => options.publicPath + include 137 | ); 138 | return options; 139 | }; 140 | 141 | const normalizePath = path => (path === "/" ? "/" : path.replace(/\/$/, "")); 142 | 143 | /** 144 | * 145 | * @param {{page: Page, basePath: string}} opt 146 | */ 147 | const preloadResources = opt => { 148 | const { 149 | page, 150 | basePath, 151 | preloadImages, 152 | cacheAjaxRequests, 153 | preconnectThirdParty, 154 | http2PushManifest, 155 | ignoreForPreload 156 | } = opt; 157 | const ajaxCache = {}; 158 | const http2PushManifestItems = []; 159 | const uniqueResources = new Set(); 160 | page.on("response", async response => { 161 | const responseUrl = response.url(); 162 | if (/^data:|blob:/i.test(responseUrl)) return; 163 | const ct = response.headers()["content-type"] || ""; 164 | const route = responseUrl.replace(basePath, ""); 165 | if (/^http:\/\/localhost/i.test(responseUrl)) { 166 | if (uniqueResources.has(responseUrl)) return; 167 | if (preloadImages && /\.(png|jpg|jpeg|webp|gif|svg)$/.test(responseUrl)) { 168 | if (http2PushManifest) { 169 | http2PushManifestItems.push({ 170 | link: route, 171 | as: "image" 172 | }); 173 | } else { 174 | await page.evaluate(route => { 175 | const linkTag = document.createElement("link"); 176 | linkTag.setAttribute("rel", "preload"); 177 | linkTag.setAttribute("as", "image"); 178 | linkTag.setAttribute("href", route); 179 | document.body.appendChild(linkTag); 180 | }, route); 181 | } 182 | } else if (cacheAjaxRequests && ct.includes("json")) { 183 | const json = await response.json(); 184 | ajaxCache[route] = json; 185 | } else if (http2PushManifest && /\.(js)$/.test(responseUrl)) { 186 | const fileName = url 187 | .parse(responseUrl) 188 | .pathname.split("/") 189 | .pop(); 190 | if (!ignoreForPreload.includes(fileName)) { 191 | http2PushManifestItems.push({ 192 | link: route, 193 | as: "script" 194 | }); 195 | } 196 | } else if (http2PushManifest && /\.(css)$/.test(responseUrl)) { 197 | const fileName = url 198 | .parse(responseUrl) 199 | .pathname.split("/") 200 | .pop(); 201 | if (!ignoreForPreload.includes(fileName)) { 202 | http2PushManifestItems.push({ 203 | link: route, 204 | as: "style" 205 | }); 206 | } 207 | } 208 | uniqueResources.add(responseUrl); 209 | } else if (preconnectThirdParty) { 210 | const urlObj = url.parse(responseUrl); 211 | const domain = `${urlObj.protocol}//${urlObj.host}`; 212 | if (uniqueResources.has(domain)) return; 213 | uniqueResources.add(domain); 214 | await page.evaluate(route => { 215 | const linkTag = document.createElement("link"); 216 | linkTag.setAttribute("rel", "preconnect"); 217 | linkTag.setAttribute("href", route); 218 | document.head.appendChild(linkTag); 219 | }, domain); 220 | } 221 | }); 222 | return { ajaxCache, http2PushManifestItems }; 223 | }; 224 | 225 | const removeStyleTags = ({ page }) => 226 | page.evaluate(() => { 227 | Array.from(document.querySelectorAll("style")).forEach(ell => { 228 | ell.parentElement && ell.parentElement.removeChild(ell); 229 | }); 230 | }); 231 | 232 | const removeScriptTags = ({ page }) => 233 | page.evaluate(() => { 234 | Array.from(document.querySelectorAll("script")).forEach(ell => { 235 | ell.parentElement && ell.parentElement.removeChild(ell); 236 | }); 237 | }); 238 | 239 | const preloadPolyfill = nativeFs.readFileSync( 240 | `${__dirname}/vendor/preload_polyfill.min.js`, 241 | "utf8" 242 | ); 243 | 244 | /** 245 | * 246 | * @param {{page: Page}} opt 247 | * @return Promise 248 | */ 249 | const removeBlobs = async opt => { 250 | const { page } = opt; 251 | return page.evaluate(() => { 252 | const stylesheets = Array.from( 253 | document.querySelectorAll("link[rel=stylesheet]") 254 | ); 255 | stylesheets.forEach(link => { 256 | if (link.href && link.href.startsWith("blob:")) { 257 | link.parentNode && link.parentNode.removeChild(link); 258 | } 259 | }); 260 | }); 261 | }; 262 | 263 | /** 264 | * @param {{page: Page, pageUrl: string, options: {skipThirdPartyRequests: boolean, userAgent: string}, basePath: string, browser: Browser}} opt 265 | * @return {Promise} 266 | */ 267 | const inlineCss = async opt => { 268 | const { page, pageUrl, options, basePath, browser } = opt; 269 | 270 | const minimalcssResult = await minimalcss.minimize({ 271 | urls: [pageUrl], 272 | skippable: request => 273 | options.skipThirdPartyRequests && !request.url().startsWith(basePath), 274 | browser: browser, 275 | userAgent: options.userAgent 276 | }); 277 | const criticalCss = minimalcssResult.finalCss; 278 | const criticalCssSize = Buffer.byteLength(criticalCss, "utf8"); 279 | 280 | const result = await page.evaluate(async () => { 281 | const stylesheets = Array.from( 282 | document.querySelectorAll("link[rel=stylesheet]") 283 | ); 284 | const cssArray = await Promise.all( 285 | stylesheets.map(async link => { 286 | const response = await fetch(link.href); 287 | return response.text(); 288 | }) 289 | ); 290 | return { 291 | cssFiles: stylesheets.map(link => link.href), 292 | allCss: cssArray.join("") 293 | }; 294 | }); 295 | const allCss = new CleanCSS(options.minifyCss).minify(result.allCss).styles; 296 | const allCssSize = Buffer.byteLength(allCss, "utf8"); 297 | 298 | let cssStrategy, cssSize; 299 | if (criticalCssSize * 2 >= allCssSize) { 300 | cssStrategy = "inline"; 301 | cssSize = allCssSize; 302 | } else { 303 | cssStrategy = "critical"; 304 | cssSize = criticalCssSize; 305 | } 306 | 307 | if (cssSize > twentyKb) 308 | console.log( 309 | `⚠️ warning: inlining CSS more than 20kb (${cssSize / 310 | 1024}kb, ${cssStrategy})` 311 | ); 312 | 313 | if (cssStrategy === "critical") { 314 | await page.evaluate( 315 | (criticalCss, preloadPolyfill) => { 316 | const head = document.head || document.getElementsByTagName("head")[0], 317 | style = document.createElement("style"); 318 | style.type = "text/css"; 319 | style.appendChild(document.createTextNode(criticalCss)); 320 | head.appendChild(style); 321 | const noscriptTag = document.createElement("noscript"); 322 | document.head.appendChild(noscriptTag); 323 | 324 | const stylesheets = Array.from( 325 | document.querySelectorAll("link[rel=stylesheet]") 326 | ); 327 | stylesheets.forEach(link => { 328 | noscriptTag.appendChild(link.cloneNode(false)); 329 | link.setAttribute("rel", "preload"); 330 | link.setAttribute("as", "style"); 331 | link.setAttribute("react-snap-onload", "this.rel='stylesheet'"); 332 | document.head.appendChild(link); 333 | }); 334 | 335 | const scriptTag = document.createElement("script"); 336 | scriptTag.type = "text/javascript"; 337 | scriptTag.text = preloadPolyfill; 338 | // scriptTag.id = "preloadPolyfill"; 339 | document.body.appendChild(scriptTag); 340 | }, 341 | criticalCss, 342 | preloadPolyfill 343 | ); 344 | } else { 345 | await page.evaluate(allCss => { 346 | if (!allCss) return; 347 | 348 | const head = document.head || document.getElementsByTagName("head")[0], 349 | style = document.createElement("style"); 350 | style.type = "text/css"; 351 | style.appendChild(document.createTextNode(allCss)); 352 | 353 | if (!head) throw new Error("No element found in document"); 354 | 355 | head.appendChild(style); 356 | 357 | const stylesheets = Array.from( 358 | document.querySelectorAll("link[rel=stylesheet]") 359 | ); 360 | stylesheets.forEach(link => { 361 | link.parentNode && link.parentNode.removeChild(link); 362 | }); 363 | }, allCss); 364 | } 365 | return { 366 | cssFiles: cssStrategy === "inline" ? result.cssFiles : [] 367 | }; 368 | }; 369 | 370 | const asyncScriptTags = ({ page }) => { 371 | return page.evaluate(() => { 372 | Array.from(document.querySelectorAll("script[src]")).forEach(x => { 373 | x.setAttribute("async", "true"); 374 | }); 375 | }); 376 | }; 377 | 378 | const fixWebpackChunksIssue1 = ({ 379 | page, 380 | basePath, 381 | http2PushManifest, 382 | inlineCss 383 | }) => { 384 | return page.evaluate( 385 | (basePath, http2PushManifest, inlineCss) => { 386 | const localScripts = Array.from(document.scripts).filter( 387 | x => x.src && x.src.startsWith(basePath) 388 | ); 389 | // CRA v1|v2.alpha 390 | const mainRegexp = /main\.[\w]{8}.js|main\.[\w]{8}\.chunk\.js/; 391 | const mainScript = localScripts.find(x => mainRegexp.test(x.src)); 392 | const firstStyle = document.querySelector("style"); 393 | 394 | if (!mainScript) return; 395 | 396 | const chunkRegexp = /(\w+)\.[\w]{8}(\.chunk)?\.js/g; 397 | const chunkScripts = localScripts.filter(x => { 398 | const matched = chunkRegexp.exec(x.src); 399 | // we need to reset state of RegExp https://stackoverflow.com/a/11477448 400 | chunkRegexp.lastIndex = 0; 401 | return matched && matched[1] !== "main" && matched[1] !== "vendors"; 402 | }); 403 | 404 | const mainScripts = localScripts.filter(x => { 405 | const matched = chunkRegexp.exec(x.src); 406 | // we need to reset state of RegExp https://stackoverflow.com/a/11477448 407 | chunkRegexp.lastIndex = 0; 408 | return matched && (matched[1] === "main" || matched[1] === "vendors"); 409 | }); 410 | 411 | const createLink = x => { 412 | if (http2PushManifest) return; 413 | const linkTag = document.createElement("link"); 414 | linkTag.setAttribute("rel", "preload"); 415 | linkTag.setAttribute("as", "script"); 416 | linkTag.setAttribute("href", x.src.replace(basePath, "")); 417 | if (inlineCss) { 418 | firstStyle.parentNode.insertBefore(linkTag, firstStyle); 419 | } else { 420 | document.head.appendChild(linkTag); 421 | } 422 | }; 423 | 424 | mainScripts.map(x => createLink(x)); 425 | for (let i = chunkScripts.length - 1; i >= 0; --i) { 426 | const x = chunkScripts[i]; 427 | if (x.parentElement && mainScript.parentNode) { 428 | x.parentElement.removeChild(x); 429 | createLink(x); 430 | } 431 | } 432 | }, 433 | basePath, 434 | http2PushManifest, 435 | inlineCss 436 | ); 437 | }; 438 | 439 | const fixWebpackChunksIssue2 = ({ 440 | page, 441 | basePath, 442 | http2PushManifest, 443 | inlineCss 444 | }) => { 445 | return page.evaluate( 446 | (basePath, http2PushManifest, inlineCss) => { 447 | const localScripts = Array.from(document.scripts).filter( 448 | x => x.src && x.src.startsWith(basePath) 449 | ); 450 | // CRA v2 451 | const mainRegexp = /main\.[\w]{8}\.chunk\.js/; 452 | const mainScript = localScripts.find(x => mainRegexp.test(x.src)); 453 | const firstStyle = document.querySelector("style"); 454 | 455 | if (!mainScript) return; 456 | 457 | const chunkRegexp = /(\w+)\.[\w]{8}\.chunk\.js/g; 458 | 459 | const headScripts = Array.from(document.querySelectorAll("head script")) 460 | .filter(x => x.src && x.src.startsWith(basePath)) 461 | .filter(x => { 462 | const matched = chunkRegexp.exec(x.src); 463 | // we need to reset state of RegExp https://stackoverflow.com/a/11477448 464 | chunkRegexp.lastIndex = 0; 465 | return matched; 466 | }); 467 | 468 | const chunkScripts = localScripts.filter(x => { 469 | const matched = chunkRegexp.exec(x.src); 470 | // we need to reset state of RegExp https://stackoverflow.com/a/11477448 471 | chunkRegexp.lastIndex = 0; 472 | return matched; 473 | }); 474 | 475 | const createLink = x => { 476 | if (http2PushManifest) return; 477 | const linkTag = document.createElement("link"); 478 | linkTag.setAttribute("rel", "preload"); 479 | linkTag.setAttribute("as", "script"); 480 | linkTag.setAttribute("href", x.src.replace(basePath, "")); 481 | if (inlineCss) { 482 | firstStyle.parentNode.insertBefore(linkTag, firstStyle); 483 | } else { 484 | document.head.appendChild(linkTag); 485 | } 486 | }; 487 | 488 | for (let i = headScripts.length; i <= chunkScripts.length - 1; i++) { 489 | const x = chunkScripts[i]; 490 | if (x.parentElement && mainScript.parentNode) { 491 | createLink(x); 492 | } 493 | } 494 | 495 | for (let i = headScripts.length - 1; i >= 0; --i) { 496 | const x = headScripts[i]; 497 | if (x.parentElement && mainScript.parentNode) { 498 | x.parentElement.removeChild(x); 499 | createLink(x); 500 | } 501 | } 502 | }, 503 | basePath, 504 | http2PushManifest, 505 | inlineCss 506 | ); 507 | }; 508 | 509 | const fixParcelChunksIssue = ({ 510 | page, 511 | basePath, 512 | http2PushManifest, 513 | inlineCss 514 | }) => { 515 | return page.evaluate( 516 | (basePath, http2PushManifest, inlineCss) => { 517 | const localScripts = Array.from(document.scripts) 518 | .filter(x => x.src && x.src.startsWith(basePath)) 519 | 520 | const mainRegexp = /main\.[\w]{8}\.js/; 521 | const mainScript = localScripts.find(x => mainRegexp.test(x.src)); 522 | const firstStyle = document.querySelector("style"); 523 | 524 | if (!mainScript) return; 525 | 526 | const chunkRegexp = /(\w+)\.[\w]{8}\.js/g; 527 | const chunkScripts = localScripts.filter(x => { 528 | const matched = chunkRegexp.exec(x.src); 529 | // we need to reset state of RegExp https://stackoverflow.com/a/11477448 530 | chunkRegexp.lastIndex = 0; 531 | return matched && matched[1] !== "main"; 532 | }); 533 | 534 | const createLink = x => { 535 | if (http2PushManifest) return; 536 | const linkTag = document.createElement("link"); 537 | linkTag.setAttribute("rel", "preload"); 538 | linkTag.setAttribute("as", "script"); 539 | linkTag.setAttribute("href", x.src.replace(`${basePath}/`, "")); 540 | if (inlineCss) { 541 | firstStyle.parentNode.insertBefore(linkTag, firstStyle); 542 | } else { 543 | document.head.appendChild(linkTag); 544 | } 545 | }; 546 | 547 | for (let i = 0; i <= chunkScripts.length - 1; i++) { 548 | const x = chunkScripts[i]; 549 | if (x.parentElement && mainScript.parentNode) { 550 | x.parentElement.removeChild(x); 551 | createLink(x); 552 | } 553 | } 554 | }, 555 | basePath, 556 | http2PushManifest, 557 | inlineCss 558 | ); 559 | }; 560 | 561 | const fixInsertRule = ({ page }) => { 562 | return page.evaluate(() => { 563 | Array.from(document.querySelectorAll("style")).forEach(style => { 564 | if (style.innerHTML === "") { 565 | style.innerHTML = Array.from(style.sheet.rules) 566 | .map(rule => rule.cssText) 567 | .join(""); 568 | } 569 | }); 570 | }); 571 | }; 572 | 573 | const fixFormFields = ({ page }) => { 574 | return page.evaluate(() => { 575 | Array.from(document.querySelectorAll("[type=radio]")).forEach(element => { 576 | if (element.checked) { 577 | element.setAttribute("checked", "checked"); 578 | } else { 579 | element.removeAttribute("checked"); 580 | } 581 | }); 582 | Array.from(document.querySelectorAll("[type=checkbox]")).forEach( 583 | element => { 584 | if (element.checked) { 585 | element.setAttribute("checked", "checked"); 586 | } else { 587 | element.removeAttribute("checked"); 588 | } 589 | } 590 | ); 591 | Array.from(document.querySelectorAll("option")).forEach(element => { 592 | if (element.selected) { 593 | element.setAttribute("selected", "selected"); 594 | } else { 595 | element.removeAttribute("selected"); 596 | } 597 | }); 598 | }); 599 | }; 600 | 601 | const saveAsHtml = async ({ page, filePath, options, route, fs }) => { 602 | let content = await page.content(); 603 | content = content.replace(/react-snap-onload/g, "onload"); 604 | const title = await page.title(); 605 | const minifiedContent = options.minifyHtml 606 | ? minify(content, options.minifyHtml) 607 | : content; 608 | filePath = filePath.replace(/\//g, path.sep); 609 | if (route.endsWith(".html")) { 610 | if (route.endsWith("/404.html") && !title.includes("404")) 611 | console.log('⚠️ warning: 404 page title does not contain "404" string'); 612 | mkdirp.sync(path.dirname(filePath)); 613 | fs.writeFileSync(filePath, minifiedContent); 614 | } else { 615 | if (title.includes("404")) 616 | console.log(`⚠️ warning: page not found ${route}`); 617 | mkdirp.sync(filePath); 618 | fs.writeFileSync(path.join(filePath, "index.html"), minifiedContent); 619 | } 620 | }; 621 | 622 | const saveAsPng = ({ page, filePath, options, route }) => { 623 | mkdirp.sync(path.dirname(filePath)); 624 | let screenshotPath; 625 | if (route.endsWith(".html")) { 626 | screenshotPath = filePath.replace(/\.html$/, ".png"); 627 | } else if (route === "/") { 628 | screenshotPath = `${filePath}index.png`; 629 | } else { 630 | screenshotPath = `${filePath.replace(/\/$/, "")}.png`; 631 | } 632 | return page.screenshot({ path: screenshotPath }); 633 | }; 634 | 635 | const saveAsJpeg = ({ page, filePath, options, route }) => { 636 | mkdirp.sync(path.dirname(filePath)); 637 | let screenshotPath; 638 | if (route.endsWith(".html")) { 639 | screenshotPath = filePath.replace(/\.html$/, ".jpeg"); 640 | } else if (route === "/") { 641 | screenshotPath = `${filePath}index.jpeg`; 642 | } else { 643 | screenshotPath = `${filePath.replace(/\/$/, "")}.jpeg`; 644 | } 645 | return page.screenshot({ path: screenshotPath }); 646 | }; 647 | 648 | const run = async (userOptions, { fs } = { fs: nativeFs }) => { 649 | let options; 650 | try { 651 | options = defaults(userOptions); 652 | } catch (e) { 653 | return Promise.reject(e.message); 654 | } 655 | 656 | const sourceDir = path.normalize(`${process.cwd()}/${options.source}`); 657 | const destinationDir = path.normalize( 658 | `${process.cwd()}/${options.destination}` 659 | ); 660 | const startServer = options => { 661 | const app = express() 662 | .use(options.publicPath, serveStatic(sourceDir)) 663 | .use(fallback("200.html", { root: sourceDir })); 664 | const server = http.createServer(app); 665 | server.listen(options.port); 666 | return server; 667 | }; 668 | 669 | if ( 670 | destinationDir === sourceDir && 671 | options.saveAs === "html" && 672 | fs.existsSync(path.join(sourceDir, "200.html")) 673 | ) { 674 | console.log( 675 | `🔥 200.html is present in the sourceDir (${sourceDir}). You can not run react-snap twice - this will break the build` 676 | ); 677 | return Promise.reject(""); 678 | } 679 | 680 | fs.createReadStream(path.join(sourceDir, "index.html")).pipe( 681 | fs.createWriteStream(path.join(sourceDir, "200.html")) 682 | ); 683 | 684 | if (destinationDir !== sourceDir && options.saveAs === "html") { 685 | mkdirp.sync(destinationDir); 686 | fs.createReadStream(path.join(sourceDir, "index.html")).pipe( 687 | fs.createWriteStream(path.join(destinationDir, "200.html")) 688 | ); 689 | } 690 | 691 | const server = options.externalServer ? null : startServer(options); 692 | 693 | const basePath = `http://localhost:${options.port}`; 694 | const publicPath = options.publicPath; 695 | const ajaxCache = {}; 696 | const { http2PushManifest } = options; 697 | const http2PushManifestItems = {}; 698 | 699 | await crawl({ 700 | options, 701 | basePath, 702 | publicPath, 703 | sourceDir, 704 | beforeFetch: async ({ page, route }) => { 705 | const { 706 | preloadImages, 707 | cacheAjaxRequests, 708 | preconnectThirdParty 709 | } = options; 710 | if ( 711 | preloadImages || 712 | cacheAjaxRequests || 713 | preconnectThirdParty || 714 | http2PushManifest 715 | ) { 716 | const { ajaxCache: ac, http2PushManifestItems: hpm } = preloadResources( 717 | { 718 | page, 719 | basePath, 720 | preloadImages, 721 | cacheAjaxRequests, 722 | preconnectThirdParty, 723 | http2PushManifest, 724 | ignoreForPreload: options.ignoreForPreload 725 | } 726 | ); 727 | ajaxCache[route] = ac; 728 | http2PushManifestItems[route] = hpm; 729 | } 730 | }, 731 | afterFetch: async ({ page, route, browser, addToQueue }) => { 732 | const pageUrl = `${basePath}${route}`; 733 | if (options.removeStyleTags) await removeStyleTags({ page }); 734 | if (options.removeScriptTags) await removeScriptTags({ page }); 735 | if (options.removeBlobs) await removeBlobs({ page }); 736 | if (options.inlineCss) { 737 | const { cssFiles } = await inlineCss({ 738 | page, 739 | pageUrl, 740 | options, 741 | basePath, 742 | browser 743 | }); 744 | 745 | if (http2PushManifest) { 746 | const filesToRemove = cssFiles 747 | .filter(file => file.startsWith(basePath)) 748 | .map(file => file.replace(basePath, "")); 749 | 750 | for (let i = http2PushManifestItems[route].length - 1; i >= 0; i--) { 751 | const x = http2PushManifestItems[route][i]; 752 | filesToRemove.forEach(fileToRemove => { 753 | if (x.link.startsWith(fileToRemove)) { 754 | http2PushManifestItems[route].splice(i, 1); 755 | } 756 | }); 757 | } 758 | } 759 | } 760 | 761 | if (options.fixWebpackChunksIssue === "Parcel") { 762 | await fixParcelChunksIssue({ 763 | page, 764 | basePath, 765 | http2PushManifest, 766 | inlineCss: options.inlineCss 767 | }); 768 | } else if (options.fixWebpackChunksIssue === "CRA2") { 769 | await fixWebpackChunksIssue2({ 770 | page, 771 | basePath, 772 | http2PushManifest, 773 | inlineCss: options.inlineCss 774 | }); 775 | } else if (options.fixWebpackChunksIssue === "CRA1") { 776 | await fixWebpackChunksIssue1({ 777 | page, 778 | basePath, 779 | http2PushManifest, 780 | inlineCss: options.inlineCss 781 | }); 782 | } 783 | if (options.asyncScriptTags) await asyncScriptTags({ page }); 784 | 785 | await page.evaluate(ajaxCache => { 786 | const snapEscape = (() => { 787 | const UNSAFE_CHARS_REGEXP = /[<>\/\u2028\u2029]/g; 788 | // Mapping of unsafe HTML and invalid JavaScript line terminator chars to their 789 | // Unicode char counterparts which are safe to use in JavaScript strings. 790 | const ESCAPED_CHARS = { 791 | "<": "\\u003C", 792 | ">": "\\u003E", 793 | "/": "\\u002F", 794 | "\u2028": "\\u2028", 795 | "\u2029": "\\u2029" 796 | }; 797 | const escapeUnsafeChars = unsafeChar => ESCAPED_CHARS[unsafeChar]; 798 | return str => str.replace(UNSAFE_CHARS_REGEXP, escapeUnsafeChars); 799 | })(); 800 | // TODO: as of now it only prevents XSS attack, 801 | // but can stringify only basic data types 802 | // e.g. Date, Set, Map, NaN won't be handled right 803 | const snapStringify = obj => snapEscape(JSON.stringify(obj)); 804 | 805 | let scriptTagText = ""; 806 | if (ajaxCache && Object.keys(ajaxCache).length > 0) { 807 | scriptTagText += `window.snapStore=${snapEscape( 808 | JSON.stringify(ajaxCache) 809 | )};`; 810 | } 811 | let state; 812 | if ( 813 | window.snapSaveState && 814 | (state = window.snapSaveState()) && 815 | Object.keys(state).length !== 0 816 | ) { 817 | scriptTagText += Object.keys(state) 818 | .map(key => `window["${key}"]=${snapStringify(state[key])};`) 819 | .join(""); 820 | } 821 | if (scriptTagText !== "") { 822 | const scriptTag = document.createElement("script"); 823 | scriptTag.type = "text/javascript"; 824 | scriptTag.text = scriptTagText; 825 | const firstScript = Array.from(document.scripts)[0]; 826 | firstScript.parentNode.insertBefore(scriptTag, firstScript); 827 | } 828 | }, ajaxCache[route]); 829 | delete ajaxCache[route]; 830 | if (options.fixInsertRule) await fixInsertRule({ page }); 831 | await fixFormFields({ page }); 832 | 833 | let routePath = route.replace(publicPath, ""); 834 | let filePath = path.join(destinationDir, routePath); 835 | if (options.saveAs === "html") { 836 | await saveAsHtml({ page, filePath, options, route, fs }); 837 | let newRoute = await page.evaluate(() => location.toString()); 838 | newPath = normalizePath( 839 | newRoute.replace(publicPath, "").replace(basePath, "") 840 | ); 841 | routePath = normalizePath(routePath); 842 | if (routePath !== newPath) { 843 | console.log(newPath) 844 | console.log(`💬 in browser redirect (${newPath})`); 845 | addToQueue(newRoute); 846 | } 847 | } else if (options.saveAs === "png") { 848 | await saveAsPng({ page, filePath, options, route, fs }); 849 | } else if (options.saveAs === "jpeg") { 850 | await saveAsJpeg({ page, filePath, options, route, fs }); 851 | } 852 | }, 853 | onEnd: () => { 854 | if (server) server.close(); 855 | if (http2PushManifest) { 856 | const manifest = Object.keys(http2PushManifestItems).reduce( 857 | (accumulator, key) => { 858 | if (http2PushManifestItems[key].length !== 0) 859 | accumulator.push({ 860 | source: key, 861 | headers: [ 862 | { 863 | key: "Link", 864 | value: http2PushManifestItems[key] 865 | .map(x => `<${x.link}>;rel=preload;as=${x.as}`) 866 | .join(",") 867 | } 868 | ] 869 | }); 870 | return accumulator; 871 | }, 872 | [] 873 | ); 874 | fs.writeFileSync( 875 | `${destinationDir}/http2-push-manifest.json`, 876 | JSON.stringify(manifest) 877 | ); 878 | } 879 | } 880 | }); 881 | }; 882 | 883 | exports.defaultOptions = defaultOptions; 884 | exports.run = run; 885 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "react-snap", 3 | "version": "1.23.0", 4 | "description": "Zero-configuration framework-agnostic static prerendering for SPAs", 5 | "main": "index.js", 6 | "author": "stereobooster", 7 | "license": "MIT", 8 | "repository": "stereobooster/react-snap", 9 | "engines": { 10 | "node": ">= 8.6.0" 11 | }, 12 | "dependencies": { 13 | "clean-css": "4.2.1", 14 | "express": "4.17.1", 15 | "express-history-api-fallback": "2.2.1", 16 | "highland": "2.13.4", 17 | "html-minifier": "4.0.0", 18 | "minimalcss": "0.8.2", 19 | "mkdirp": "0.5.1", 20 | "puppeteer": "^1.8.0", 21 | "serve-static": "1.14.1", 22 | "sourcemapped-stacktrace-node": "2.1.8" 23 | }, 24 | "devDependencies": { 25 | "dev-null-stream": "0.0.1", 26 | "jest": "24.9.0", 27 | "markdown-toc": "1.2.0", 28 | "prettier": "1.18.2" 29 | }, 30 | "scripts": { 31 | "toc": "yarn run markdown-toc -i doc/recipes.md", 32 | "test": "jest", 33 | "precommit": "prettier --write {*,src/*}.{js,json,css}" 34 | }, 35 | "bin": { 36 | "react-snap": "./run.js" 37 | }, 38 | "files": [ 39 | "index.js", 40 | "run.js", 41 | "src/", 42 | "vendor/" 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /run.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const url = require("url"); 4 | const { run } = require("./index.js"); 5 | const { 6 | reactSnap, 7 | homepage, 8 | devDependencies, 9 | dependencies 10 | } = require(`${process.cwd()}/package.json`); 11 | 12 | const publicUrl = process.env.PUBLIC_URL || homepage; 13 | 14 | const reactScriptsVersion = parseInt( 15 | (devDependencies && devDependencies["react-scripts"]) 16 | || (dependencies && dependencies["react-scripts"]) 17 | ); 18 | let fixWebpackChunksIssue; 19 | switch (reactScriptsVersion) { 20 | case 1: 21 | fixWebpackChunksIssue = "CRA1"; 22 | break; 23 | case 2: 24 | fixWebpackChunksIssue = "CRA2"; 25 | break; 26 | } 27 | 28 | const parcel = Boolean( 29 | (devDependencies && devDependencies["parcel-bundler"]) 30 | || (dependencies && dependencies["parcel-bundler"]) 31 | ); 32 | 33 | if (parcel) { 34 | if (fixWebpackChunksIssue) { 35 | console.log("Detected both Parcel and CRA. Fixing chunk names for CRA!") 36 | } else { 37 | fixWebpackChunksIssue = "Parcel"; 38 | } 39 | } 40 | 41 | run({ 42 | publicPath: publicUrl ? url.parse(publicUrl).pathname : "/", 43 | fixWebpackChunksIssue, 44 | ...reactSnap 45 | }).catch(error => { 46 | console.error(error); 47 | process.exit(1); 48 | }); -------------------------------------------------------------------------------- /src/puppeteer_utils.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require("puppeteer"); 2 | const _ = require("highland"); 3 | const url = require("url"); 4 | const mapStackTrace = require("sourcemapped-stacktrace-node").default; 5 | const path = require("path"); 6 | const fs = require("fs"); 7 | const { createTracker, augmentTimeoutError } = require("./tracker"); 8 | 9 | const errorToString = jsHandle => 10 | jsHandle.executionContext().evaluate(e => e.toString(), jsHandle); 11 | 12 | const objectToJson = jsHandle => jsHandle.jsonValue(); 13 | 14 | /** 15 | * @param {{page: Page, options: {skipThirdPartyRequests: true}, basePath: string }} opt 16 | * @return {Promise} 17 | */ 18 | const skipThirdPartyRequests = async opt => { 19 | const { page, options, basePath } = opt; 20 | if (!options.skipThirdPartyRequests) return; 21 | await page.setRequestInterception(true); 22 | page.on("request", request => { 23 | if (request.url().startsWith(basePath)) { 24 | request.continue(); 25 | } else { 26 | request.abort(); 27 | } 28 | }); 29 | }; 30 | 31 | /** 32 | * @param {{page: Page, options: {sourceMaps: boolean}, route: string, onError: ?function }} opt 33 | * @return {void} 34 | */ 35 | const enableLogging = opt => { 36 | const { page, options, route, onError, sourcemapStore } = opt; 37 | page.on("console", msg => { 38 | const text = msg.text(); 39 | if (text === "JSHandle@object") { 40 | Promise.all(msg.args().map(objectToJson)).then(args => 41 | console.log(`💬 console.log at ${route}:`, ...args) 42 | ); 43 | } else if (text === "JSHandle@error") { 44 | Promise.all(msg.args().map(errorToString)).then(args => 45 | console.log(`💬 console.log at ${route}:`, ...args) 46 | ); 47 | } else { 48 | console.log(`️️️💬 console.log at ${route}:`, text); 49 | } 50 | }); 51 | page.on("error", msg => { 52 | console.log(`🔥 error at ${route}:`, msg); 53 | onError && onError(); 54 | }); 55 | page.on("pageerror", e => { 56 | if (options.sourceMaps) { 57 | mapStackTrace(e.stack || e.message, { 58 | isChromeOrEdge: true, 59 | store: sourcemapStore || {} 60 | }) 61 | .then(result => { 62 | // TODO: refactor mapStackTrace: return array not a string, return first row too 63 | const stackRows = result.split("\n"); 64 | const puppeteerLine = 65 | stackRows.findIndex(x => x.includes("puppeteer")) || 66 | stackRows.length - 1; 67 | 68 | console.log( 69 | `🔥 pageerror at ${route}: ${(e.stack || e.message).split( 70 | "\n" 71 | )[0] + "\n"}${stackRows.slice(0, puppeteerLine).join("\n")}` 72 | ); 73 | }) 74 | .catch(e2 => { 75 | console.log(`🔥 pageerror at ${route}:`, e); 76 | console.log( 77 | `️️️⚠️ warning at ${route} (error in source maps):`, 78 | e2.message 79 | ); 80 | }); 81 | } else { 82 | console.log(`🔥 pageerror at ${route}:`, e); 83 | } 84 | onError && onError(); 85 | }); 86 | page.on("response", response => { 87 | if (response.status() >= 400) { 88 | let route = ""; 89 | try { 90 | route = response._request 91 | .headers() 92 | .referer.replace(`http://localhost:${options.port}`, ""); 93 | } catch (e) {} 94 | console.log( 95 | `️️️⚠️ warning at ${route}: got ${response.status()} HTTP code for ${response.url()}` 96 | ); 97 | } 98 | }); 99 | // page.on("requestfailed", msg => 100 | // console.log(`️️️⚠️ ${route} requestfailed:`, msg) 101 | // ); 102 | }; 103 | 104 | /** 105 | * @param {{page: Page}} opt 106 | * @return {Promise>} 107 | */ 108 | const getLinks = async opt => { 109 | const { page } = opt; 110 | const anchors = await page.evaluate(() => 111 | Array.from(document.querySelectorAll("a,link[rel='alternate']")).map(anchor => { 112 | if (anchor.href.baseVal) { 113 | const a = document.createElement("a"); 114 | a.href = anchor.href.baseVal; 115 | return a.href; 116 | } 117 | return anchor.href; 118 | }) 119 | ); 120 | 121 | const iframes = await page.evaluate(() => 122 | Array.from(document.querySelectorAll("iframe")).map(iframe => iframe.src) 123 | ); 124 | return anchors.concat(iframes); 125 | }; 126 | 127 | /** 128 | * can not use null as default for function because of TS error https://github.com/Microsoft/TypeScript/issues/14889 129 | * 130 | * @param {{options: *, basePath: string, beforeFetch: ?(function({ page: Page, route: string }):Promise), afterFetch: ?(function({ page: Page, browser: Browser, route: string }):Promise), onEnd: ?(function():void)}} opt 131 | * @return {Promise} 132 | */ 133 | const crawl = async opt => { 134 | const { 135 | options, 136 | basePath, 137 | beforeFetch, 138 | afterFetch, 139 | onEnd, 140 | publicPath, 141 | sourceDir 142 | } = opt; 143 | let shuttingDown = false; 144 | let streamClosed = false; 145 | 146 | const onSigint = () => { 147 | if (shuttingDown) { 148 | process.exit(1); 149 | } else { 150 | shuttingDown = true; 151 | console.log( 152 | "\nGracefully shutting down. To exit immediately, press ^C again" 153 | ); 154 | } 155 | }; 156 | process.on("SIGINT", onSigint); 157 | 158 | const onUnhandledRejection = error => { 159 | console.log("🔥 UnhandledPromiseRejectionWarning", error); 160 | shuttingDown = true; 161 | }; 162 | process.on("unhandledRejection", onUnhandledRejection); 163 | 164 | const queue = _(); 165 | let enqued = 0; 166 | let processed = 0; 167 | // use Set instead 168 | const uniqueUrls = new Set(); 169 | const sourcemapStore = {}; 170 | 171 | /** 172 | * @param {string} path 173 | * @returns {void} 174 | */ 175 | const addToQueue = newUrl => { 176 | const { hostname, search, hash, port } = url.parse(newUrl); 177 | newUrl = newUrl.replace(`${search || ""}${hash || ""}`, ""); 178 | 179 | // Ensures that only link on the same port are crawled 180 | // 181 | // url.parse returns a string, 182 | // but options port is passed by a user and default value is a number 183 | // we are converting both to string to be sure 184 | // Port can be null, therefore we need the null check 185 | const isOnAppPort = port && port.toString() === options.port.toString(); 186 | 187 | if (hostname === "localhost" && isOnAppPort && !uniqueUrls.has(newUrl) && !streamClosed) { 188 | uniqueUrls.add(newUrl); 189 | enqued++; 190 | queue.write(newUrl); 191 | if (enqued == 2 && options.crawl) { 192 | addToQueue(`${basePath}${publicPath}/404.html`); 193 | } 194 | } 195 | }; 196 | 197 | const browser = await puppeteer.launch({ 198 | headless: options.headless, 199 | args: options.puppeteerArgs, 200 | executablePath: options.puppeteerExecutablePath, 201 | ignoreHTTPSErrors: options.puppeteerIgnoreHTTPSErrors, 202 | handleSIGINT: false 203 | }); 204 | 205 | /** 206 | * @param {string} pageUrl 207 | * @returns {Promise} 208 | */ 209 | const fetchPage = async pageUrl => { 210 | const route = pageUrl.replace(basePath, ""); 211 | 212 | let skipExistingFile = false; 213 | const routePath = route.replace(/\//g, path.sep); 214 | const { ext } = path.parse(routePath); 215 | if (ext !== ".html" && ext !== "") { 216 | const filePath = path.join(sourceDir, routePath); 217 | skipExistingFile = fs.existsSync(filePath); 218 | } 219 | 220 | if (!shuttingDown && !skipExistingFile) { 221 | try { 222 | const page = await browser.newPage(); 223 | await page._client.send("ServiceWorker.disable"); 224 | await page.setCacheEnabled(options.puppeteer.cache); 225 | if (options.viewport) await page.setViewport(options.viewport); 226 | if (options.skipThirdPartyRequests) 227 | await skipThirdPartyRequests({ page, options, basePath }); 228 | enableLogging({ 229 | page, 230 | options, 231 | route, 232 | onError: () => { 233 | shuttingDown = true; 234 | }, 235 | sourcemapStore 236 | }); 237 | beforeFetch && beforeFetch({ page, route }); 238 | await page.setUserAgent(options.userAgent); 239 | const tracker = createTracker(page); 240 | try { 241 | await page.goto(pageUrl, { waitUntil: "networkidle0" }); 242 | } catch (e) { 243 | e.message = augmentTimeoutError(e.message, tracker); 244 | throw e; 245 | } finally { 246 | tracker.dispose(); 247 | } 248 | if (options.waitFor) await page.waitFor(options.waitFor); 249 | if (options.crawl) { 250 | const links = await getLinks({ page }); 251 | links.forEach(addToQueue); 252 | } 253 | afterFetch && (await afterFetch({ page, route, browser, addToQueue })); 254 | await page.close(); 255 | console.log(`✅ crawled ${processed + 1} out of ${enqued} (${route})`); 256 | } catch (e) { 257 | if (!shuttingDown) { 258 | console.log(`🔥 error at ${route}`, e); 259 | } 260 | shuttingDown = true; 261 | } 262 | } else { 263 | // this message creates a lot of noise 264 | // console.log(`🚧 skipping (${processed + 1}/${enqued}) ${route}`); 265 | } 266 | processed++; 267 | if (enqued === processed) { 268 | streamClosed = true; 269 | queue.end(); 270 | } 271 | return pageUrl; 272 | }; 273 | 274 | if (options.include) { 275 | options.include.map(x => addToQueue(`${basePath}${x}`)); 276 | } 277 | 278 | return new Promise((resolve, reject) => { 279 | queue 280 | .map(x => _(fetchPage(x))) 281 | .mergeWithLimit(options.concurrency) 282 | .toArray(async () => { 283 | process.removeListener("SIGINT", onSigint); 284 | process.removeListener("unhandledRejection", onUnhandledRejection); 285 | await browser.close(); 286 | onEnd && onEnd(); 287 | if (shuttingDown) return reject(""); 288 | resolve(); 289 | }); 290 | }); 291 | }; 292 | 293 | exports.skipThirdPartyRequests = skipThirdPartyRequests; 294 | exports.enableLogging = enableLogging; 295 | exports.getLinks = getLinks; 296 | exports.crawl = crawl; 297 | -------------------------------------------------------------------------------- /src/tracker.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Sets up event listeners on the Browser.Page instance to maintain a set 3 | * of URLs that have started but never finished or failed. 4 | * 5 | * @param {Object} page 6 | * @return Object 7 | */ 8 | const createTracker = page => { 9 | const requests = new Set(); 10 | const onStarted = request => requests.add(request); 11 | const onFinished = request => requests.delete(request); 12 | page.on("request", onStarted); 13 | page.on("requestfinished", onFinished); 14 | page.on("requestfailed", onFinished); 15 | return { 16 | urls: () => Array.from(requests).map(r => r.url()), 17 | dispose: () => { 18 | page.removeListener("request", onStarted); 19 | page.removeListener("requestfinished", onFinished); 20 | page.removeListener("requestfailed", onFinished); 21 | } 22 | }; 23 | }; 24 | 25 | /** 26 | * Adds information about timed out URLs if given message is about Timeout. 27 | * 28 | * @param {string} message error message 29 | * @param {Object} tracker ConnectionTracker 30 | * @returns {string} 31 | */ 32 | const augmentTimeoutError = (message, tracker) => { 33 | if (message.startsWith("Navigation Timeout Exceeded")) { 34 | const urls = tracker.urls(); 35 | if (urls.length > 1) { 36 | message += `\nTracked URLs that have not finished: ${urls.join(", ")}`; 37 | } else if (urls.length > 0) { 38 | message += `\nFor ${urls[0]}`; 39 | } else { 40 | message += `\nBut there are no pending connections`; 41 | } 42 | } 43 | return message; 44 | }; 45 | 46 | module.exports = { createTracker, augmentTimeoutError }; 47 | -------------------------------------------------------------------------------- /tests/__snapshots__/defaultOptions.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`defaultOptions 1`] = ` 4 | Object { 5 | "asyncScriptTags": false, 6 | "cacheAjaxRequests": false, 7 | "concurrency": 4, 8 | "crawl": true, 9 | "destination": null, 10 | "externalServer": false, 11 | "fixInsertRule": true, 12 | "fixWebpackChunksIssue": "CRA1", 13 | "headless": true, 14 | "http2PushManifest": false, 15 | "ignoreForPreload": Array [ 16 | "service-worker.js", 17 | ], 18 | "include": Array [ 19 | "/", 20 | ], 21 | "inlineCss": false, 22 | "minifyCss": Object {}, 23 | "minifyHtml": Object { 24 | "collapseBooleanAttributes": true, 25 | "collapseWhitespace": true, 26 | "decodeEntities": true, 27 | "keepClosingSlash": true, 28 | "sortAttributes": true, 29 | "sortClassName": false, 30 | }, 31 | "port": 45678, 32 | "preconnectThirdParty": true, 33 | "preloadImages": false, 34 | "publicPath": "/", 35 | "puppeteer": Object { 36 | "cache": true, 37 | }, 38 | "puppeteerArgs": Array [], 39 | "puppeteerExecutablePath": undefined, 40 | "puppeteerIgnoreHTTPSErrors": false, 41 | "removeBlobs": true, 42 | "removeScriptTags": false, 43 | "removeStyleTags": false, 44 | "saveAs": "html", 45 | "skipThirdPartyRequests": false, 46 | "source": "build", 47 | "sourceMaps": true, 48 | "userAgent": "ReactSnap", 49 | "viewport": Object { 50 | "height": 850, 51 | "width": 480, 52 | }, 53 | "waitFor": false, 54 | } 55 | `; 56 | -------------------------------------------------------------------------------- /tests/defaultOptions.test.js: -------------------------------------------------------------------------------- 1 | const { defaultOptions } = require("./../index.js"); 2 | 3 | test("defaultOptions", () => { 4 | expect(defaultOptions).toMatchSnapshot(); 5 | }); 6 | -------------------------------------------------------------------------------- /tests/examples/cra/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |
10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /tests/examples/cra/static/js/0.35040230.chunk.js: -------------------------------------------------------------------------------- 1 | webpackJsonp([0],{10:function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return document.getElementById("root").appendChild(document.createTextNode("submodule"))}}}); 2 | -------------------------------------------------------------------------------- /tests/examples/cra/static/js/main.42105999.js: -------------------------------------------------------------------------------- 1 | !function(t){function e(r){if(n[r])return n[r].exports;var o=n[r]={i:r,l:!1,exports:{}};return t[r].call(o.exports,o,o.exports,e),o.l=!0,o.exports}var r=window.webpackJsonp;window.webpackJsonp=function(e,n,i){for(var s,a,u=0,f=[];uf){for(var e=0,r=s.length-u;e-1?e:t}function d(t,e){e=e||{};var r=e.body;if(t instanceof d){if(t.bodyUsed)throw new TypeError("Already read");this.url=t.url,this.credentials=t.credentials,e.headers||(this.headers=new o(t.headers)),this.method=t.method,this.mode=t.mode,r||null==t._bodyInit||(r=t._bodyInit,t.bodyUsed=!0)}else this.url=String(t);if(this.credentials=e.credentials||this.credentials||"omit",!e.headers&&this.headers||(this.headers=new o(e.headers)),this.method=h(e.method||this.method||"GET"),this.mode=e.mode||this.mode||null,this.referrer=null,("GET"===this.method||"HEAD"===this.method)&&r)throw new TypeError("Body not allowed for GET or HEAD requests");this._initBody(r)}function p(t){var e=new FormData;return t.trim().split("&").forEach(function(t){if(t){var r=t.split("="),n=r.shift().replace(/\+/g," "),o=r.join("=").replace(/\+/g," ");e.append(decodeURIComponent(n),decodeURIComponent(o))}}),e}function y(t){var e=new o;return t.split(/\r?\n/).forEach(function(t){var r=t.split(":"),n=r.shift().trim();if(n){var o=r.join(":").trim();e.append(n,o)}}),e}function b(t,e){e||(e={}),this.type="default",this.status="status"in e?e.status:200,this.ok=this.status>=200&&this.status<300,this.statusText="statusText"in e?e.statusText:"OK",this.headers=new o(e.headers),this.url=e.url||"",this._initBody(t)}if(!t.fetch){var m={searchParams:"URLSearchParams"in t,iterable:"Symbol"in t&&"iterator"in Symbol,blob:"FileReader"in t&&"Blob"in t&&function(){try{return new Blob,!0}catch(t){return!1}}(),formData:"FormData"in t,arrayBuffer:"ArrayBuffer"in t};if(m.arrayBuffer)var w=["[object Int8Array]","[object Uint8Array]","[object Uint8ClampedArray]","[object Int16Array]","[object Uint16Array]","[object Int32Array]","[object Uint32Array]","[object Float32Array]","[object Float64Array]"],v=function(t){return t&&DataView.prototype.isPrototypeOf(t)},_=ArrayBuffer.isView||function(t){return t&&w.indexOf(Object.prototype.toString.call(t))>-1};o.prototype.append=function(t,n){t=e(t),n=r(n);var o=this.map[t];this.map[t]=o?o+","+n:n},o.prototype.delete=function(t){delete this.map[e(t)]},o.prototype.get=function(t){return t=e(t),this.has(t)?this.map[t]:null},o.prototype.has=function(t){return this.map.hasOwnProperty(e(t))},o.prototype.set=function(t,n){this.map[e(t)]=r(n)},o.prototype.forEach=function(t,e){for(var r in this.map)this.map.hasOwnProperty(r)&&t.call(e,this.map[r],r,this)},o.prototype.keys=function(){var t=[];return this.forEach(function(e,r){t.push(r)}),n(t)},o.prototype.values=function(){var t=[];return this.forEach(function(e){t.push(e)}),n(t)},o.prototype.entries=function(){var t=[];return this.forEach(function(e,r){t.push([r,e])}),n(t)},m.iterable&&(o.prototype[Symbol.iterator]=o.prototype.entries);var g=["DELETE","GET","HEAD","OPTIONS","POST","PUT"];d.prototype.clone=function(){return new d(this,{body:this._bodyInit})},l.call(d.prototype),l.call(b.prototype),b.prototype.clone=function(){return new b(this._bodyInit,{status:this.status,statusText:this.statusText,headers:new o(this.headers),url:this.url})},b.error=function(){var t=new b(null,{status:0,statusText:""});return t.type="error",t};var j=[301,302,303,307,308];b.redirect=function(t,e){if(-1===j.indexOf(e))throw new RangeError("Invalid status code");return new b(null,{status:e,headers:{location:t}})},t.Headers=o,t.Request=d,t.Response=b,t.fetch=function(t,e){return new Promise(function(r,n){var o=new d(t,e),i=new XMLHttpRequest;i.onload=function(){var t={status:i.status,statusText:i.statusText,headers:y(i.getAllResponseHeaders()||"")};t.url="responseURL"in i?i.responseURL:t.headers.get("X-Request-URL");var e="response"in i?i.response:i.responseText;r(new b(e,t))},i.onerror=function(){n(new TypeError("Network request failed"))},i.ontimeout=function(){n(new TypeError("Network request failed"))},i.open(o.method,o.url,!0),"include"===o.credentials&&(i.withCredentials=!0),"responseType"in i&&m.blob&&(i.responseType="blob"),o.headers.forEach(function(t,e){i.setRequestHeader(e,t)}),i.send("undefined"===typeof o._bodyInit?null:o._bodyInit)})},t.fetch.polyfill=!0}}("undefined"!==typeof self?self:this)},function(t,e,r){"use strict";function n(t){if(null===t||void 0===t)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(t)}var o=Object.getOwnPropertySymbols,i=Object.prototype.hasOwnProperty,s=Object.prototype.propertyIsEnumerable;t.exports=function(){try{if(!Object.assign)return!1;var t=new String("abc");if(t[5]="de","5"===Object.getOwnPropertyNames(t)[0])return!1;for(var e={},r=0;r<10;r++)e["_"+String.fromCharCode(r)]=r;if("0123456789"!==Object.getOwnPropertyNames(e).map(function(t){return e[t]}).join(""))return!1;var n={};return"abcdefghijklmnopqrst".split("").forEach(function(t){n[t]=t}),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},n)).join("")}catch(t){return!1}}()?Object.assign:function(t,e){for(var r,a,u=n(t),f=1;f 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | go to section 10 | 1 11 | 2 12 | 2 13 | 2 14 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/examples/one-page/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /tests/examples/other/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 404 6 | 7 | 8 | 9 | 404 dummy file 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/examples/other/ajax-request.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /tests/examples/other/css/bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stereobooster/react-snap/e2746ebe63cc75a4943d076585fff5aa5b50751d/tests/examples/other/css/bg.png -------------------------------------------------------------------------------- /tests/examples/other/css/small.css: -------------------------------------------------------------------------------- 1 | /* div is present in the document that is why it will be preserved */ 2 | div { 3 | background: url('bg.png'); 4 | height: 10px; 5 | } 6 | /* p is not present in the document but it will be preserved, 7 | because whole css file get inlined 8 | */ 9 | p { 10 | background: black; 11 | } 12 | -------------------------------------------------------------------------------- /tests/examples/other/fix-insert-rule.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |

test

9 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /tests/examples/other/form-elements.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | 12 | 15 | 22 | -------------------------------------------------------------------------------- /tests/examples/other/history-push-more.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /tests/examples/other/history-push.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /tests/examples/other/index.html: -------------------------------------------------------------------------------- 1 | dummy file 2 | -------------------------------------------------------------------------------- /tests/examples/other/js/main.js: -------------------------------------------------------------------------------- 1 | document.body.appendChild(document.createElement("p")); 2 | -------------------------------------------------------------------------------- /tests/examples/other/js/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": 1 3 | } 4 | -------------------------------------------------------------------------------- /tests/examples/other/link-to-file.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | img 9 | html 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/examples/other/localhost-links-different-port.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | localhost link on a different port 9 | localhost link with no port 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/examples/other/remove-blobs.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |

test

9 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/examples/other/snap-save-state.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/examples/other/svg.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | LINK 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /tests/examples/other/third-party-resource.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/examples/other/with-big-css.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /tests/examples/other/with-image.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/examples/other/with-script-error.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /tests/examples/other/with-script.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /tests/examples/other/with-small-css.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /tests/examples/partial/index.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/examples/partial/index.js: -------------------------------------------------------------------------------- 1 | if (navigator.userAgent === "ReactSnap") { 2 | // Strip out all content except the root 3 | while (document.firstChild) 4 | document.removeChild(document.firstChild); 5 | 6 | let div = document.createElement("div"); 7 | div.className = `root`; 8 | div.innerHTML = `This is my content`; 9 | 10 | document.appendChild(div); 11 | } -------------------------------------------------------------------------------- /tests/examples/processed/200.html: -------------------------------------------------------------------------------- 1 | dummy file 2 | -------------------------------------------------------------------------------- /tests/helper.js: -------------------------------------------------------------------------------- 1 | const nativeFs = require("fs"); 2 | 3 | const DevNullStream = require("dev-null-stream"); 4 | const cwd = process.cwd(); 5 | 6 | const mockFs = () => { 7 | const devNullStream = new DevNullStream(); 8 | const createReadStreamMock = jest.fn(); 9 | const createWriteStreamMock = jest.fn(); 10 | const writeFileSyncMock = jest.fn(); 11 | const fs = { 12 | existsSync: nativeFs.existsSync, 13 | createReadStream: path => { 14 | createReadStreamMock(path.replace(cwd, "")); 15 | return nativeFs.createReadStream(path); 16 | }, 17 | createWriteStream: path => { 18 | createWriteStreamMock(path.replace(cwd, "")); 19 | return devNullStream; 20 | }, 21 | writeFileSync: (path, content) => { 22 | writeFileSyncMock(path.replace(cwd, ""), content); 23 | } 24 | }; 25 | const filesCreated = () => writeFileSyncMock.mock.calls.length; 26 | const name = index => writeFileSyncMock.mock.calls[index][0]; 27 | const content = index => writeFileSyncMock.mock.calls[index][1]; 28 | const names = () => writeFileSyncMock.mock.calls.map(x => x[0]); 29 | return { 30 | // mocks 31 | createReadStreamMock, 32 | createWriteStreamMock, 33 | writeFileSyncMock, 34 | fs, 35 | // helpers 36 | filesCreated, 37 | content, 38 | name, 39 | names 40 | }; 41 | }; 42 | 43 | exports.mockFs = mockFs; 44 | -------------------------------------------------------------------------------- /tests/run.test.js: -------------------------------------------------------------------------------- 1 | // FIX: tests are slow - use unit tests instead of integration tests 2 | // TODO: capture console log from run function 3 | const fs = require("fs"); 4 | const writeFileSpy = jest.spyOn(fs, "writeFile"); 5 | writeFileSpy.mockImplementation((file, data, cb) => cb()); 6 | 7 | const { mockFs } = require("./helper.js"); 8 | const { run } = require("./../index.js"); 9 | const snapRun = (fs, options) => 10 | run( 11 | { 12 | // for Travis CI 13 | puppeteerArgs: ["--no-sandbox", "--disable-setuid-sandbox"], 14 | // sometimes web server from previous test have not enough time to shut down 15 | // as a result you get `Error: listen EADDRINUSE :::45678` 16 | // to prevent this we use random port 17 | port: Math.floor(Math.random() * 1000 + 45000), 18 | ...options 19 | }, 20 | { 21 | fs 22 | } 23 | ); 24 | 25 | describe("validates options", () => { 26 | test("include option should be an non-empty array", () => 27 | run({ include: "" }) 28 | .then(() => expect(true).toEqual(false)) 29 | .catch(e => expect(e).toEqual(""))); 30 | 31 | test("preloadResources option deprecated. Use preloadImages or cacheAjaxRequests", () => 32 | run({ preloadResources: true }) 33 | .then(() => expect(true).toEqual(false)) 34 | .catch(e => expect(e).toEqual(""))); 35 | 36 | test("saveAs supported values are html and png", () => 37 | run({ saveAs: "json" }) 38 | .then(() => expect(true).toEqual(false)) 39 | .catch(e => expect(e).toEqual(""))); 40 | }); 41 | 42 | describe("one page", () => { 43 | const source = "tests/examples/one-page"; 44 | const { 45 | fs, 46 | createReadStreamMock, 47 | createWriteStreamMock, 48 | filesCreated, 49 | content, 50 | name 51 | } = mockFs(); 52 | beforeAll(() => snapRun(fs, { source })); 53 | test("crawls / and saves as index.html to the same folder", () => { 54 | expect(filesCreated()).toEqual(1); 55 | expect(name(0)).toEqual(`/${source}/index.html`); 56 | expect(content(0)).toEqual( 57 | 'test' 58 | ); 59 | }); 60 | test("copies (original) index.html to 200.html", () => { 61 | expect(createReadStreamMock.mock.calls).toEqual([ 62 | [`/${source}/index.html`] 63 | ]); 64 | expect(createWriteStreamMock.mock.calls).toEqual([[`/${source}/200.html`]]); 65 | }); 66 | }); 67 | 68 | describe("saveAs png", () => { 69 | const source = "tests/examples/one-page"; 70 | const cwd = process.cwd(); 71 | const { 72 | fs: mockedFs, 73 | createReadStreamMock, 74 | createWriteStreamMock 75 | } = mockFs(); 76 | beforeAll(() => snapRun(mockedFs, { source, saveAs: "png" })); 77 | afterAll(() => writeFileSpy.mockClear()); 78 | test("crawls / and saves as index.png to the same folder", () => { 79 | expect(writeFileSpy).toHaveBeenCalledTimes(1); 80 | expect(writeFileSpy.mock.calls[0][0]).toEqual(cwd + `/${source}/index.png`); 81 | }); 82 | test("copies (original) index.html to 200.html", () => { 83 | expect(createReadStreamMock.mock.calls).toEqual([ 84 | [`/${source}/index.html`] 85 | ]); 86 | expect(createWriteStreamMock.mock.calls).toEqual([[`/${source}/200.html`]]); 87 | }); 88 | }); 89 | 90 | describe("saveAs jpeg", () => { 91 | const source = "tests/examples/one-page"; 92 | const cwd = process.cwd(); 93 | const { 94 | fs: mockedFs, 95 | createReadStreamMock, 96 | createWriteStreamMock 97 | } = mockFs(); 98 | beforeAll(() => snapRun(mockedFs, { source, saveAs: "jpeg" })); 99 | afterAll(() => writeFileSpy.mockClear()); 100 | test("crawls / and saves as index.png to the same folder", () => { 101 | expect(writeFileSpy).toHaveBeenCalledTimes(1); 102 | expect(writeFileSpy.mock.calls[0][0]).toEqual( 103 | cwd + `/${source}/index.jpeg` 104 | ); 105 | }); 106 | test("copies (original) index.html to 200.html", () => { 107 | expect(createReadStreamMock.mock.calls).toEqual([ 108 | [`/${source}/index.html`] 109 | ]); 110 | expect(createWriteStreamMock.mock.calls).toEqual([[`/${source}/200.html`]]); 111 | }); 112 | }); 113 | 114 | describe("respects destination", () => { 115 | const source = "tests/examples/one-page"; 116 | const destination = "tests/examples/destination"; 117 | const { 118 | fs, 119 | createReadStreamMock, 120 | createWriteStreamMock, 121 | filesCreated, 122 | content, 123 | name 124 | } = mockFs(); 125 | beforeAll(() => snapRun(fs, { source, destination })); 126 | test("crawls / and saves as index.html to destination folder", () => { 127 | expect(filesCreated()).toEqual(1); 128 | expect(name(0)).toEqual(`/${destination}/index.html`); 129 | }); 130 | test("copies (original) index.html to 200.html (to source folder)", () => { 131 | expect(createReadStreamMock.mock.calls[0]).toEqual([ 132 | `/${source}/index.html` 133 | ]); 134 | expect(createWriteStreamMock.mock.calls[0]).toEqual([ 135 | `/${source}/200.html` 136 | ]); 137 | }); 138 | test("copies (original) index.html to 200.html (to destination folder)", () => { 139 | expect(createReadStreamMock.mock.calls[1]).toEqual([ 140 | `/${source}/index.html` 141 | ]); 142 | expect(createWriteStreamMock.mock.calls[1]).toEqual([ 143 | `/${destination}/200.html` 144 | ]); 145 | }); 146 | }); 147 | 148 | describe("many pages", () => { 149 | const source = "tests/examples/many-pages"; 150 | const { 151 | fs, 152 | createReadStreamMock, 153 | createWriteStreamMock, 154 | filesCreated, 155 | name, 156 | names 157 | } = mockFs(); 158 | beforeAll(() => snapRun(fs, { source })); 159 | test("crawls all links and saves as index.html in separate folders", () => { 160 | expect(filesCreated()).toEqual(7); 161 | expect(names()).toEqual( 162 | expect.arrayContaining([ 163 | `/${source}/1/index.html`, // without slash in the end 164 | `/${source}/2/index.html`, // with slash in the end 165 | `/${source}/3/index.html`, // ignores hash 166 | `/${source}/4/index.html`, // ignores query 167 | `/${source}/5/index.html`, // link rel="alternate" 168 | ]) 169 | ); 170 | }); 171 | test("crawls / and saves as index.html to the same folder", () => { 172 | expect(name(0)).toEqual(`/${source}/index.html`); 173 | }); 174 | test("if there is more than page it crawls 404.html", () => { 175 | expect(names()).toEqual(expect.arrayContaining([`/${source}/404.html`])); 176 | }); 177 | test("copies (original) index.html to 200.html", () => { 178 | expect(createReadStreamMock.mock.calls).toEqual([ 179 | [`/${source}/index.html`] 180 | ]); 181 | expect(createWriteStreamMock.mock.calls).toEqual([[`/${source}/200.html`]]); 182 | }); 183 | }); 184 | 185 | describe("possible to disable crawl option", () => { 186 | const source = "tests/examples/many-pages"; 187 | const { 188 | fs, 189 | createReadStreamMock, 190 | createWriteStreamMock, 191 | filesCreated, 192 | names 193 | } = mockFs(); 194 | beforeAll(() => 195 | snapRun(fs, { 196 | source, 197 | crawl: false, 198 | include: ["/1", "/2/", "/3#test", "/4?test"] 199 | }) 200 | ); 201 | test("crawls all links and saves as index.html in separate folders", () => { 202 | // no / or /404.html 203 | expect(filesCreated()).toEqual(4); 204 | expect(names()).toEqual( 205 | expect.arrayContaining([ 206 | `/${source}/1/index.html`, // without slash in the end 207 | `/${source}/2/index.html`, // with slash in the end 208 | `/${source}/3/index.html`, // ignores hash 209 | `/${source}/4/index.html` // ignores query 210 | ]) 211 | ); 212 | }); 213 | test("copies (original) index.html to 200.html", () => { 214 | expect(createReadStreamMock.mock.calls).toEqual([ 215 | [`/${source}/index.html`] 216 | ]); 217 | expect(createWriteStreamMock.mock.calls).toEqual([[`/${source}/200.html`]]); 218 | }); 219 | }); 220 | 221 | describe("inlineCss - small file", () => { 222 | const source = "tests/examples/other"; 223 | const { fs, filesCreated, content } = mockFs(); 224 | beforeAll(() => 225 | snapRun(fs, { 226 | source, 227 | inlineCss: true, 228 | include: ["/with-small-css.html"] 229 | }) 230 | ); 231 | // 1. I want to change this behaviour 232 | // see https://github.com/stereobooster/react-snap/pull/133/files 233 | // 2. There is a bug with relative url in inlined CSS url(bg.png) 234 | test("whole CSS got inlined for small", () => { 235 | expect(filesCreated()).toEqual(1); 236 | expect(content(0)).toMatch( 237 | '' 238 | ); 239 | }); 240 | test("removes ", () => { 241 | expect(content(0)).not.toMatch( 242 | '' 243 | ); 244 | }); 245 | }); 246 | 247 | describe("inlineCss - big file", () => { 248 | const source = "tests/examples/other"; 249 | const include = ["/with-big-css.html"]; 250 | const { fs, filesCreated, content } = mockFs(); 251 | beforeAll(() => snapRun(fs, { source, include, inlineCss: true })); 252 | test("inline style", () => { 253 | expect(filesCreated()).toEqual(1); 254 | expect(content(0)).toMatch(''); 344 | }); 345 | }); 346 | 347 | describe("removeStyleTags", () => { 348 | const source = "tests/examples/other"; 349 | const include = ["/fix-insert-rule.html"]; 350 | const { fs, filesCreated, content } = mockFs(); 351 | beforeAll(() => 352 | snapRun(fs, { 353 | source, 354 | include, 355 | removeStyleTags: true 356 | }) 357 | ); 358 | test("removes all