├── .gitignore ├── .travis.yml ├── README.md ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.11" 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # koa-prerender [![Build Status](https://travis-ci.org/RisingStack/koa-prerender.svg)](https://travis-ci.org/RisingStack/koa-prerender) 2 | 3 | [![NPM](https://nodei.co/npm/koa-prerender.png)](https://nodei.co/npm/koa-prerender/) 4 | 5 | **KOA middleware for prerendering javascript-rendered pages on the fly for SEO** 6 | 7 | This [koa](https://koajs.com) middleware intercepts requests to your Node.js website from crawlers, and then makes a call to the (external) 8 | [Prerender](https://prerender.io/) service to get the static HTML instead of the javascript for that page. 9 | 10 | ## Setup 11 | 12 | ### Prerequisites 13 | 14 | Install [Prerender](https://github.com/prerender/prerender) on a server of your choice. 15 | 16 | ### Install 17 | 18 | Install the [package](https://npmjs.org/package/koa-prerender) with [npm](https://npmjs.org): 19 | 20 | ```sh 21 | $ npm install koa-prerender 22 | ``` 23 | 24 | ### Usage 25 | 26 | ```js 27 | var prerender = require('koa-prerender'); 28 | 29 | // Options 30 | var options = { 31 | prerender: PRERENDER_SERVER_URL // optional, default:'http://service.prerender.io/' 32 | protocol: 'http', // optional, default: this.protocol 33 | host: 'www.risingstack.com' // optional, default: this.host, 34 | prerenderToken: '' // optional or process.env.PRERENDER_TOKEN 35 | }; 36 | 37 | // Use as middleware 38 | app.use(prerender(options)); 39 | ``` 40 | 41 | ## License 42 | 43 | ISC 44 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @module koa-prerender 3 | * 4 | * @author Peter Marton, Gergely Nemeth 5 | */ 6 | 7 | var url = require('url'); 8 | 9 | var request = require('request'); 10 | var thunkify = require('thunkify-wrap'); 11 | 12 | // Turn callback into a thunk 13 | var requestGet = thunkify(request.get); 14 | 15 | var crawlerUserAgents = [ 16 | 'baiduspider', 17 | 'facebookexternalhit', 18 | 'twitterbot', 19 | 'rogerbot', 20 | 'linkedinbot', 21 | 'embedly', 22 | 'quora link preview', 23 | 'showyoubot', 24 | 'outbrain', 25 | 'pinterest', 26 | 'developers.google.com/+/web/snippet' 27 | ]; 28 | 29 | var extensionsToIgnore = [ 30 | '.js', 31 | '.css', 32 | '.xml', 33 | '.less', 34 | '.png', 35 | '.jpg', 36 | '.jpeg', 37 | '.gif', 38 | '.pdf', 39 | '.doc', 40 | '.txt', 41 | '.ico', 42 | '.rss', 43 | '.zip', 44 | '.mp3', 45 | '.rar', 46 | '.exe', 47 | '.wmv', 48 | '.doc', 49 | '.avi', 50 | '.ppt', 51 | '.mpg', 52 | '.mpeg', 53 | '.tif', 54 | '.wav', 55 | '.mov', 56 | '.psd', 57 | '.ai', 58 | '.xls', 59 | '.mp4', 60 | '.m4a', 61 | '.swf', 62 | '.dat', 63 | '.dmg', 64 | '.iso', 65 | '.flv', 66 | '.m4v', 67 | '.torrent' 68 | ]; 69 | 70 | var DEFAULT_PRERENDER = 'http://service.prerender.io/'; 71 | 72 | 73 | /* 74 | * Should pre-render? 75 | * 76 | * @method shouldPreRender 77 | * @param {Object} options 78 | * @return {Boolean} 79 | */ 80 | function shouldPreRender (options) { 81 | var hasExtensionToIgnore = extensionsToIgnore.some(function (extension) { 82 | return options.url.indexOf(extension) !== -1; 83 | }); 84 | 85 | var isBot = crawlerUserAgents.some(function (crawlerUserAgent) { 86 | return options.userAgent.toLowerCase().indexOf(crawlerUserAgent.toLowerCase()) !== -1; 87 | }); 88 | 89 | // do not pre-rend when: 90 | if (!options.userAgent) { 91 | return false; 92 | } 93 | 94 | if (options.method !== 'GET') { 95 | return false; 96 | } 97 | 98 | if (hasExtensionToIgnore) { 99 | return false; 100 | } 101 | 102 | // do pre-render when: 103 | var query = url.parse(options.url, true).query; 104 | if (query && query.hasOwnProperty('_escaped_fragment_')) { 105 | return true; 106 | } 107 | 108 | if (options.bufferAgent) { 109 | return true; 110 | } 111 | 112 | return isBot; 113 | } 114 | 115 | 116 | /* 117 | * Pre-render middleware 118 | * 119 | * @method preRenderMiddleware 120 | * @param {Object} options 121 | */ 122 | module.exports = function preRenderMiddleware (options) { 123 | options = options || {}; 124 | options.prerender = options.prerender || DEFAULT_PRERENDER; 125 | 126 | /* 127 | * Pre-render 128 | * 129 | * @method preRender 130 | * @param {Generator} next 131 | */ 132 | return function *preRender(next) { 133 | var protocol = options.protocol || this.protocol; 134 | var host = options.host || this.host; 135 | var headers = { 136 | 'User-Agent': this.accept.headers['user-agent'] 137 | }; 138 | 139 | var prePreRenderToken = options.prerenderToken || process.env.PRERENDER_TOKEN; 140 | 141 | if(prePreRenderToken) { 142 | headers['X-Prerender-Token'] = prePreRenderToken; 143 | } 144 | 145 | var isPreRender = shouldPreRender({ 146 | userAgent: this.get('user-agent'), 147 | bufferAgent: this.get('x-bufferbot'), 148 | method: this.method, 149 | url: this.url 150 | }); 151 | 152 | var body = ''; 153 | 154 | var renderUrl; 155 | var preRenderUrl; 156 | var response; 157 | 158 | // Pre-render generate the site and return 159 | if (isPreRender) { 160 | renderUrl = protocol + '://' + host + this.url; 161 | preRenderUrl = options.prerender + renderUrl; 162 | response = yield requestGet({ 163 | url: preRenderUrl, 164 | headers: headers, 165 | gzip: true 166 | }); 167 | 168 | body = response[1] || ''; 169 | 170 | yield* next; 171 | 172 | this.body = body.toString(); 173 | this.set('X-Prerender', 'true'); 174 | } else { 175 | yield* next; 176 | this.set('X-Prerender', 'false'); 177 | } 178 | }; 179 | }; 180 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "koa-prerender", 3 | "version": "1.0.3", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha test.js --harmony --reporter spec" 8 | }, 9 | "author": "Gergely Nemeth (http://twitter.com/nthgergo)", 10 | "license": "ISC", 11 | "dependencies": { 12 | "request": "^2.40.0", 13 | "thunkify-wrap": "^1.0.2" 14 | }, 15 | "devDependencies": { 16 | "chai": "^1.9.1", 17 | "koa": "koajs/koa", 18 | "mocha": "~1.12.0", 19 | "supertest": "^0.13.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var prerender = require('./'); 2 | var request = require('supertest'); 3 | var koa = require('koa'); 4 | var expect = require('chai').expect; 5 | 6 | describe('Koa prerender middleware', function() { 7 | this.timeout(5000); 8 | 9 | it('exists', function () { 10 | expect(prerender).to.be.ok; 11 | }); 12 | 13 | describe('prerenders when', function() { 14 | 15 | it('url contains _escaped_fragment_', function (done) { 16 | var app = koa(); 17 | 18 | app.use(prerender()); 19 | 20 | request(app.listen()) 21 | .get('/?_escaped_fragment_') 22 | .expect('X-Prerender', 'true') 23 | .expect(200, done); 24 | }); 25 | 26 | it('user-agent looks like a bot', function (done) { 27 | var app = koa(); 28 | 29 | app.use(prerender()); 30 | 31 | request(app.listen()) 32 | .get('/') 33 | .set('user-agent', 'twitterbot') 34 | .expect('X-Prerender', 'true') 35 | .expect(200, done); 36 | }); 37 | 38 | it('x-bufferbot is set in options', function (done) { 39 | var app = koa(); 40 | 41 | app.use(prerender()); 42 | 43 | request(app.listen()) 44 | .get('/') 45 | .set('x-bufferbot', 'whatever') 46 | .expect('X-Prerender', 'true') 47 | .expect(200, done); 48 | 49 | }); 50 | 51 | }); 52 | 53 | }); 54 | --------------------------------------------------------------------------------