This is a test
' 65 | }]); 66 | ``` 67 | 68 | Work with `bottleneck` 69 | -------------------- 70 | Control rate limits for each connection, usually used with proxy. 71 | 72 | ```javascript 73 | var Crawler = require("node-webcrawler"); 74 | 75 | var c = new Crawler({ 76 | maxConnections : 3, 77 | rateLimits:2000, 78 | callback : function (error, result, $) { 79 | if(error){ 80 | console.error(error); 81 | }else{ 82 | console.log($('title').text()); 83 | } 84 | } 85 | }); 86 | 87 | c.queue({ 88 | uri:"http://www.google.com", 89 | limiter:"key1",// for connection of 'key1' 90 | proxy:"http://user:pass@127.0.0.1:8080" 91 | }); 92 | 93 | c.queue({ 94 | uri:"http://www.google.com", 95 | limiter:"key2", // for connection of 'key2' 96 | proxy:"http://user:pass@127.0.0.1:8082" 97 | }); 98 | 99 | c.queue({ 100 | uri:"http://www.google.com", 101 | limiter:"key3", // for connection of 'key3' 102 | proxy:"http://user:pass@127.0.0.1:8081" 103 | }); 104 | 105 | ``` 106 | 107 | Options reference 108 | ----------------- 109 | 110 | You can pass these options to the Crawler() constructor if you want them to be global or as 111 | items in the queue() calls if you want them to be specific to that item (overwriting global options) 112 | 113 | This options list is a strict superset of [mikeal's request options](https://github.com/mikeal/request#requestoptions-callback) and will be directly passed to 114 | the request() method. 115 | 116 | Basic request options: 117 | 118 | * `uri`: String, the URL you want to crawl 119 | * `timeout` : Number, in milliseconds (Default 15000) 120 | * [All mikeal's requests options are accepted](https://github.com/mikeal/request#requestoptions-callback) 121 | 122 | Callbacks: 123 | 124 | * `callback(error, result, $)`: A request was completed 125 | * `onDrain(pool)`: There is no more queued requests, call `pool.destroyAllNow()` if you wanna release resources in pool to, or if you have follow-up tasks to queue you can ignore. 126 | 127 | Pool options: 128 | 129 | * `maxConnections`: Number, Size of the worker pool (Default 10), 130 | * `priorityRange`: Number, Range of acceptable priorities starting from 0 (Default 10), 131 | * `priority`: Number, Priority of this request (Default 5), 132 | 133 | Retry options: 134 | 135 | * `retries`: Number of retries if the request fails (Default 3), 136 | * `retryTimeout`: Number of milliseconds to wait before retrying (Default 10000), 137 | 138 | Server-side DOM options: 139 | 140 | * `jQuery`: true, false or ConfObject (Default true) 141 | 142 | Charset encoding: 143 | 144 | * `forceUTF8`: Boolean, if true will get charset from HTTP headers or meta tag in html and convert it to UTF8 if necessary. Never worry about encoding anymore! (Default false), 145 | * `incomingEncoding`: String, with forceUTF8: true to set encoding manually (Default null) 146 | `incomingEncoding : 'windows-1255'` for example 147 | 148 | Cache: 149 | 150 | * `cache`: Boolean, if true stores requests' result in memory (Default false), not recommend if you are doing with huge amount of pages as the process will exhaust momery 151 | * `skipDuplicates`: Boolean, if true skips URIs that were already crawled, without even calling callback() (Default false) 152 | 153 | Other: 154 | * `rotateUA`: Boolean, if true, `userAgent` should be an array, and rotate it (Default false) 155 | * `userAgent`: String or Array, if `rotateUA` is false, but `userAgent` is array, will use first one. 156 | * `referer`: String, if truthy sets the HTTP referer header 157 | * `rateLimits`: Number of milliseconds to delay between each requests (Default 0) 158 | 159 | 160 | Class:Crawler 161 | ------------- 162 | 163 | Instance of Crawler 164 | 165 | __crawler.queue(uri|options)__ 166 | * `uri` String, `options` is [Options](#options-reference) 167 | 168 | __crawler.queueSize__ 169 | 170 | Size of queue, read-only 171 | 172 | 173 | Working with Cheerio or JSDOM 174 | ----------------------------- 175 | 176 | Crawler by default use [Cheerio](https://github.com/cheeriojs/cheerio) instead of [Jsdom](https://github.com/tmpvar/jsdom). Jsdom is more robust but can be hard to install (espacially on windows) because of [contextify](https://github.com/tmpvar/jsdom#contextify). 177 | Which is why, if you want to use jsdom you will have to build it, and `require('jsdom')` in your own script before passing it to crawler. This is to avoid cheerio crawler user to build jsdom when installing crawler. 178 | 179 | ###Working with Cheerio 180 | ```javascript 181 | jQuery: true //(default) 182 | //OR 183 | jQuery: 'cheerio' 184 | //OR 185 | jQuery: { 186 | name: 'cheerio', 187 | options: { 188 | normalizeWhitespace: true, 189 | xmlMode: true 190 | } 191 | } 192 | ``` 193 | These parsing options are taken directly from [htmlparser2](https://github.com/fb55/htmlparser2/wiki/Parser-options), therefore any options that can be used in `htmlparser2` are valid in cheerio as well. The default options are: 194 | 195 | ```js 196 | { 197 | normalizeWhitespace: false, 198 | xmlMode: false, 199 | decodeEntities: true 200 | } 201 | ``` 202 | 203 | For a full list of options and their effects, see [this](https://github.com/fb55/DomHandler) and 204 | [htmlparser2's options](https://github.com/fb55/htmlparser2/wiki/Parser-options). 205 | [source](https://github.com/cheeriojs/cheerio#loading) 206 | 207 | ###Working with JSDOM 208 | 209 | In order to work with JSDOM you will have to install it in your project folder `npm install jsdom`, deal with [compiling C++](https://github.com/tmpvar/jsdom#contextify) and pass it to crawler. 210 | ```javascript 211 | var jsdom = require('jsdom'); 212 | var Crawler = require('node-webcrawler'); 213 | 214 | var c = new Crawler({ 215 | jQuery: jsdom 216 | }); 217 | ``` 218 | 219 | How to test 220 | ----------- 221 | 222 | ### Install and run Httpbin 223 | 224 | node-webcrawler use a local httpbin for testing purpose. You can install httpbin as a library from PyPI and run it as a WSGI app. For example, using Gunicorn: 225 | 226 | $ pip install httpbin 227 | // launch httpbin as a daemon with 6 worker on localhost 228 | $ gunicorn httpbin:app -b 127.0.0.1:8000 -w 6 --daemon 229 | 230 | // Finally 231 | $ npm install && npm test 232 | 233 | ### Alternative: Docker 234 | 235 | After [installing Docker](http://docs.docker.com/), you can run: 236 | 237 | // Builds the local test environment 238 | $ docker build -t node-webcrawler . 239 | 240 | // Runs tests 241 | $ docker run node-webcrawler sh -c "gunicorn httpbin:app -b 127.0.0.1:8000 -w 6 --daemon && npm install && npm test" 242 | 243 | // You can also ssh into the container for easier debugging 244 | $ docker run -i -t node-webcrawler bash 245 | 246 | 247 | [](https://travis-ci.org/bda-research/node-webcrawler) 248 | 249 | Rough todolist 250 | -------------- 251 | 252 | * Using bottleneck to deal with rate limits 253 | * Introducing zombie to deal with page with complex ajax 254 | * Refactoring the code to be more maintenable, it's spaghetti code in there ! 255 | * Proxy feature 256 | * This issue: https://github.com/sylvinus/node-crawler/issues/118 257 | * Make Sizzle tests pass (jsdom bug? https://github.com/tmpvar/jsdom/issues#issue/81) 258 | * More crawling tests 259 | * Document the API more (+ the result object) 260 | * Option to wait for callback to finish before freeing the pool resource (via another callback like next()) 261 | 262 | 263 | ChangeLog 264 | --------- 265 | 266 | See https://github.com/bda-research/node-webcrawler/releases 267 | -------------------------------------------------------------------------------- /lib/crawler.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var path = require('path'), 4 | util = require('util'), 5 | EventEmitter = require('events').EventEmitter, 6 | request = require('request'), 7 | _ = require('lodash'), 8 | cheerio = require('cheerio'), 9 | fs = require('fs'), 10 | Pool = require('generic-pool').Pool, 11 | charsetParser = require('charset-parser'), 12 | Bottleneck = require('bottleneck'), 13 | seenreq = require('seenreq'); 14 | 15 | var logger = null; 16 | // Fallback on iconv-lite if we didn't succeed compiling iconv 17 | // https://github.com/sylvinus/node-crawler/pull/29 18 | var iconv, iconvLite; 19 | try { 20 | iconv = require('iconv').Iconv; 21 | } catch (e) {} 22 | 23 | if (!iconv) { 24 | iconvLite = require('iconv-lite'); 25 | } 26 | 27 | function useCache (options) { 28 | return (options.uri || options.url) && options.cache; 29 | } 30 | 31 | function checkJQueryNaming (options) { 32 | if ('jquery' in options) { 33 | options.jQuery = options.jquery; 34 | delete options.jquery; 35 | } 36 | return options; 37 | } 38 | 39 | function readJqueryUrl (url, callback) { 40 | if (url.match(/^(file\:\/\/|\w+\:|\/)/)) { 41 | fs.readFile(url.replace(/^file\:\/\//,''),'utf-8', function(err,jq) { 42 | callback(err, jq); 43 | }); 44 | } else { 45 | callback(null, url); 46 | } 47 | } 48 | 49 | function Crawler (options) { 50 | var self = this; 51 | self.init(options); 52 | } 53 | // augment the prototype for node events using util.inherits 54 | util.inherits(Crawler, EventEmitter); 55 | 56 | Crawler.prototype.init = function init (options) { 57 | var self = this; 58 | 59 | var defaultOptions = { 60 | autoWindowClose: true, 61 | cache: false, 62 | forceUTF8: false, 63 | gzip: true, 64 | incomingEncoding: null, //TODO remove or optimize 65 | jQuery: true, 66 | maxConnections: 10, 67 | bottleneckConcurrent: 10000, 68 | method: 'GET', 69 | onDrain: false, 70 | priority: 5, 71 | priorityRange: 10, 72 | rateLimits: 0, 73 | referer: false, 74 | retries: 3, 75 | retryTimeout: 10000, 76 | timeout: 15000, 77 | skipDuplicates: false, 78 | rotateUA: false 79 | }; 80 | 81 | //return defaultOptions with overriden properties from options. 82 | self.options = _.extend(defaultOptions, options); 83 | 84 | // you can use jquery or jQuery 85 | self.options = checkJQueryNaming(self.options); 86 | 87 | if (self.options.rateLimits !== 0 && self.options.maxConnections == 1) { 88 | //self.options.limiter = "default"; 89 | } else { 90 | //self.limiters = null;//self.options.maxConnections = 1; 91 | } 92 | 93 | // Don't make these options persist to individual queries 94 | self.globalOnlyOptions = ['maxConnections', 'priorityRange', 'onDrain']; 95 | 96 | //Setup a worker pool w/ https://github.com/coopernurse/node-pool 97 | self.pool = Pool({ 98 | name : 'crawler', 99 | max : self.options.maxConnections, 100 | min : self.options.minConnections, 101 | log : self.options.debug && self.options.logger && function(){self.options.logger.log(arguments[1],arguments[0]);}, 102 | priorityRange: self.options.priorityRange, 103 | create : function(callback) { 104 | callback(new Object()); 105 | }, 106 | destroy : function() {} 107 | }); 108 | 109 | self.limiters = new Bottleneck.Cluster(self.options.bottleneckConcurrent,self.options.rateLimits); 110 | self.plannedQueueCallsCount = 0; 111 | self.queueItemSize = 0; 112 | 113 | self.cache = {}; 114 | self.seen = new seenreq(); 115 | self.debug = self.options.debug || false; 116 | self.mapEntity = Object.create(null); 117 | self.entityList = ["jar"]; 118 | logger = self.options.logger || console; 119 | 120 | self.on('pool:release', function(options) { 121 | self._release(options); 122 | }); 123 | 124 | self.on("request",function(options){ 125 | if(_.isFunction(self.options.preRequest)){ 126 | self.options.preRequest(options); 127 | } 128 | }); 129 | 130 | self.on('pool:drain', function() { 131 | if (self.options.onDrain) { 132 | self.options.onDrain.call(self, self.pool); 133 | } 134 | }); 135 | }; 136 | 137 | Crawler.prototype._release = function _release (options) { 138 | var self = this; 139 | 140 | self.queueItemSize--; 141 | if (options._poolReference) { 142 | if(self.debug){ 143 | logger.info("Releasing resource, limiter:%s", options.limiter || "default"); 144 | } 145 | self.pool.release(options._poolReference); 146 | } 147 | 148 | // Pool stats are behaving weird - have to implement our own counter 149 | if (self.queueItemSize + self.plannedQueueCallsCount === 0) { 150 | self.emit('pool:drain'); 151 | } 152 | }; 153 | 154 | Crawler.prototype._inject = function _inject (response, options, callback) { 155 | var $; 156 | var self = this; 157 | 158 | if (options.jQuery === 'cheerio' || options.jQuery.name === 'cheerio' || options.jQuery === true) { 159 | var defaultCheerioOptions = { 160 | normalizeWhitespace: false, 161 | xmlMode: false, 162 | decodeEntities: true 163 | }; 164 | var cheerioOptions = options.jQuery.options || defaultCheerioOptions; 165 | $ = cheerio.load(response.body, cheerioOptions); 166 | 167 | callback(null, $); 168 | } 169 | 170 | else if (options.jQuery.jsdom) { 171 | var jsdom = options.jQuery.jsdom; 172 | var scriptLocation = path.resolve(__dirname, '../vendor/jquery-2.1.1.min.js'); 173 | 174 | //Use promises 175 | readJqueryUrl(scriptLocation, function(err, jquery) { 176 | try { 177 | jsdom.env({ 178 | url: options.uri, 179 | html: response.body, 180 | src: [jquery], 181 | done: function (errors, window) { 182 | $ = window.jQuery; 183 | callback(errors, $); 184 | 185 | try { 186 | window.close(); 187 | window = null; 188 | } catch (err) { 189 | logger.error(err); 190 | } 191 | 192 | } 193 | }); 194 | } catch (e) { 195 | options.callback(e); 196 | self.emit('pool:release', options); 197 | } 198 | }); 199 | } 200 | // Jquery is set to false are not set 201 | else { 202 | callback(null); 203 | } 204 | }; 205 | 206 | Crawler.prototype.queue = function queue (options) { 207 | var self = this; 208 | 209 | // Did you get a single object or string? Make it compatible. 210 | options = _.isArray(options) ? options : [options]; 211 | 212 | options = _.flattenDeep(options); 213 | 214 | for(var i = 0; i < options.length; ++i) { 215 | if(_.isNull(options[i]) || _.isUndefined(options[i]) || (!_.isString(options[i]) && !_.isPlainObject(options[i]))) { 216 | if(self.debug) { 217 | logger.warn("Illegal queue option: ", JSON.stringify(options[i])); 218 | } 219 | continue; 220 | } 221 | self._pushToQueue( 222 | _.isString(options[i]) ? {uri: options[i]} : options[i] 223 | ); 224 | } 225 | }; 226 | 227 | Crawler.prototype._pushToQueue = function _pushToQueue (options) { 228 | var self = this; 229 | self.queueItemSize++; 230 | 231 | // you can use jquery or jQuery 232 | options = checkJQueryNaming(options); 233 | 234 | _.defaults(options, self.options); 235 | 236 | // Remove all the global options from our options 237 | // TODO we are doing this for every _pushToQueue, find a way to avoid this 238 | _.each(self.globalOnlyOptions, function(globalOnlyOption) { 239 | delete options[globalOnlyOption]; 240 | }); 241 | 242 | // If duplicate skipping is enabled, avoid queueing entirely for URLs we already crawled 243 | if (options.skipDuplicates && self.seen.exists(options)) { 244 | return self.emit('pool:release', options); 245 | } 246 | 247 | // acquire connection - callback function is called 248 | // once a resource becomes available 249 | // self.pool.acquire( 250 | var acquired = function(error, poolReference) { 251 | options._poolReference = poolReference; 252 | 253 | // this is and operation error 254 | if (error) { 255 | logger.error(error); 256 | options.callback(error);// need release 257 | return self.emit('pool:release',options); 258 | } 259 | 260 | if(self.debug){ 261 | logger.info("Acquired resource, limiter:%s, uri:%s", options.limiter || "default", options.uri); 262 | logger.info("pool queue size:%s, bottleneck '%s' queue size:%s", self.waitingCount, options.limiter||"default", self.limiters.key(options.limiter||"default")._queue.length); 263 | } 264 | 265 | //Static HTML was given, skip request 266 | if (options.html) { 267 | self._onContent(null, options, {body:options.html}); 268 | } else if (typeof options.uri === 'function') { 269 | options.uri(function(uri) { 270 | options.uri = uri; 271 | self._makeCrawlerRequest(options); 272 | }); 273 | } else { 274 | self._makeCrawlerRequest(options); 275 | } 276 | }//, options.priority); 277 | 278 | var acquireWrapped = function(priority,cb){ 279 | if(self.debug){ 280 | logger.info("Called by bottleneck, limiter:%s, uri:%s", options.limiter || "default", options.uri); 281 | } 282 | 283 | return self.pool.acquire(cb,priority); 284 | } 285 | 286 | self.limiters.key(options.limiter||"default").submit(acquireWrapped,options.priority,acquired); 287 | }; 288 | 289 | Crawler.prototype._makeCrawlerRequest = function _makeCrawlerRequest (options) { 290 | var self = this; 291 | //var cacheData = self.cache[self.seen.normalize(options)]; 292 | 293 | // if(useCache(options) && cacheData){ 294 | // if(self.debug){ 295 | // logger.info("using cache."); 296 | // } 297 | 298 | // self._onContent(null, options, cacheData, true); 299 | // return; 300 | // } 301 | 302 | // if (typeof options.rateLimits === 'number' && options.rateLimits !== 0) { 303 | // setTimeout(function() { 304 | // self._buildHttpRequest(options); 305 | // }, options.rateLimits); 306 | // } else { 307 | self._buildHttpRequest(options); 308 | // } 309 | }; 310 | 311 | Crawler.prototype._deleteEntity = function _deleteEntity(options){ 312 | var self = this; 313 | this.entityList.forEach(function(name){ 314 | if(typeof options[name] == "object"){ 315 | self.mapEntity[name] = options[name]; 316 | delete options[name]; 317 | } 318 | }) 319 | } 320 | 321 | Crawler.prototype._attachEntity = function _attachEntity(options){ 322 | var self = this; 323 | return this.entityList.reduce(function(target,name){ 324 | if(typeof self.mapEntity[name] == "object") 325 | target[name] = self.mapEntity[name]; 326 | 327 | return target; 328 | }, options); 329 | } 330 | 331 | 332 | Crawler.prototype._buildHttpRequest = function _buildHTTPRequest (options) { 333 | var self = this; 334 | 335 | if (self.debug) { 336 | logger.info(options.method+' '+options.uri); 337 | if(options.proxy) 338 | logger.info("Use proxy: %s", options.proxy); 339 | } 340 | 341 | // Cloning keeps the opts parameter clean: 342 | // - some versions of "request" apply the second parameter as a 343 | // property called "callback" to the first parameter 344 | // - keeps the query object fresh in case of a retry 345 | // Doing parse/stringify instead of _.clone will do a deep clone and remove functions 346 | 347 | self._deleteEntity(options); 348 | var ropts = JSON.parse(JSON.stringify(options)); 349 | self._attachEntity(ropts); 350 | 351 | if (!ropts.headers) { ropts.headers={}; } 352 | if (ropts.forceUTF8) { 353 | // if (!ropts.headers['Accept-Charset'] && !ropts.headers['accept-charset']) { 354 | // ropts.headers['Accept-Charset'] = 'utf-8;q=0.7,*;q=0.3'; 355 | // } 356 | 357 | ropts.encoding=null; 358 | } 359 | 360 | if (ropts.userAgent) { 361 | if(ropts.rotateUA && _.isArray(ropts.userAgent)){ 362 | ropts.headers['User-Agent'] = ropts.userAgent[0]; 363 | // If "rotateUA" is true, rotate User-Agent 364 | options.userAgent.push(options.userAgent.shift()); 365 | }else{ 366 | ropts.headers['User-Agent'] = ropts.userAgent; 367 | } 368 | if(self.debug){ 369 | logger.info(ropts.headers['User-Agent']); 370 | } 371 | } 372 | if (ropts.referer) { 373 | ropts.headers.Referer = ropts.referer; 374 | } 375 | if (ropts.proxies && ropts.proxies.length) { 376 | ropts.proxy = ropts.proxies[0]; 377 | } 378 | 379 | this.emit("request",ropts); 380 | 381 | var requestArgs = ['uri','url','qs','method','headers','body','form','json','multipart','followRedirect', 382 | 'followAllRedirects', 'maxRedirects','encoding','pool','timeout','proxy','auth','oauth','strictSSL', 383 | 'jar','aws','gzip','time','tunnel','proxyHeaderWhiteList','proxyHeaderExclusiveList','localAddress','forever']; 384 | 385 | var req = request(_.pick.apply(this,[ropts].concat(requestArgs)), function(error,response) { 386 | if (error) { 387 | return self._onContent(error, options); 388 | } 389 | 390 | response.uri = response.request.href; 391 | self._onContent(error,options,response); 392 | }); 393 | }; 394 | 395 | Crawler.prototype._onContent = function _onContent (error, options, response, fromCache) { 396 | var self = this; 397 | 398 | if (error) { 399 | if (self.debug) { 400 | logger.error('Error '+error+' when fetching '+ 401 | options.uri+(options.retries?' ('+options.retries+' retries left)':'')); 402 | } 403 | if (options.retries) { 404 | self.plannedQueueCallsCount++; 405 | setTimeout(function() { 406 | options.retries--; 407 | self.plannedQueueCallsCount--; 408 | 409 | // If there is a "proxies" option, rotate it so that we don't keep hitting the same one 410 | // if (options.proxies) { 411 | // options.proxies.push(options.proxies.shift()); 412 | // } 413 | self.queue(options); 414 | },options.retryTimeout); 415 | 416 | } else if (options.callback) { 417 | options.callback(error,{options:options}); 418 | } 419 | 420 | return self.emit('pool:release', options); 421 | } 422 | 423 | if (!response.body) { response.body=''; } 424 | 425 | if (self.debug) { 426 | logger.info('Got '+(options.uri||'html')+' ('+response.body.length+' bytes)...'); 427 | } 428 | 429 | if(!fromCache){ 430 | try{ 431 | self._doEncoding(options,response); 432 | }catch(e){ 433 | logger.error(e); 434 | if(options.callback){ 435 | options.callback(e); 436 | } 437 | return self.emit('pool:release',options); 438 | } 439 | } 440 | 441 | // if(useCache(options)){ 442 | // self.cache[self.seen.normalize(options)] = response; 443 | // } 444 | 445 | if (!options.callback) { 446 | return self.emit('pool:release', options); 447 | } 448 | 449 | response.options = options; 450 | 451 | // This could definitely be improved by *also* matching content-type headers 452 | var isHTML = _.isString(response.body) && response.body.match(/^\s*hello