├── .gitignore ├── .npmignore ├── History.md ├── Makefile ├── Readme.md ├── index.js ├── package.json ├── test.js └── test └── phantom.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | support 2 | test 3 | examples 4 | *.sock 5 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 2 | 1.0.1 / 2015-07-09 3 | ================== 4 | 5 | * Merge pull request #9 from skkeeper/master 6 | * Fix a bug where options weren't being loaded into Nightmare 7 | * update test 8 | 9 | 1.0.0 / 2015-04-28 10 | ================== 11 | 12 | * New API 13 | 14 | 0.0.1 / 2010-01-03 15 | ================== 16 | 17 | * Initial release 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | test: 3 | @./node_modules/.bin/mocha \ 4 | --reporter spec \ 5 | --timeout 30000 6 | 7 | .PHONY: test 8 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | 2 | # x-ray-phantom 3 | 4 | phantom driver for [x-ray](https://github.com/lapwinglabs/x-ray). 5 | 6 | ## Installation 7 | 8 | ``` 9 | npm install x-ray-phantom 10 | ``` 11 | 12 | ## Usage 13 | 14 | ```js 15 | var phantom = require('x-ray-phantom'); 16 | var Xray = require('x-ray'); 17 | 18 | var x = Xray() 19 | .driver(phantom()); 20 | 21 | x('http://google.com', 'title')(function(err, str) { 22 | if (err) return done(err); 23 | assert.equal('Google', str); 24 | done(); 25 | }) 26 | ``` 27 | 28 | ## API 29 | 30 | ### phantom([options|fn], [fn]) 31 | 32 | Initialize the phantom driver with `options` being passed to Nightmare and 33 | an optional custom `fn` with the signature `function(nightmare, done)`. 34 | 35 | ## Test 36 | 37 | ``` 38 | npm install 39 | make test 40 | ``` 41 | 42 | ## License 43 | 44 | MIT 45 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module Dependencies 3 | */ 4 | 5 | var debug = require('debug')('x-ray:phantom'); 6 | var normalize = require('normalizeurl'); 7 | var Nightmare = require('nightmare'); 8 | var wrapfn = require('wrap-fn'); 9 | 10 | /** 11 | * Export `driver` 12 | */ 13 | 14 | module.exports = driver; 15 | 16 | /** 17 | * Initialize the `driver` 18 | * with the following `options` 19 | * 20 | * @param {Object} options 21 | * @param {Function} fn 22 | * @return {Function} 23 | * @api public 24 | */ 25 | 26 | function driver(options, fn) { 27 | if ('function' == typeof options) fn = options, options = {}; 28 | options = options || {}; 29 | fn = fn || phantom; 30 | var nightmare = new Nightmare(options); 31 | 32 | 33 | return function phantom_driver(ctx, done) { 34 | debug('going to %s', ctx.url); 35 | 36 | nightmare 37 | .on('error', error) 38 | .on('timeout', function(timeout) { 39 | return done(new Error(timeout)); 40 | }) 41 | .on('resourceReceived', function(resource) { 42 | if (normalize(resource.url) == normalize(ctx.url)) { 43 | debug('got response from %s: %s', resource.url, resource.status); 44 | ctx.status = resource.status; 45 | }; 46 | }) 47 | .on('urlChanged', function(url) { 48 | debug('redirect: %s', url); 49 | ctx.url = url; 50 | }) 51 | 52 | wrapfn(fn, select)(ctx, nightmare); 53 | 54 | function select(err, ret) { 55 | if (err) return done(err); 56 | 57 | nightmare 58 | .evaluate(function() { 59 | return document.documentElement.outerHTML; 60 | }, function(body) { 61 | ctx.body = body; 62 | }) 63 | .run(function(err) { 64 | if (err) return done(err); 65 | debug('%s - %s', ctx.url, ctx.status); 66 | done(null, ctx); 67 | }); 68 | }; 69 | } 70 | } 71 | 72 | /** 73 | * Default phantom driver 74 | * 75 | * @param {HTTP Context} ctx 76 | * @param {Nightmare} nightmare 77 | * @param {Function} fn 78 | */ 79 | 80 | function phantom(ctx, nightmare) { 81 | return nightmare.goto(ctx.url); 82 | } 83 | 84 | /** 85 | * Phantom errors go here 86 | * 87 | * @param {String} msg 88 | */ 89 | 90 | function error(msg) { 91 | debug('client-side javascript error %s', msg); 92 | } 93 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "x-ray-phantom", 3 | "version": "1.0.1", 4 | "description": "phantom driver for x-ray", 5 | "keywords": [], 6 | "author": "Matthew Mueller ", 7 | "repository": { 8 | "type": "git", 9 | "url": "git://github.com/lapwinglabs/x-ray-phantom.git" 10 | }, 11 | "dependencies": { 12 | "debug": "^2.1.1", 13 | "nightmare": "^1.7.0", 14 | "normalizeurl": "^0.1.3", 15 | "wrap-fn": "^0.1.4" 16 | }, 17 | "devDependencies": { 18 | "cheerio": "^0.19.0", 19 | "mocha": "*", 20 | "nightmare-swiftly": "^0.2.4", 21 | "x-ray-crawler": "^2.0.1" 22 | }, 23 | "main": "index" 24 | } -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module Dependencies 3 | */ 4 | 5 | var crawler = require('x-ray-crawler'); 6 | var driver = require('./'); 7 | 8 | /** 9 | * Setup the driver 10 | */ 11 | 12 | function *phantom(ctx, nightmare) { 13 | nightmare.goto(ctx.url) 14 | return nightmare; 15 | } 16 | 17 | /** 18 | * Crawl google 19 | */ 20 | 21 | crawler('http://google.com') 22 | .driver(driver(phantom)) 23 | .on('response', function($, ctx) { 24 | console.log('%s - %s', ctx.status, $('title').text().trim()); 25 | }) 26 | .crawl(function(err) { 27 | if (err) throw err; 28 | }) 29 | -------------------------------------------------------------------------------- /test/phantom.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module Dependencies 3 | */ 4 | 5 | var Crawler = require('x-ray-crawler'); 6 | var cheerio = require('cheerio'); 7 | var join = require('path').join; 8 | var assert = require('assert'); 9 | var phantom = require('../'); 10 | var fs = require('fs'); 11 | 12 | /** 13 | * Tests 14 | */ 15 | 16 | describe('phantom driver', function() { 17 | 18 | it('should have sensible defaults', function(done) { 19 | var crawler = Crawler() 20 | .driver(phantom()) 21 | 22 | crawler('http://google.com', function(err, ctx) { 23 | if (err) return done(err); 24 | var $ = cheerio.load(ctx.body); 25 | var title = $('title').text(); 26 | assert.equal('Google', title); 27 | done(); 28 | }) 29 | }); 30 | 31 | it('should work with client-side pages', function(done) { 32 | var crawler = Crawler() 33 | .driver(phantom()); 34 | 35 | crawler('https://exchange.coinbase.com/trade', function(err, ctx) { 36 | if (err) return done(err); 37 | var $ = cheerio.load(ctx.body); 38 | var price = $('.market-num').text(); 39 | assert.equal(false, isNaN(+price)); 40 | done(); 41 | }) 42 | }) 43 | 44 | it('should support custom functions', function(done) { 45 | var crawler = Crawler() 46 | .driver(phantom(runner)); 47 | 48 | crawler('http://mat.io', function(err, ctx) { 49 | if (err) return done(err); 50 | var $ = cheerio.load(ctx.body); 51 | var title = $('title').text(); 52 | assert.equal('Lapwing Labs', title); 53 | done(); 54 | }) 55 | 56 | function runner(ctx, nightmare) { 57 | return nightmare 58 | .goto(ctx.url) 59 | .click('.Header-logo-item+ .Header-list-item a') 60 | .wait() 61 | } 62 | }) 63 | }) 64 | 65 | /** 66 | * Read 67 | */ 68 | 69 | function get(path) { 70 | return require(join(__dirname, 'fixtures', path)); 71 | } 72 | --------------------------------------------------------------------------------