├── .gitignore ├── browser.js ├── package.json ├── readme.txt └── test ├── cookies.js └── get.js /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | node_modules/* 3 | -------------------------------------------------------------------------------- /browser.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var parse = require('url').parse; 3 | var EventEmitter = require('events').EventEmitter; 4 | var BufferList = require('bufferlist').BufferList; 5 | var Hash = require('hashish'); 6 | var unescape = require('querystring').unescape; 7 | var escape = require('querystring').escape; 8 | 9 | module.exports = Browser; 10 | function Browser () { 11 | if (!(this instanceof Browser)) return new Browser(); 12 | var self = this; 13 | var actionQueue = []; 14 | var emitter = new EventEmitter; 15 | var storage = {}; 16 | var cookies = {}; 17 | 18 | function newAction(f, args) { 19 | actionQueue.push({ f : f, args : args }); 20 | } 21 | 22 | var looxup = { 23 | get : get, 24 | post : post, 25 | tap : tap 26 | }; 27 | 'get post tap'.split(' ').forEach(function (x) { 28 | self[x] = function () { 29 | newAction(looxup[x], arguments); 30 | return self; 31 | } 32 | }); 33 | 34 | function updateCookies (cookieStr) { 35 | // I'm looking for an aspiring hacker who wants to write a real cookie parser. 36 | // This parser just stores the value, ignores domain, expire, etc. 37 | var components = cookieStr.split(/;\s*/); 38 | var moo = components[0].split('='); 39 | cookies[unescape(moo[0])] = unescape(moo[1]); 40 | } 41 | 42 | Client.prototype = new EventEmitter; 43 | function Client (url, opts) { 44 | if (!(this instanceof Client)) return new Client(url, opts); 45 | var self = this; 46 | 47 | opts = opts || {}; 48 | opts.method = opts.method || 'GET'; 49 | opts.headers = opts.headers || {}; 50 | 51 | if (opts.cookies && Hash(opts.cookies).size) { 52 | var toJoin = []; 53 | Hash(opts.cookies).forEach(function (k,v) { 54 | shit.push(escape(k) + '=' + escape(v)); 55 | }); 56 | opts.headers.cookie = toJoin.join('; '); 57 | } 58 | 59 | var parsed = parse(url); 60 | var client = http.createClient(parsed.port || 80, parsed.hostname); 61 | var path = (parsed.pathname || '/') + (parsed.search || ''); 62 | var headers = Hash({ 63 | host : parsed.hostname 64 | }).merge(opts.headers).items; 65 | var request = client.request(opts.method, path, headers); 66 | request.end(opts.data || ''); 67 | var data = new BufferList; 68 | request.on('response', function (response) { 69 | if (response.headers['set-cookie'] instanceof Array) { 70 | response.headers['set-cookie'].forEach(function (cookie) { 71 | updateCookies(cookie); 72 | }); 73 | } 74 | else { 75 | updateCookies(response.headers['set-cookie']); 76 | } 77 | console.log(cookies); 78 | response.on('data', function (chunk) { 79 | data.push(chunk); 80 | }); 81 | response.on('end', function () { 82 | self.emit('done', response, data.toString()); 83 | }); 84 | }); 85 | } 86 | 87 | function get (opts) { 88 | if (typeof opts === 'string') 89 | var opts = { url : opts } 90 | var client = new Client(opts.url, { cookies : cookies }); 91 | client.on('done', function (response, data) { 92 | emitter.emit('done', response, data); 93 | }); 94 | } 95 | 96 | function post (opts, data) { 97 | if (typeof opts === 'string') 98 | var opts = { url : opts } 99 | var formData = data; 100 | var client = new Client(opts.url, { method : 'POST', data : formData, cookies : cookies }); 101 | client.on('done', function (response, data) { 102 | emitter.emit('done', response, data); 103 | }); 104 | } 105 | 106 | function tap (f, storage, response, data) { 107 | f(storage, response, data) 108 | } 109 | 110 | self.end = function () { 111 | function next (response, data) { 112 | var action = actionQueue.shift(); 113 | if (!action) return; 114 | var args = [].slice.apply(action.args); 115 | args.push(storage); 116 | args.push(response); 117 | args.push(data); 118 | action.f.apply(self, args); 119 | } 120 | emitter.on('done', function (response, data) { 121 | next(response, data); 122 | }); 123 | emitter.emit('done'); 124 | } 125 | } 126 | 127 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "browserx", 3 | "version": "1.0.1", 4 | "description": "Web browser for node.js", 5 | "main" : "./browser.js", 6 | "keywords": [ 7 | "browser", 8 | "http", 9 | "html", 10 | "web" 11 | ], 12 | "author": { 13 | "name": "Peteris Krumins", 14 | "email": "peteris.krumins@gmail.com", 15 | "web": "http://www.catonmat.net", 16 | "twitter": "pkrumins" 17 | }, 18 | "license": { 19 | "type": "MIT" 20 | }, 21 | "repository": { 22 | "type": "git", 23 | "url": "http://github.com/pkrumins/node-browser.git" 24 | }, 25 | "engines": { 26 | "node": ">=0.2.0" 27 | }, 28 | "dependencies": { 29 | "bufferlist": "*", 30 | "hashish": "*" 31 | } 32 | } 33 | 34 | -------------------------------------------------------------------------------- /readme.txt: -------------------------------------------------------------------------------- 1 | A node.js module for browsing a web in an awesome way! I wrote it for my 2 | social-submitter software (http://github.com/pkrumins/social-submitter). 3 | 4 | It was written by Peteris Krumins (peter@catonmat.net). 5 | His blog is at http://www.catonmat.net -- good coders code, great reuse. 6 | 7 | ------------------------------------------------------------------------------ 8 | 9 | Check this out: 10 | 11 | var Browser = require('browser'); 12 | 13 | var browser = new Browser; 14 | browser 15 | .get('http://www.reddit.com') 16 | .post( 17 | 'http://www.reddit.com/api/login/' + data.username, 18 | { 19 | op : 'login-main', 20 | user : data.username, 21 | passwd : data.password, 22 | id : '#login_login-main', 23 | renderstyle : 'html' 24 | } 25 | ) 26 | .get('http://www.reddit.com/r/' + data.subreddit) 27 | .get('http://www.reddit.com/r/' + data.subreddit + '/submit') 28 | .post( 29 | 'http://www.reddit.com/api/submit', 30 | { 31 | uh : 'todo', 32 | kind : 'link', 33 | sr : data.subreddit, 34 | url : data.url, 35 | title : data.title, 36 | id : '#newlink', 37 | r : data.subreddit, 38 | renderstyle : 'html' 39 | } 40 | ) 41 | .end(); 42 | 43 | This logs you into Reddit and submits a story to data.subreddit subreddit. 44 | 45 | Also check out social-submitter software that submits to hacker news, twitter, 46 | facebook, and other sites: 47 | 48 | http://github.com/pkrumins/social-submitter 49 | 50 | 51 | ------------------------------------------------------------------------------ 52 | 53 | Happy browsing! 54 | 55 | 56 | Sincerely, 57 | Peteris Krumins 58 | http://www.catonmat.net 59 | 60 | -------------------------------------------------------------------------------- /test/cookies.js: -------------------------------------------------------------------------------- 1 | var Browser = require('browser'); 2 | var http = require('http'); 3 | 4 | var port = parseInt(5000 + Math.random()*30000); 5 | http.createServer(function (request, response) { 6 | response.writeHead(200, { 7 | 'Set-Cookie' : [ 8 | 'auth_token=; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT', 9 | 'lang=en; path=/', 10 | 'auth_token=moo; domain=.twitter.com; path=/', 11 | '_twitter_sess=foo; domain=.twitter.com; path=/' 12 | ] 13 | }); 14 | response.end(); 15 | }).listen(port); 16 | 17 | module.exports.cookie = function (assert) { 18 | setTimeout(function () { 19 | var browser = new Browser; 20 | var executed = 0; 21 | browser 22 | .get('http://localhost:' + port) 23 | .tap(function (storage, response, data) { 24 | executed++; 25 | console.dir(response); 26 | }) 27 | .end(); 28 | 29 | setTimeout(function () { 30 | assert.equal(executed, 1) 31 | }, 2000); 32 | }, 1000); 33 | } 34 | 35 | -------------------------------------------------------------------------------- /test/get.js: -------------------------------------------------------------------------------- 1 | var Browser = require('browser'); 2 | 3 | module.exports.get = function (assert) { 4 | var browser = new Browser; 5 | var executed = 0; 6 | browser 7 | .get('http://www.reddit.com') 8 | .tap(function (storage, response, data) { 9 | executed++; 10 | assert.deepEqual(storage, {}); 11 | assert.ok(/reddit/.test(response.headers['set-cookie'])); 12 | }) 13 | .end(); 14 | 15 | setTimeout(function () { 16 | assert.equal(executed, 1) 17 | }, 2000); 18 | } 19 | 20 | module.exports['get with opts'] = function (assert) { 21 | var browser = new Browser; 22 | var executed = 0; 23 | browser 24 | .get({ url : 'http://www.reddit.com' }) 25 | .tap(function (storage, response, data) { 26 | executed++; 27 | assert.deepEqual(storage, {}); 28 | assert.ok(/reddit/.test(response.headers['set-cookie'])); 29 | }) 30 | .end(); 31 | 32 | setTimeout(function () { 33 | assert.equal(executed, 1) 34 | }, 2000); 35 | } 36 | 37 | --------------------------------------------------------------------------------