├── .gitignore ├── .npmignore ├── .travis.yml ├── Readme.md ├── examples └── requests.js ├── index.js ├── lib ├── Requester.js └── RequesterHandler.js ├── package.json └── test ├── echo-server.js └── request-test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.8 -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # DEPRECATED IN FAVOR OF [REQUEST](https://github.com/request/request) / [REQUEST PROMISE](https://github.com/request/request-promise) 2 | 3 | ## node-requester [![Build Status](https://travis-ci.org/icodeforlove/node-requester.png?branch=master)](https://travis-ci.org/icodeforlove/node-requester) 4 | 5 | A simple network request helper that is geared towards crawling. (a few keywords GZIP, XML, JSON, PROXIES) 6 | 7 | ## installation 8 | 9 | $ npm install requester 10 | 11 | ## super simple to use 12 | 13 | ```javascript 14 | var Requester = require('requester'), 15 | requester = new Requester({debug: 1}); 16 | 17 | requester.get(/* URL */, function (body) { 18 | console.log(this.statusCode, body); 19 | }); 20 | 21 | requester.get(/* URL */, /* REQUEST_OBJECT */, function (body) { 22 | console.log(this.statusCode, body); 23 | }); 24 | ``` 25 | 26 | you can even use this simple [request translation tool](http://codepen.io/icodeforlove/full/nKuwa) 27 | 28 | ## initialization 29 | 30 | ```javascript 31 | var Requester = ('requester'); 32 | 33 | var requester = new Requester({ 34 | cookiejar: true, // basic cookie support, currently doesnt care about domain or path rules 35 | cookies: {}, 36 | headers: {}, 37 | timeout: 4000, 38 | retries: 3, 39 | encoding 'utf8', 40 | // didRequestFail: null, (this has its own section) 41 | // signRequest: null, (this has its own section) 42 | // processResponse: null, (this has its own section) 43 | dataType: 'RAW' // JSON or XML, 44 | auth: {username: 'username', password: 'password'}, // basic auth for all requests 45 | proxies: [{ip: '127.0.0.1', port: 1337}, {ip: '127.0.0.2', port: 1337}, {ip: '127.0.0.3', port: 1337}] // rotating proxy array 46 | }); 47 | ``` 48 | 49 | if you initialize the request object with any of the above properties every request will default to those settings, you can over ride them on a per request basis 50 | 51 | ```javascript 52 | var options = { 53 | encoding: 'binary', 54 | proxy: {ip: '127.0.0.1', port: 1337}, 55 | data: {foo: 'bar'}, 56 | cookies: {foo: 'bar'}, 57 | auth: {username: 'username', password: 'password'} // basic auth for request 58 | }; 59 | 60 | requester.get(/* URL */, options, function (body) { 61 | console.log(body) 62 | }); 63 | ``` 64 | ## request objects 65 | 66 | they support the following properties 67 | * {Object} data 68 | * {String} dataType 69 | * {Object} headers 70 | * {Object} cookies 71 | * {Object} proxy 72 | * {Object} auth 73 | * {String} encoding 74 | * {Number} timeout 75 | * {Number} retries 76 | * {Function} didRequestFail 77 | * {Function} signRequest 78 | * {Function} processResponse 79 | * {Boolean} follow 80 | * {Number} followMax 81 | * {Number} debug 82 | 83 | ## debugging 84 | 85 | you can set debug to the following 86 | * 1 - outgoing requests 87 | * 2 - outgoing requests and responses with headers 88 | * 3 - outgoing requests, responses with headers, and response body 89 | 90 | ## methods 91 | * get 92 | * post 93 | * multipart 94 | * put 95 | * del 96 | 97 | ## proxies 98 | 99 | request objects support proxies but you also can add / remove them from the proxy rotation like this 100 | 101 | ```javascript 102 | var requester = new Requester({ 103 | proxies: [{ip: 127.0.0.1, port: 1337}] 104 | }); 105 | 106 | requester.addProxies({ip: 127.0.0.1, port: 1337}, {ip: 127.0.0.2, port: 1337}, {ip: 127.0.0.1, port: 1337, auth: {username: 'foo', password: 'bar'}}); 107 | 108 | requester.removeProxies({ip: 127.0.0.1, port: 1337}); 109 | ``` 110 | 111 | this allows you to do custom checking outside of requester to maintain the proxy list 112 | 113 | ## request response checking 114 | 115 | this is a method that gets ran before the actual response callback gets run to ensure that the content is what you're expecting, for example if the content rotates and you're looking for something special you can do 116 | 117 | ```javascript 118 | requester.get( 119 | / * URL */, 120 | { 121 | didRequestFail: function (data) { 122 | return !data.match(/something/); 123 | }, 124 | retries: 10 125 | }, 126 | function (data) { 127 | console.log(data); 128 | } 129 | ); 130 | ``` 131 | 132 | this would request the url until it matches the string 'something' in the response (up to 10 attempts) 133 | 134 | ## response preprocessing 135 | 136 | lets say the server responds back with invalid JSON 137 | 138 | ```javascript 139 | var json = {foo: 'bar'} 140 | ``` 141 | you can use a processResponse function to clean it up like this 142 | 143 | ```javascript 144 | requester.get( 145 | /* URL */, 146 | { 147 | dataType: 'JSON', 148 | processResponse: function (body) { 149 | return body.replace(/^var json = /, ''); 150 | } 151 | }, 152 | function (body) { 153 | console.log(body); 154 | } 155 | ); 156 | ``` 157 | 158 | this is really useful if you want to not repeat response cleanup code 159 | 160 | ## request signatures 161 | 162 | you can create a custom request signature function like this 163 | 164 | ```javascript 165 | var qs = require('querystring'); 166 | 167 | var requester = new Requester({ 168 | signRequest: function (data) { 169 | // do something with the data 170 | return qs.stringify(data); 171 | } 172 | }); 173 | ``` 174 | 175 | ## posting 176 | 177 | ```javascript 178 | requester.post(/* URL */, {data: {foo: 'bar', bar: 'foo'}}, function (body) { 179 | console.log(body) 180 | }); 181 | ``` 182 | 183 | ## multipart 184 | 185 | the multipart request works a little different, in the data object you can prefix a values key with '@' like this 186 | 187 | ```javascript 188 | requester.multipart(/* URL */, {data: {'@file': /* FILEPATH */, '@file2': /* FILEPATH */, bar: 'foo'}}, function (body) { 189 | console.log(body) 190 | }); 191 | ``` 192 | 193 | this will create a multipart request and upload files 194 | -------------------------------------------------------------------------------- /examples/requests.js: -------------------------------------------------------------------------------- 1 | /*jshint node:true, strict:false*/ 2 | var Requester = require('../lib/requester'); 3 | 4 | var requester = new Requester(); 5 | 6 | requester.get('http://www.google.com', function (data) { 7 | //console.log(data); 8 | }); 9 | 10 | requester.post('http://www.google.com', function (data) { 11 | //console.log(data); 12 | }); 13 | 14 | requester.get('http://www.google.com', {data: {foo: 'bar'}, headers: {'user-agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)'}}, function (data) { 15 | //console.log(data); 16 | }); 17 | 18 | requester.get('http://www.google.com', {data: {foo: 'bar'}, headers: {'user-agent': ''}}, function (data) { 19 | //console.log(data); 20 | }); 21 | 22 | requester.get('http://tumblruptime.apigee.com/json', {dataType: 'JSON'}, function (data) { 23 | //console.log(data); 24 | }); -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /*jshint node:true*/ 2 | 'use strict'; 3 | 4 | module.exports = exports = require('./lib/RequesterHandler'); -------------------------------------------------------------------------------- /lib/Requester.js: -------------------------------------------------------------------------------- 1 | var http = require('http'), 2 | https = require('https'), 3 | url = require('url'), 4 | qs = require('qs'), 5 | FormData = require('form-data'), 6 | fs = require('fs'), 7 | xml2js = require('xml2js'), 8 | async = require('async'), 9 | zlib = require('zlib'), 10 | _ = require('underscore'), 11 | colors = require('colors'), 12 | Promise = require('bluebird'), 13 | PromiseObject = require('promise-object')(Promise), 14 | EventsMixin = require('promise-object/mixins/events'); 15 | 16 | var Requester = PromiseObject.create(EventsMixin, { 17 | initialize: function ($url, $options, callback) { 18 | this._callback = callback; 19 | this._retry = 0; 20 | this._followCount = 0; 21 | this._url = $url; 22 | this._handler = $options.handler; 23 | 24 | this._multipart = $options.multipart; 25 | this._method = $options.method; 26 | this._cookies = $options.cookies; 27 | this._headers = $options.headers; 28 | this._timeout = $options.timeout; 29 | this._retries = $options.retries; 30 | this._encoding = $options.encoding; 31 | this._didRequestFail = $options.didRequestFail; 32 | this._signRequest = $options.signRequest; 33 | this._processResponse = $options.processResponse; 34 | this._dataType = $options.dataType; 35 | 36 | this._cookiejar = $options.cookiejar; 37 | this._follow = $options.follow; 38 | this._followMax = $options.followMax; 39 | this._proxy = $options.proxy; 40 | this._debug = $options.debug; 41 | this._data = $options.data; 42 | this._protocol = $url.protocol === 'https:' && !this._proxy ? https : http; 43 | this._port = $url.protocol === 'https:' && this._proxy ? 443 : 80; 44 | 45 | this._auth = this._proxy && this._proxy.auth ? this._proxy.auth : $options.auth; 46 | 47 | if (_.isObject(this._data) && this._headers['content-type'] === 'application/json') { 48 | this._queryString = JSON.stringify(this._data); 49 | } else { 50 | this._queryString = this._signRequest && this._method !== 'GET' ? this._signRequest(this._data) : this._data ? qs.stringify(this._data) : ''; 51 | } 52 | 53 | this._attempt = this._attempt.bind(this); 54 | this._prepareHttpOptions(); 55 | }, 56 | 57 | _prepareHttpOptions: function () { 58 | var self = this; 59 | 60 | if (!this._headers.cookie && !this._headers.Cookie && this._cookies) this._headers.cookie = this._stringifyCookies(this._cookiejar || this._cookies); 61 | if (this._auth) this._headers[this._proxy ? 'Proxy-Authorization' : 'Authorization'] = 'Basic ' + new Buffer(this._auth.username + ':' + this._auth.password).toString('base64'); 62 | 63 | if (this._multipart) { 64 | this._prepareFormData(function () { 65 | self._attempt(); 66 | }); 67 | } else { 68 | if (this._method === 'GET' && this._data) { 69 | this._url.search = this._url.search ? this._url.search + '&' + this._queryString : '?' + this._queryString; 70 | } else if (this._method !== 'GET' && this._data) { 71 | if (!this._headers['content-type']) this._headers['content-type'] = 'application/x-www-form-urlencoded'; 72 | this._headers['content-length'] = this._queryString ? this._queryString.length : 0; 73 | } 74 | 75 | this._attempt(); 76 | } 77 | }, 78 | 79 | _prepareFormData: function (callback) { 80 | var files = [], 81 | self = this; 82 | 83 | this._formData = new FormData(); 84 | 85 | for (var key in this._data) { 86 | if (key.substr(0,1) === '@') { 87 | this._formData.append(key.substr(1), fs.createReadStream(this._data[key])); 88 | } else { 89 | this._formData.append(key, this._data[key]); 90 | } 91 | } 92 | 93 | this._formData.getLength(function (error, length) { 94 | self._headers = _.extend(self._headers, self._formData.getHeaders({'Content-Length': length})); 95 | callback(); 96 | }); 97 | }, 98 | 99 | _attempt: function () { 100 | this._retry++; 101 | this._prepareRequestObject(); 102 | 103 | if (this._formData) { 104 | this._formData.pipe(this._requestObject); 105 | } else { 106 | if (this._method !== 'GET') { 107 | if (this._debug > 1 && this._data) console.log(JSON.stringify(this._data, null, '\t').grey); 108 | this._requestObject.write(this._queryString); 109 | } 110 | this._requestObject.end(); 111 | } 112 | }, 113 | 114 | _toCapitalizedString: function (string) { 115 | return string.replace(/\b(\w)/g, function (firstLetter) { return firstLetter.toUpperCase()}); 116 | }, 117 | 118 | _prepareRequestObject: function () { 119 | var self = this; 120 | 121 | // strip cookie if we dont have one 122 | if (!this._headers.cookie) delete this._headers.cookie; 123 | 124 | var headers = {}; 125 | for (var i in this._headers) headers[this._toCapitalizedString(i)] = this._headers[i]; 126 | 127 | var options = { 128 | host: this._proxy ? this._proxy.ip : this._url.hostname, 129 | port: this._proxy ? this._proxy.port : this._url.port, 130 | path: this._proxy ? url.format(this._url) : this._url.pathname + (this._url.search || ''), 131 | method: this._method, 132 | headers: headers 133 | }; 134 | 135 | if (this._debug > 0) console.log(this._method.grey + ' ' + url.format(this._url)); 136 | if (this._debug > 1) console.log(JSON.stringify(headers, null, '\t').grey); 137 | 138 | this._requestObject = this._protocol.request( 139 | options, 140 | function (response) { 141 | if (!self._encoding && response.headers['content-encoding'] === 'gzip') { 142 | self._gzipResponse(response); 143 | } else { 144 | self._normalResponse(response); 145 | } 146 | } 147 | ); 148 | 149 | this._requestObject.setTimeout(this._timeout, function () { 150 | if (self._debug > 0) console.log('FAILED '.red + self._method.grey + ' ' + url.format(self._url) + (' (' + 'timeout' + ')').red); 151 | self._requestFailed(null); 152 | }); 153 | 154 | this._requestObject.on('error', function(e) { 155 | if (self._debug > 0) console.log('FAILED '.red + self._method.grey + ' ' + url.format(self._url) + (' (' + e.message + ')').red); 156 | self._requestFailed(null); 157 | }); 158 | }, 159 | 160 | _requestFailed: function (response) { 161 | if (this._retries > this._retry) { 162 | this._attempt(); 163 | } else { 164 | this._callback.call(response, null); 165 | } 166 | }, 167 | 168 | _normalResponse: function (response) { 169 | var self = this, 170 | data = ''; 171 | 172 | response.setEncoding(this._encoding); 173 | 174 | response.on('data', function (chunk) { 175 | data += chunk; 176 | }); 177 | 178 | response.on('end', function () { 179 | response.parsedUrl = self._url; 180 | response.proxy = self._proxy; 181 | 182 | response.body = data; 183 | self._checkResponse(response); 184 | }); 185 | }, 186 | 187 | _gzipResponse: function (response) { 188 | var self = this, 189 | data = '', 190 | gunzip = zlib.createGunzip(); 191 | 192 | response.setEncoding('binary'); 193 | 194 | gunzip = zlib.createGunzip(); 195 | 196 | gunzip.on('data', function(chunk) { 197 | data += chunk; 198 | }); 199 | 200 | gunzip.on('end', function () { 201 | response.body = data; 202 | self._checkResponse(response); 203 | }); 204 | 205 | response.on('data', function (chunk) { 206 | gunzip.write(new Buffer(chunk, 'binary')); 207 | }); 208 | 209 | response.on('end', function () { 210 | response.parsedUrl = self._url; 211 | response.proxy = self._proxy; 212 | 213 | gunzip.end(); 214 | }); 215 | }, 216 | 217 | _checkResponse: function (response) { 218 | var self = this; 219 | 220 | if (this._debug > 1) { 221 | console.log(this._method.grey + ' RESPONSE '.grey + String(response.statusCode).red + ' ' + url.format(this._url) + ' - '.grey + String((response.connection.bytesRead / 1024).toFixed(2) + 'kb').grey); 222 | console.log(JSON.stringify(response.headers, null, '\t').grey); 223 | if (this._debug > 2) console.log(response.body.grey); 224 | } 225 | 226 | response.cookies = {}; 227 | if (response.headers['set-cookie']) { 228 | response.headers['set-cookie'].forEach(function (value) { 229 | response.cookies = _.extend(response.cookies, this._parseCookies(value)); 230 | }, this); 231 | } 232 | if (this._cookiejar) this._cookiejar = _.extend(this._cookiejar, response.cookies); 233 | 234 | if (this._follow && this._followCount <= this._followMax && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) { 235 | var location = url.parse(response.headers.location); 236 | if (!location.hostname) location.hostname = this._url.hostname; 237 | if (!location.port) location.port = this._url.port; 238 | if (!location.protocol) location.protocol = this._url.protocol; 239 | this._handler.get(url.format(location), this._callback); 240 | return; 241 | } 242 | 243 | if (this._processResponse) response.body = this._processResponse.call(response, response.body); 244 | 245 | if (this._dataType === 'JSON') { 246 | var json; 247 | try { 248 | json = JSON.parse(response.body); 249 | } catch (e) {} 250 | 251 | if (json) { 252 | if (this._didRequestFail && !this._didRequestFail(json) || !this._didRequestFail) { 253 | this.dispatchEvent('response-recieved', {data: json}); 254 | this._callback.call(response, json); 255 | } else { 256 | this._requestFailed(response); 257 | } 258 | } else { 259 | this._requestFailed(response); 260 | } 261 | } else if (this._dataType === 'XML') { 262 | var parser = new xml2js.Parser(); 263 | parser.parseString(response.body, function (error, xml) { 264 | if (error) { 265 | self._requestFailed(response); 266 | } else { 267 | self.dispatchEvent('response-recieved', {data: xml}); 268 | self._callback.call(response, xml); 269 | } 270 | }); 271 | } else { 272 | if (this._didRequestFail && !this._didRequestFail(response.body) || !this._didRequestFail) { 273 | this.dispatchEvent('response-recieved', {data: response.body}); 274 | this._callback.call(response, response.body); 275 | } else { 276 | this._requestFailed(response); 277 | } 278 | } 279 | }, 280 | 281 | _stringifyCookies: function (cookies) { 282 | var string = ''; 283 | for (var cookie in cookies) string += cookie + '=' + cookies[cookie] + '; '; 284 | return string.slice(0,-2); 285 | }, 286 | 287 | _parseCookies: function (string) { 288 | var cookies = {}; 289 | string.split('; ').forEach(function (value) { 290 | var parts = value.split('='); 291 | if (!parts[0].match(/domain|path|expires|secure|httponly/i)) cookies[parts[0]] = parts[1] || ''; 292 | }); 293 | 294 | return cookies; 295 | } 296 | }); 297 | 298 | module.exports = Requester; -------------------------------------------------------------------------------- /lib/RequesterHandler.js: -------------------------------------------------------------------------------- 1 | var http = require('http'), 2 | https = require('https'), 3 | url = require('url'), 4 | qs = require('qs'), 5 | FormData = require('form-data'), 6 | fs = require('fs'), 7 | xml2js = require('xml2js'), 8 | async = require('async'), 9 | zlib = require('zlib'), 10 | _ = require('underscore'), 11 | colors = require('colors'), 12 | Promise = require('bluebird'), 13 | PromiseObject = require('promise-object')(Promise), 14 | EventsMixin = require('promise-object/mixins/events'), 15 | Requester = require('./Requester'); 16 | 17 | var RequesterHandler = PromiseObject.create(EventsMixin, { 18 | initialize: function ($config) { 19 | this._debug = $config.debug; 20 | this._cookies = $config.cookies || {}; 21 | this._headers = $config.headers || {}; 22 | this._timeout = $config.timeout || 30000; 23 | this._proxies = $config.proxies || []; 24 | this._retries = $config.retries || 0; 25 | this._encoding = $config.encoding || 'utf8'; 26 | this._cookiejar = $config.cookiejar ? {} : false; 27 | this._follow = _.isUndefined($config.follow) ? true : $config.follow; 28 | this._followMax = _.isUndefined($config.followMax) ? 5 : $config.followMax; 29 | this._requestID = 0; 30 | this._didRequestFail = $config.didRequestFail; 31 | this._signRequest = $config.signRequest; 32 | this._processResponse = $config.processResponse; 33 | this._dataType = $config.dataType; 34 | this._auth = $config.auth; 35 | 36 | if (this._cookiejar) this._cookiejar = _.extend(this._cookiejar, this._cookies); 37 | }, 38 | 39 | post: function ($url, $options, callback) { 40 | $options = $options || {}; 41 | 42 | if (arguments.length === 2 && typeof arguments[1] === 'function') { 43 | callback = arguments[1]; 44 | } 45 | 46 | if (!$options.method) $options.method = 'POST'; 47 | return this._makeRequest($url, $options, callback); 48 | }, 49 | 50 | put: function ($url, $options, callback) { 51 | $options = $options || {}; 52 | 53 | if (arguments.length === 2 && typeof arguments[1] === 'function') { 54 | callback = arguments[1]; 55 | } 56 | 57 | if (!$options.method) $options.method = 'PUT'; 58 | return this._makeRequest($url, $options, callback); 59 | }, 60 | 61 | get: function ($url, $options, callback) { 62 | $options = $options || {}; 63 | 64 | if (arguments.length === 2 && typeof arguments[1] === 'function') { 65 | callback = arguments[1]; 66 | } 67 | 68 | if (!$options.method) $options.method = 'GET'; 69 | return this._makeRequest($url, $options, callback); 70 | }, 71 | 72 | del: function ($url, $options, callback) { 73 | $options = $options || {}; 74 | 75 | if (arguments.length === 2 && typeof arguments[1] === 'function') { 76 | callback = arguments[1]; 77 | } 78 | 79 | if (!$options.method) $options.method = 'DELETE'; 80 | return this._makeRequest($url, $options, callback); 81 | }, 82 | 83 | addProxies: function () { 84 | for (var proxy = 0; proxy < arguments.length; proxy++) { 85 | if (this._proxyCheck(arguments[proxy]) === -1) { 86 | this._proxies.push(arguments[proxy]); 87 | } 88 | } 89 | }, 90 | 91 | multipart: function ($url, $options, callback) { 92 | $options.multipart = true; 93 | return this.post($url, $options, callback); 94 | }, 95 | 96 | removeProxies: function () { 97 | for (var proxy = 0; proxy < arguments.length; proxy++) { 98 | var index = this._proxyCheck(arguments[proxy]); 99 | if (index !== -1) this._proxies.splice(index, 1); 100 | } 101 | }, 102 | 103 | _proxyCheck: function (query) { 104 | var index = -1; 105 | 106 | this._proxies.forEach(function (proxy, key) { 107 | if (_.isEqual(query, proxy)) index = key; 108 | }); 109 | 110 | return index; 111 | }, 112 | 113 | _makeRequest: function ($deferred, $url, $options, callback) { 114 | // prepare options 115 | $options.debug = $options.debug || this._debug; 116 | $options.cookies = $options.cookies && this._cookies ? _.extend({}, this._cookies, $options.cookies) : $options.cookies || _.extend({}, this._cookies); 117 | $options.headers = $options.headers && this._headers ? _.extend({}, this._headers, $options.headers) : $options.headers || _.extend({}, this._headers); 118 | $options.timeout = $options.timeout || this._timeout; 119 | $options.retries = $options.retries || this._retries; 120 | $options.encoding = $options.encoding || this._encoding; 121 | $options.didRequestFail = $options.didRequestFail || this._didRequestFail; 122 | $options.signRequest = $options.signRequest || this._signRequest; 123 | $options.processResponse = $options.processResponse || this._processResponse; 124 | $options.dataType = $options.dataType || this._dataType; 125 | 126 | if (this._cookiejar) $options.cookiejar = _.extend(this._cookiejar, $options.cookies); 127 | $options.follow = _.isUndefined($options.follow) ? this._follow : $options.follow; 128 | $options.followMax = $options.followMax || this._followMax; 129 | $options.auth = $options.auth || this._auth; 130 | $options.proxy = this._proxies.length && _.isUndefined($options.proxy) ? this._proxies[this._requestID % this._proxies.length] : $options.proxy; 131 | $options.id = this._requestID; 132 | $options.handler = this; 133 | 134 | this._requestID++; 135 | 136 | return new Requester(url.parse($url), $options, function (body) { 137 | if (callback) { 138 | callback.call(this, body); 139 | } else { 140 | if (body === null) { 141 | $deferred.reject(); 142 | } else { 143 | $deferred.resolve({body: body, request: this}); 144 | } 145 | } 146 | }); 147 | } 148 | }); 149 | 150 | module.exports = RequesterHandler; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "requester", 3 | "version": "0.1.22", 4 | "description": "swiss army knife for requests", 5 | "main": "index.js", 6 | "directories": { 7 | "example": "examples", 8 | "test": "tests" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git://github.com/icodeforlove/node-requester.git" 13 | }, 14 | "keywords": [ 15 | "http", 16 | "https", 17 | "xml", 18 | "json", 19 | "proxies" 20 | ], 21 | "author": "Chad Scira", 22 | "license": "MIT", 23 | "devDependencies": { 24 | "async": "~0.1.22", 25 | "vows": "0.7.0" 26 | }, 27 | "dependencies": { 28 | "async": "~0.1.22", 29 | "bluebird": "~2.3.5", 30 | "colors": "~0.6.0-1", 31 | "form-data": "~0.0.6", 32 | "promise-object": "~0.1.4", 33 | "qs": "~0.5.3", 34 | "underscore": "~1.4.0", 35 | "xml2js": "~0.2.0" 36 | }, 37 | "scripts": { 38 | "pretest": "node ./test/echo-server.js &", 39 | "test": "vows --spec" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/echo-server.js: -------------------------------------------------------------------------------- 1 | /*jshint node:true, strict:false*/ 2 | var colors = require('colors'), 3 | http = require('http'), 4 | port = 1338, 5 | host = '127.0.0.1'; 6 | 7 | http 8 | .createServer(function (request, response) { 9 | var data = ''; 10 | 11 | request.setEncoding('utf8'); 12 | request.on('data', function(chunk) { 13 | data += chunk; 14 | }); 15 | 16 | request.on('end', function() { 17 | response.end(JSON.stringify({ 18 | headers: request.headers, 19 | url: request.url, 20 | method: request.method, 21 | body: data 22 | })); 23 | }); 24 | }) 25 | .listen(port, host, function () { 26 | //process.send({port: port, host: host}); 27 | }); -------------------------------------------------------------------------------- /test/request-test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'), 3 | Requester = require('../index'), 4 | requester = new Requester({ 5 | headers: {'connection': 'keep-alive'}, 6 | debug: 0 7 | }), 8 | basePath = 'http://127.0.0.1:1338'; 9 | 10 | var request = function () { 11 | var params = Array.prototype.slice.call(arguments), 12 | method = params.shift(); 13 | 14 | return function () { 15 | var self = this, 16 | callback = function (details) { 17 | self.callback(null, {request: this, details: JSON.parse(details)}); 18 | }; 19 | 20 | params.push(callback); 21 | 22 | requester[method].apply(requester, params); 23 | }; 24 | }; 25 | 26 | exports.getRequests2 = vows.describe('GET Requests').addBatch({ 27 | 'Standard Request': { 28 | topic: function () { 29 | var self = this; 30 | 31 | requester.get(basePath + '?something=something').then(function (response) { 32 | self.callback(null, response.body); 33 | }, function (error) { 34 | self.callback(error); 35 | }); 36 | }, 37 | 38 | 'is response correct': function (topic) { 39 | assert.equal(topic, '{"headers":{"connection":"keep-alive","host":"127.0.0.1:1338"},"url":"/?something=something","method":"GET","body":""}'); 40 | } 41 | } 42 | }); 43 | 44 | exports.getRequests = vows.describe('GET Requests').addBatch({ 45 | 'Standard Request': { 46 | topic: request('get', basePath + '?something=something'), 47 | 48 | 'is response correct': function (topic) { 49 | assert.equal(topic.request.statusCode, 200); 50 | assert.equal(JSON.stringify(topic.details), '{"headers":{"connection":"keep-alive","host":"127.0.0.1:1338"},"url":"/?something=something","method":"GET","body":""}'); 51 | } 52 | }, 53 | 54 | 'Mixed Request': { 55 | topic: request('get', basePath + '?something=something', {data: {somethingElse: 'somethingElse'}}), 56 | 57 | 'is response correct': function (topic) { 58 | assert.equal(topic.request.statusCode, 200); 59 | assert.equal(JSON.stringify(topic.details), '{"headers":{"connection":"keep-alive","host":"127.0.0.1:1338"},"url":"/?something=something&somethingElse=somethingElse","method":"GET","body":""}'); 60 | } 61 | } 62 | }); 63 | 64 | exports.postRequests = vows.describe('POST Requests').addBatch({ 65 | 'Standard Request': { 66 | topic: request('post', basePath, {data: {something: 'something'}}), 67 | 68 | 'is response correct': function (topic) { 69 | assert.equal(topic.request.statusCode, 200); 70 | assert.equal(JSON.stringify(topic.details), '{"headers":{"connection":"keep-alive","content-type":"application/x-www-form-urlencoded","content-length":"19","host":"127.0.0.1:1338"},"url":"/","method":"POST","body":"something=something"}'); 71 | } 72 | }, 73 | 'Mixed Request': { 74 | topic: request('post', basePath + '?something=something', {data: {something: 'something'}}), 75 | 76 | 'is response correct': function (topic) { 77 | assert.equal(topic.request.statusCode, 200); 78 | assert.equal(JSON.stringify(topic.details), '{"headers":{"connection":"keep-alive","content-type":"application/x-www-form-urlencoded","content-length":"19","host":"127.0.0.1:1338"},"url":"/?something=something","method":"POST","body":"something=something"}'); 79 | } 80 | }, 81 | 'Headers': { 82 | topic: request('post', basePath, {headers: {'user-agent': 'something'}}), 83 | 84 | 'is response correct': function (topic) { 85 | assert.equal(topic.request.statusCode, 200); 86 | assert.equal(topic.details.headers['user-agent'], 'something'); 87 | } 88 | }, 89 | 90 | 'Custom Content': { 91 | topic: request('post', basePath, {data: {something: 'something'}, headers: {'content-type': 'something'}}), 92 | 93 | 'is response correct': function (topic) { 94 | assert.equal(topic.request.statusCode, 200); 95 | assert.equal(topic.details.body, 'something=something'); 96 | assert.equal(topic.details.headers['content-type'], 'something'); 97 | } 98 | }, 99 | 100 | 'JSON Content': { 101 | topic: request('post', basePath, {data: {something: 'something'}, headers: {'content-type': 'application/json'}}), 102 | 103 | 'is response correct': function (topic) { 104 | assert.equal(topic.request.statusCode, 200); 105 | assert.equal(topic.details.body, '{"something":"something"}'); 106 | assert.equal(topic.details.headers['content-type'], 'application/json'); 107 | } 108 | } 109 | }); 110 | 111 | exports.putRequests = vows.describe('PUT Requests').addBatch({ 112 | 'Standard Request': { 113 | topic: request('put', basePath, {data: {something: 'something'}}), 114 | 115 | 'is response correct': function (topic) { 116 | assert.equal(topic.request.statusCode, 200); 117 | assert.equal(topic.details.method, 'PUT'); 118 | assert.equal(topic.details.body, 'something=something'); 119 | } 120 | } 121 | }); 122 | 123 | exports.delRequests = vows.describe('DELETE Requests').addBatch({ 124 | 'Standard Request': { 125 | topic: request('del', basePath + '/post/123'), 126 | 127 | 'is response correct': function (topic) { 128 | assert.equal(topic.request.statusCode, 200); 129 | assert.equal(topic.details.method, 'DELETE'); 130 | assert.equal(topic.details.url, '/post/123'); 131 | } 132 | } 133 | }); 134 | 135 | exports.multipartRequests = vows.describe('Multipart Requests').addBatch({ 136 | 'Standard Request': { 137 | topic: request('multipart', basePath, {data: {something: 'something'}}), 138 | 139 | 'is response correct': function (topic) { 140 | assert.equal(topic.request.statusCode, 200); 141 | assert.match(topic.details.body, /name="something"/); 142 | } 143 | } 144 | }); --------------------------------------------------------------------------------