├── .gitignore ├── LICENSE ├── README.md ├── database.js ├── index.js ├── package.json ├── rotating_proxy.js ├── rotating_proxy_manager.js └── test └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Node template 3 | # Logs 4 | logs 5 | *.log 6 | npm-debug.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # node-waf configuration 27 | .lock-wscript 28 | 29 | # Compiled binary addons (http://nodejs.org/api/addons.html) 30 | build/Release 31 | 32 | # Dependency directories 33 | node_modules 34 | jspm_packages 35 | 36 | # Optional npm cache directory 37 | .npm 38 | 39 | # Optional eslint cache 40 | .eslintcache 41 | 42 | # Optional REPL history 43 | .node_repl_history 44 | 45 | # Output of 'npm pack' 46 | *.tgz 47 | 48 | # Yarn Integrity file 49 | .yarn-integrity 50 | 51 | .idea -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Joel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-rotating-proxy-manager 2 | Rotating Proxy Manager module for Node.js 3 | 4 | This module was written in an ES6 environment and uses SQLite to store proxy usage info so you can use this across multiple scripts with the same database file. 5 | 6 | const RotatingProxyManager = require('rotating-proxy-manager'); 7 | const RotatingProxy = RotatingProxyManager.RotatingProxy; 8 | 9 | let proxies = []; // should be an array of RotatingProxy 10 | 11 | // or you can use RotatingProxy.buildArray() to build an array of RotatingProxy 12 | // proxiesStr can be either a path to a file or a multi-line string of proxies 13 | let proxiesStr = "123.123.123:8080\n123.123.123:8081"; 14 | proxies = RotatingProxy.buildArray(proxiesStr, 1, 3); // wait 1-3 seconds before re-using proxy 15 | 16 | let proxyManager = new RotatingProxyManager(proxies, __dirname, true); // set true to recreate proxy sqlite file 17 | proxyManager.on('ready', () => { 18 | proxyManager.nextProxy(function(err, proxy) { 19 | if (err) return console.log(err); 20 | // proxy will be the next proxy in the rotation 21 | console.log(proxy); // 123.123.123:8080 22 | }); 23 | proxyManager.nextProxy(function(err, proxy) { 24 | // you don't need to call this function nested or as a promise - 25 | // it will wait for any previous nextProxy() calls to complete first 26 | if (err) return console.log(err); 27 | console.log(proxy); // 123.123.123:8081 28 | }); 29 | }); -------------------------------------------------------------------------------- /database.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const fs = require('fs'); 3 | const sqlite3 = require('sqlite3'); 4 | 5 | class Database { 6 | constructor(dbFileName, newInstance, cb) { 7 | let fileExists = false; 8 | try { 9 | fs.accessSync(dbFileName, fs.F_OK); 10 | fileExists = true; 11 | } catch (err) {} 12 | if (newInstance && fileExists) { 13 | fs.unlinkSync(dbFileName); 14 | } 15 | let db = new sqlite3.Database(dbFileName); 16 | db.serialize(); 17 | this.db = db; 18 | if (newInstance || !fileExists) { 19 | db.run(`CREATE TABLE \`proxies\` ( 20 | \`id\` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, 21 | \`proxy\` text NOT NULL, 22 | \`min_wait\` INTEGER NOT NULL DEFAULT \'0\', 23 | \`max_wait\` INTEGER NOT NULL DEFAULT \'0\', 24 | \`num_uses\` INTEGER NOT NULL DEFAULT \'0\', 25 | \`block_until\` INTEGER NOT NULL DEFAULT \'0\', 26 | \`created_at\` INTEGER NOT NULL DEFAULT \'0\', 27 | \`updated_at\` INTEGER NOT NULL DEFAULT \'0\' 28 | )`, (err) => { 29 | if (err) throw err; 30 | cb(this); 31 | }); 32 | // TODO: block_until 33 | } else { 34 | cb(this); 35 | } 36 | } 37 | 38 | addProxy(proxy, minWait, maxWait) { 39 | return new Promise((resolve, reject) => { 40 | this.db.get(`SELECT COUNT(1) as count FROM proxies WHERE proxy = '${proxy}';`, (err, row) => { 41 | if (err) { 42 | reject(err); 43 | return; 44 | } 45 | if (row.count == 0) { 46 | let timestamp = Date.now(); 47 | this.db.run(`INSERT INTO proxies (proxy, min_wait, max_wait, created_at, updated_at) VALUES ('${proxy}', ${minWait}, ${maxWait}, ${timestamp}, ${timestamp});`, (err) => { 48 | if (err) { 49 | reject(err); 50 | return; 51 | } 52 | resolve(); 53 | }); 54 | } else { 55 | resolve(); 56 | } 57 | }); 58 | }); 59 | } 60 | 61 | nextProxy(callback) { 62 | const timestamp = Date.now(); 63 | this.db.get(`SELECT * FROM proxies WHERE block_until < ${timestamp} ORDER BY updated_at ASC, num_uses ASC LIMIT 1;`, (err, row) => { 64 | if (err) { 65 | callback(err); 66 | return; 67 | } 68 | if (!row) { 69 | return callback('No more proxies to use.'); 70 | } 71 | this.incrementProxy(row.id, () => { 72 | callback(null, row); 73 | }); 74 | }); 75 | } 76 | 77 | getProxies(callback) { 78 | this.db.all('SELECT * FROM proxies ORDER BY updated_at ASC, num_uses ASC;', (err, rows) => { 79 | if (err) { 80 | callback(err); 81 | return; 82 | } 83 | callback(null, rows); 84 | }); 85 | } 86 | 87 | incrementProxy(id, callback) { 88 | let timestamp = Date.now(); 89 | this.db.run(`UPDATE proxies SET updated_at = ${timestamp}, num_uses = num_uses + 1 WHERE id = ${id};`, (err) => { 90 | if (err) throw err; 91 | callback(); 92 | }); 93 | } 94 | 95 | blockProxy(proxy, blockUntil, callback) { 96 | this.db.run(`UPDATE proxies SET block_until = ${blockUntil} WHERE proxy = "${proxy}";`, (err) => { 97 | if (err) throw err; 98 | callback(); 99 | }); 100 | } 101 | } 102 | 103 | module.exports = Database; -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const RotatingProxyManager = require('./rotating_proxy_manager'); 2 | const RotatingProxy = require('./rotating_proxy'); 3 | 4 | RotatingProxyManager.RotatingProxy = RotatingProxy; 5 | 6 | module.exports = RotatingProxyManager; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rotating-proxy-manager", 3 | "version": "1.1.0", 4 | "description": "Rotating Proxy Manager module for Node.js", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha --reporter spec" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/waylaidwanderer/node-rotating-proxy-manager.git" 12 | }, 13 | "keywords": [ 14 | "rotating", 15 | "proxy", 16 | "manager", 17 | "proxies" 18 | ], 19 | "author": "waylaidwanderer", 20 | "license": "MIT", 21 | "bugs": { 22 | "url": "https://github.com/waylaidwanderer/node-rotating-proxy-manager/issues" 23 | }, 24 | "homepage": "https://github.com/waylaidwanderer/node-rotating-proxy-manager#readme", 25 | "dependencies": { 26 | "sqlite3": "^3.1.8" 27 | }, 28 | "devDependencies": { 29 | "mocha": "^3.2.0" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /rotating_proxy.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const fs = require('fs'); 3 | 4 | class RotatingProxy { 5 | constructor(proxy, waitMin = 0, waitMax = 0, lastUseTimestamp = 0) { 6 | this.ip = null; 7 | this.port = null; 8 | this.username = null; 9 | this.password = null; 10 | this.waitMin = waitMin; 11 | this.waitMax = waitMax; 12 | this.lastUseTimestamp = lastUseTimestamp; 13 | let split = proxy.split('@'); 14 | if (split.length == 2) { 15 | this.parseAuthString(split[0]); 16 | this.parseProxyString(split[1]); 17 | } else if (split.length == 1) { 18 | this.parseProxyString(split[0]); 19 | } else { 20 | throw new Error("Malformed proxy string."); 21 | } 22 | } 23 | 24 | static buildArray(path, waitMin = 0, waitMax = 0, lastUseTimestamp = 0) { 25 | let arr = []; 26 | let lines = ''; 27 | try { 28 | lines = fs.accessSync(path, fs.F_OK); 29 | } catch (err) { 30 | lines = path; 31 | } 32 | lines = lines.split(/\r\n|\r|\n/g); 33 | for (let i = 0; i < lines.length; i++) { 34 | let line = lines[i]; 35 | if (line.trim() == '') continue; 36 | arr.push(new RotatingProxy(line, waitMin, waitMax, lastUseTimestamp)); 37 | } 38 | return arr; 39 | } 40 | 41 | parseAuthString(str) { 42 | str = str.split(':'); 43 | if (str.length == 2) { 44 | this.username = str[0]; 45 | this.password = str[1]; 46 | } else { 47 | throw new Error("Malformed proxy string."); 48 | } 49 | } 50 | 51 | parseProxyString(str) { 52 | str = str.split(':'); 53 | if (str.length == 2) { 54 | this.ip = str[0]; 55 | this.port = str[1]; 56 | } else { 57 | throw new Error("Malformed proxy string."); 58 | } 59 | } 60 | 61 | toString() { 62 | let output = ''; 63 | if (this.username) { 64 | output += this.username + ':'; 65 | if (this.password) { 66 | output += this.password; 67 | } 68 | output += '@'; 69 | } 70 | output += this.ip + ':' + this.port; 71 | return output; 72 | } 73 | 74 | setWaitInterval(minSeconds, maxSeconds = 0) { 75 | if (maxSeconds > minSeconds) { 76 | this.waitMax = maxSeconds; 77 | } 78 | this.waitMin = minSeconds; 79 | } 80 | 81 | get waitInterval() { 82 | if (this.waitMax > this.waitMin) { 83 | let min = Math.ceil(this.waitMin * 1000); 84 | let max = Math.floor(this.waitMax * 1000); 85 | return Math.floor(Math.random() * (max - min + 1)) + min; 86 | } 87 | return Math.round(this.waitMin * 1000); 88 | } 89 | 90 | get timeSinceLastUse() { 91 | return Date.now() - this.lastUseTimestamp; 92 | } 93 | 94 | get timeToWait() { 95 | let timeToWait = this.waitInterval - this.timeSinceLastUse; 96 | return timeToWait < 0 ? 0 : timeToWait; 97 | } 98 | } 99 | 100 | module.exports = RotatingProxy; -------------------------------------------------------------------------------- /rotating_proxy_manager.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const Database = require('./database'); 3 | const RotatingProxy = require('./rotating_proxy'); 4 | 5 | class RotatingProxyManager extends EventEmitter { 6 | constructor(proxyArr, dbPath, newInstance = false) { 7 | super(); 8 | this.nextProxyReturned = true; 9 | this.proxyArr = proxyArr; 10 | this.database = new Database(dbPath+'/sqlite.db', newInstance, database => { 11 | let addProxyPromises = []; 12 | for (let i = 0; i < proxyArr.length; i++) { 13 | let proxy = proxyArr[i]; 14 | addProxyPromises.push(database.addProxy(proxy.toString(), proxy.waitMin, proxy.waitMax)); 15 | } 16 | Promise.all(addProxyPromises).then(() => { 17 | this.emit('ready'); 18 | }).catch((err) => { 19 | throw new Error('Error adding proxy to database: ' + err); 20 | }); 21 | }); 22 | } 23 | 24 | nextProxy(callback, skipWait = false) { 25 | if (this.proxyArr.length == 0) { 26 | throw new Error('No more proxies to use.'); 27 | } 28 | if (!this.nextProxyReturned) { 29 | setTimeout(() => { 30 | this.nextProxy(callback, skipWait); 31 | }, 0); 32 | return; 33 | } 34 | this.nextProxyReturned = false; 35 | this.database.nextProxy((err, dbProxy) => { 36 | if (err) { 37 | callback(err); 38 | return; 39 | } 40 | let proxy = new RotatingProxy(dbProxy.proxy, dbProxy.min_wait, dbProxy.max_wait, dbProxy.updated_at); 41 | let timeToWait = proxy.timeToWait; 42 | if (skipWait || dbProxy.num_uses == 0) { 43 | timeToWait = 0; 44 | } 45 | setTimeout(() => { 46 | this.nextProxyReturned = true; 47 | callback(null, proxy); 48 | }, timeToWait); 49 | }); 50 | } 51 | 52 | randomProxy(callback, skipWait = false) { 53 | if (this.proxyArr.length == 0) { 54 | throw new Error('No more proxies to use.'); 55 | } 56 | if (!this.nextProxyReturned) { 57 | setTimeout(() => { 58 | this.randomProxy(callback, skipWait); 59 | }, 0); 60 | return; 61 | } 62 | this.nextProxyReturned = false; 63 | this.database.getProxies((err, proxies) => { 64 | if (err) { 65 | callback(err); 66 | return; 67 | } 68 | let proxyIndex = Math.floor(Math.random() * (this.proxyArr.length)); 69 | let dbProxy = proxies[proxyIndex]; 70 | let proxy = new RotatingProxy(dbProxy.proxy, dbProxy.min_wait, dbProxy.max_wait, dbProxy.updated_at); 71 | let timeToWait = proxy.timeToWait; 72 | if (skipWait || dbProxy.num_uses == 0) { 73 | timeToWait = 0; 74 | } 75 | this.database.incrementProxy(dbProxy.id, () => { 76 | setTimeout(() => { 77 | this.nextProxyReturned = true; 78 | callback(null, proxy); 79 | }, timeToWait); 80 | }); 81 | }); 82 | } 83 | 84 | blockProxy(proxy, blockUntil, callback) { 85 | this.database.blockProxy(proxy, blockUntil, callback); 86 | } 87 | 88 | getProxies(callback) { 89 | let proxies = []; 90 | this.database.getProxies((err, dbProxies) => { 91 | if (err) { 92 | callback(err); 93 | return; 94 | } 95 | for (let i = 0; i < dbProxies.length; i++) { 96 | let dbProxy = dbProxies[i]; 97 | proxies.push(new RotatingProxy(dbProxy.proxy, dbProxy.min_wait, dbProxy.max_wait, dbProxy.updated_at)); 98 | } 99 | callback(null, proxies); 100 | }); 101 | } 102 | } 103 | 104 | module.exports = RotatingProxyManager; -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const assert = require('assert'); 3 | const fs = require('fs'); 4 | const RotatingProxyManager = require('../index'); 5 | const RotatingProxy = RotatingProxyManager.RotatingProxy; 6 | 7 | describe('RotatingProxyManager', function() { 8 | const self = this; 9 | before(function() { 10 | self.proxies = []; 11 | let rotatingProxy = new RotatingProxy('127.0.0.1:80'); 12 | self.proxies.push(rotatingProxy); 13 | rotatingProxy = new RotatingProxy('127.0.0.2:80'); 14 | rotatingProxy.setWaitInterval(5); 15 | self.proxies.push(rotatingProxy); 16 | rotatingProxy = new RotatingProxy('127.0.0.3:81'); 17 | rotatingProxy.setWaitInterval(2, 4); 18 | self.proxies.push(rotatingProxy); 19 | }); 20 | afterEach(function(done) { 21 | self.proxyManager.database.db.close(() => { 22 | done(); 23 | }); 24 | }); 25 | after(function(done) { 26 | if (!self.proxyManager) return done(); 27 | fs.unlinkSync(__dirname + '/sqlite.db'); 28 | done(); 29 | }); 30 | describe('#getProxies()', function() { 31 | it('length should equal number of proxies inserted', function(done) { 32 | createProxyManager(self.proxies, true).then(proxyManager => { 33 | self.proxyManager = proxyManager; 34 | self.proxyManager.getProxies((err, proxies) => { 35 | if (err) done(err); 36 | assert.equal(proxies.length, 3); 37 | done(); 38 | }); 39 | }).catch(err => { 40 | done(err); 41 | }); 42 | }); 43 | }); 44 | describe('#nextProxy()', function() { 45 | it('should return correct proxy', function(done) { 46 | createProxyManager(self.proxies, true).then(proxyManager => { 47 | self.proxyManager = proxyManager; 48 | return nextProxy(self.proxyManager); 49 | }).then(proxy => { 50 | assert.equal(proxy.toString(), '127.0.0.1:80'); 51 | return nextProxy(self.proxyManager); 52 | }).then(proxy => { 53 | assert.equal(proxy.toString(), '127.0.0.2:80'); 54 | return nextProxy(self.proxyManager); 55 | }).then(proxy => { 56 | assert.equal(proxy.toString(), '127.0.0.3:81'); 57 | return nextProxy(self.proxyManager); 58 | }).then(proxy => { 59 | assert.equal(proxy.toString(), '127.0.0.1:80'); 60 | done(); 61 | }).catch(err => { 62 | done(err); 63 | }); 64 | }); 65 | it('should return correct proxy across multiple RotatingProxyManagers', function(done) { 66 | createProxyManager(self.proxies, true).then(proxyManager => { 67 | self.proxyManager = proxyManager; 68 | return nextProxy(self.proxyManager); 69 | }).then(proxy => { 70 | assert.equal(proxy.toString(), '127.0.0.1:80'); 71 | return createProxyManager(self.proxies, false, self.proxyManager); 72 | }).then(proxyManager => { 73 | self.proxyManager = proxyManager; 74 | return nextProxy(self.proxyManager); 75 | }).then(proxy => { 76 | assert.equal(proxy.toString(), '127.0.0.2:80'); 77 | return createProxyManager(self.proxies, false, self.proxyManager); 78 | }).then(proxyManager => { 79 | self.proxyManager = proxyManager; 80 | return nextProxy(self.proxyManager); 81 | }).then(proxy => { 82 | assert.equal(proxy.toString(), '127.0.0.3:81'); 83 | return createProxyManager(self.proxies, false, self.proxyManager); 84 | }).then(proxyManager => { 85 | self.proxyManager = proxyManager; 86 | return nextProxy(self.proxyManager); 87 | }).then(proxy => { 88 | assert.equal(proxy.toString(), '127.0.0.1:80'); 89 | done(); 90 | }).catch(err => { 91 | done(err); 92 | }); 93 | }); 94 | it('should not return the first proxy after blocking it', function(done) { 95 | createProxyManager(self.proxies, true).then(proxyManager => { 96 | self.proxyManager = proxyManager; 97 | blockProxy(self.proxyManager, '127.0.0.1:80', Date.now() + 1000).then(() => { 98 | return nextProxy(self.proxyManager); 99 | }).then(proxy => { 100 | assert.equal(proxy.toString(), '127.0.0.2:80'); 101 | done(); 102 | }).catch(err => { 103 | done(err); 104 | }); 105 | }); 106 | }); 107 | it('should return blocked proxy when Date.now() > blockUntil time', function(done) { 108 | createProxyManager(self.proxies, true).then(proxyManager => { 109 | self.proxyManager = proxyManager; 110 | blockProxy(self.proxyManager, '127.0.0.1:80', Date.now() + 1000).then(() => { 111 | return new Promise(resolve => setTimeout(resolve, 1100)); 112 | }).then(() => { 113 | return nextProxy(self.proxyManager); 114 | }).then(proxy => { 115 | assert.equal(proxy.toString(), '127.0.0.1:80'); 116 | done(); 117 | }).catch(err => { 118 | done(err); 119 | }); 120 | }); 121 | }); 122 | }); 123 | }); 124 | 125 | function createProxyManager(proxies, newInstance = false, proxyManager = null) { 126 | return new Promise(resolve => { 127 | if (proxyManager) { 128 | proxyManager.database.db.close(() => { 129 | proxyManager = new RotatingProxyManager(proxies, __dirname, newInstance); 130 | proxyManager.on('ready', function() { 131 | resolve(proxyManager); 132 | }); 133 | }); 134 | } else { 135 | proxyManager = new RotatingProxyManager(proxies, __dirname, newInstance); 136 | proxyManager.on('ready', function() { 137 | resolve(proxyManager); 138 | }); 139 | } 140 | }); 141 | } 142 | 143 | function nextProxy(proxyManager) { 144 | return new Promise((resolve, reject) => { 145 | proxyManager.nextProxy((err, proxy) => { 146 | if (err) return reject(err); 147 | resolve(proxy); 148 | }); 149 | }); 150 | } 151 | 152 | function blockProxy(proxyManager, proxy, blockUntil) { 153 | return new Promise((resolve, reject) => { 154 | proxyManager.blockProxy(proxy, blockUntil, err => { 155 | if (err) return reject(err); 156 | resolve(); 157 | }); 158 | }); 159 | } --------------------------------------------------------------------------------