├── .gitignore
├── test.sh
├── README.md
└── dynamic-proxy-api.js


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/*
2 | 


--------------------------------------------------------------------------------
/test.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | curl -i "http://localhost:3333/api?url=http://httpbin.org/get&proxy=http://11.22.33.44:1234/"


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ### Changing Puppeteer Proxy Dynamically
 2 | 
 3 | Blog post topic introduction here: https://incolumitas.com/2020/12/20/dynamically-changing-puppeteer-proxies/
 4 | 
 5 | Setup:
 6 | 
 7 | ```
 8 | npm i puppeteer-core express body-parser valid-url proxy-chain
 9 | ```
10 | 
11 | and then change the line in `dynamic-proxy-api.js` if necessary to point to your local chrome browser binary:
12 | 
13 | ```
14 | const CHROME_BINARY_PATH = '/usr/bin/chromium-browser';
15 | ```
16 | 
17 | and then run:
18 | 
19 | ```
20 | node dynamic-proxy-api.js
21 | ```
22 | 
23 | Then make a request to the Api by specifying the `url` and `proxy` parameter:
24 | 
25 | ```bash
26 | curl -i "http://localhost:3333/api?url=http://httpbin.org/get&proxy=http://11.22.33.44:1234/"
27 | ```
28 | 
29 | The above Api request will open a browser and visit the url with the proxy specified. All subsequent Api requests will re-use the already running browser. The proxy for requests can be changed without restarting the browser, which is not possible with vanilla puppeteer.


--------------------------------------------------------------------------------
/dynamic-proxy-api.js:
--------------------------------------------------------------------------------
  1 | const express = require('express');
  2 | const bodyParser = require('body-parser');
  3 | const validUrl = require('valid-url');
  4 | const puppeteer = require('puppeteer-core');
  5 | const ProxyChain = require('proxy-chain');
  6 | 
  7 | const CHROME_BINARY_PATH = '/usr/bin/chromium-browser';
  8 | const app = express();
  9 | const port = 3333;
 10 | const proxyServerPort = 8947;
 11 | let browser = null;
 12 | 
 13 | app.use(express.json());
 14 | app.use(bodyParser.json({ limit: '2mb' }));
 15 | 
 16 | function log(msg) {
 17 |   console.log(`[${(new Date()).getTime()}] - ${msg}`);
 18 | }
 19 | 
 20 | function validateProxy(proxy) {
 21 |   let match = false;
 22 |   let prefixes = ['http://', 'https://', 'socks://', 'socks5://', 'socks4://'];
 23 |   for (let prefix of prefixes) {
 24 |     if (proxy.startsWith(prefix)) {
 25 |       match = true;
 26 |     }
 27 |   }
 28 |   if (match === false) {
 29 |     return false;
 30 |   }
 31 |   return validUrl.isWebUri(proxy);
 32 | }
 33 | 
 34 | async function getBrowser() {
 35 |   if (browser === null) {
 36 |     browser = await puppeteer.launch({
 37 |       executablePath: CHROME_BINARY_PATH,
 38 |       headless: false,
 39 |       args: [`--proxy-server=http://localhost:` + proxyServerPort],
 40 |     });
 41 |     log('Browser started');
 42 |   }
 43 | }
 44 | 
 45 | async function startProxyServer(proxy) {
 46 |   return new Promise(function(resolve, reject) {
 47 |     const server = new ProxyChain.Server({
 48 |       port: proxyServerPort,
 49 |       verbose: false,
 50 |       prepareRequestFunction: function (params) {
 51 |         var {request, username, password, hostname, port, isHttp, connectionId} = params;
 52 |         return {
 53 |           requestAuthentication: false,
 54 |           // http://username:password@proxy.example.com:3128
 55 |           upstreamProxyUrl: proxy,
 56 |         };
 57 |       },
 58 |     });
 59 | 
 60 |     // Emitted when HTTP connection is closed
 61 |     server.on('connectionClosed', (params) => {
 62 |       var {connectionId, stats} = params;
 63 |       log(`Connection ${connectionId} closed`);
 64 |     });
 65 | 
 66 |     // Emitted when HTTP request fails
 67 |     server.on('requestFailed', (params) => {
 68 |       var {request, error} = params;
 69 |       log(`Request ${request.url} failed with error ${error}`);
 70 |     });
 71 | 
 72 |     server.listen(() => {
 73 |       log(`ProxyServer listening on port ${server.port}`);
 74 |       resolve(server);
 75 |     });
 76 |   });
 77 | }
 78 | 
 79 | async function clearCookies(page) {
 80 |   try {
 81 |     log('Deleting cookies with Network.clearBrowserCookies');
 82 |     const client = await page.target().createCDPSession();
 83 |     await client.send('Network.clearBrowserCookies');
 84 |   } catch (err) {
 85 |     log(`Could not delete cookies: ${err.toString()}`);
 86 |   }
 87 | }
 88 | 
 89 | app.get('/api', async (req, res) => {
 90 |   if (req.query.proxy) {
 91 |     if (!validateProxy(req.query.proxy)) {
 92 |       return res.status(403).send('Invalid proxy format');
 93 |     } else {
 94 |       log(`Using proxy: ${req.query.proxy}`);
 95 |     }
 96 |   }
 97 | 
 98 |   if (req.query.url) {
 99 |     if (!validUrl.isWebUri(req.query.url)) {
100 |       return res.status(403).send(`url is not valid`);
101 |     }
102 |   } else {
103 |     return res.status(403).send(`url is required`);
104 |   }
105 | 
106 |   let proxyServer = await startProxyServer(req.query.proxy);
107 | 
108 |   await getBrowser();
109 |   let page = await browser.newPage();
110 |   await page.goto(req.query.url, { waitUntil: "domcontentloaded" });
111 |   let content = await page.content();
112 |   // clear cookies after we are done
113 |   await clearCookies(page);
114 | 
115 |   proxyServer.close();
116 |   await page.close();
117 | 
118 |   res.set('Content-Type', 'text/html');
119 |   return res.send(Buffer.from(content));
120 | });
121 | 
122 | app.listen(port, () => {
123 |   log(`Dynamic proxy puppeteer Api listening on port ${port}`);
124 | });


--------------------------------------------------------------------------------