├── .github └── workflows │ └── build-docker-image.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── index.js ├── package.json └── test.js /.github/workflows/build-docker-image.yaml: -------------------------------------------------------------------------------- 1 | name: Docker build image 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | 7 | jobs: 8 | 9 | build: 10 | 11 | runs-on: 'ubuntu-latest' 12 | 13 | steps: 14 | 15 | - name: Set up QEMU 16 | uses: docker/setup-qemu-action@v2 17 | with: 18 | platforms: all 19 | 20 | - name: Check Out Repo 21 | uses: actions/checkout@v3 22 | 23 | - name: Login to Quay.io 24 | uses: docker/login-action@v2 25 | with: 26 | registry: quay.io 27 | username: ${{ secrets.QUAY_USERNAME }} 28 | password: ${{ secrets.QUAY_PASSWORD }} 29 | 30 | - name: Set up Docker Buildx 31 | uses: docker/setup-buildx-action@v2 32 | with: 33 | version: latest 34 | 35 | - name: Build and push docker image 36 | uses: docker/build-push-action@v3 37 | with: 38 | context: ./ 39 | file: ./Dockerfile 40 | platforms: linux/amd64,linux/arm64 41 | push: true 42 | tags: quay.io/unixfox/pupflare:latest, quay.io/unixfox/pupflare:master 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # Snowpack dependency directory (https://snowpack.dev/) 45 | web_modules/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | .parcel-cache 78 | 79 | # Next.js build output 80 | .next 81 | out 82 | 83 | # Nuxt.js build / generate output 84 | .nuxt 85 | dist 86 | 87 | # Gatsby files 88 | .cache/ 89 | # Comment in the public line in if your project uses Gatsby and not Next.js 90 | # https://nextjs.org/blog/next-9-1#public-directory-support 91 | # public 92 | 93 | # vuepress build output 94 | .vuepress/dist 95 | 96 | # Serverless directories 97 | .serverless/ 98 | 99 | # FuseBox cache 100 | .fusebox/ 101 | 102 | # DynamoDB Local files 103 | .dynamodb/ 104 | 105 | # TernJS port file 106 | .tern-port 107 | 108 | # Stores VSCode versions used for testing VSCode extensions 109 | .vscode-test 110 | 111 | # yarn v2 112 | .yarn/cache 113 | .yarn/unplugged 114 | .yarn/build-state.yml 115 | .yarn/install-state.gz 116 | .pnp.* 117 | package-lock.json 118 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20-alpine 2 | RUN apk add --no-cache \ 3 | chromium \ 4 | nss \ 5 | freetype \ 6 | freetype-dev \ 7 | harfbuzz \ 8 | ca-certificates \ 9 | ttf-freefont 10 | ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true 11 | ENV PUPPETEER_EXECUTABLE_PATH /usr/bin/chromium-browser 12 | WORKDIR /app 13 | COPY package*.json ./ 14 | RUN npm install 15 | COPY . . 16 | EXPOSE 3000 17 | CMD [ "npm", "start" ] 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 2 | Version 2, December 2004 3 | 4 | Copyright (C) 2004 Sam Hocevar 5 | 6 | Everyone is permitted to copy and distribute verbatim or modified 7 | copies of this license document, and changing it is allowed as long 8 | as the name is changed. 9 | 10 | DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 11 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 12 | 13 | 0. You just DO WHAT THE FUCK YOU WANT TO. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sponsor 2 | 3 | 4 | 5 | # How to launch pupflare 6 | 1. Install NodeJS 7 | 2. `npm install` 8 | 3. `npm start` 9 | 10 | # How to use 11 | Send your request to the server with the port 3000 and add your URL to the "url" query string like this: 12 | `http://localhost:3000/?url=https://example.org` 13 | 14 | This script has been configured to wait for the cloudflare challenge to pass but, you can configure the "match" for anything else using the environment variable `CHALLENGE_MATCH`. 15 | If the website that you are targeting have a protection page with "please wait" in the HTML code then launch the script like this: 16 | ``` 17 | CHALLENGE_MATCH="please wait" npm start 18 | ``` 19 | 20 | To show the browser window, set the environment variable `PUPPETEER_HEADFUL=1`. 21 | 22 | To use a proxy, 23 | set the `PUPPETEER_PROXY` environment variable, for example `PUPPETEER_PROXY=localhost:8080`. 24 | 25 | To specify user data directory, set `PUPPETEER_USERDATADIR=/path/to/dir`. 26 | 27 | To enable debugging: `DEBUG=true` and debugging with body in the logs: `DEBUG_BODY=true` 28 | 29 | # Docker 30 | Available as a Docker image here: https://quay.io/repository/unixfox/pupflare (linux/amd64,linux/arm64) 31 | 32 | 33 | ``` 34 | docker run -d -p 3000:3000 quay.io/unixfox/pupflare 35 | ``` 36 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require('puppeteer-extra'); 2 | const StealthPlugin = require('puppeteer-extra-plugin-stealth'); 3 | puppeteer.use(StealthPlugin()); 4 | const Koa = require('koa'); 5 | const bodyParser = require('koa-bodyparser'); 6 | const app = new Koa(); 7 | app.use(bodyParser()); 8 | const jsesc = require('jsesc'); 9 | 10 | const requestHeadersToRemove = [ 11 | "host", "user-agent", "accept-encoding", "content-length", 12 | "forwarded", "x-forwarded-proto", "x-forwarded-for", "x-cloud-trace-context" 13 | ]; 14 | const responseHeadersToRemove = ["Accept-Ranges", "Content-Length", "Keep-Alive", "Connection", "content-encoding", "set-cookie"]; 15 | 16 | (async () => { 17 | let options = { 18 | headless: "new", 19 | args: ['--no-sandbox', '--disable-setuid-sandbox'] 20 | }; 21 | if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) 22 | options.executablePath = '/usr/bin/chromium-browser'; 23 | if (process.env.PUPPETEER_HEADFUL) 24 | options.headless = false; 25 | if (process.env.PUPPETEER_USERDATADIR) 26 | options.userDataDir = process.env.PUPPETEER_USERDATADIR; 27 | if (process.env.PUPPETEER_PROXY) 28 | options.args.push(`--proxy-server=${process.env.PUPPETEER_PROXY}`); 29 | const browser = await puppeteer.launch(options); 30 | app.use(async ctx => { 31 | if (ctx.query.url) { 32 | const url = decodeURIComponent(ctx.url.replace("/?url=", "")); 33 | if (process.env.DEBUG) { 34 | console.log(`[DEBUG] URL: ${url}`); 35 | } 36 | let responseBody; 37 | let responseData; 38 | let responseHeaders; 39 | const page = await browser.newPage(); 40 | 41 | await page.removeAllListeners('request'); 42 | await page.setRequestInterception(true); 43 | let requestHeaders = ctx.headers; 44 | requestHeadersToRemove.forEach(header => { 45 | delete requestHeaders[header]; 46 | }); 47 | page.on('request', (request) => { 48 | requestHeaders = Object.assign({}, request.headers(), requestHeaders); 49 | if (process.env.DEBUG) { 50 | console.log(`[DEBUG] requested headers: \n${JSON.stringify(requestHeaders)}`); 51 | } 52 | if (ctx.method == "POST") { 53 | request.continue({ 54 | headers: requestHeaders, 55 | 'method': 'POST', 56 | 'postData': ctx.request.rawBody 57 | }); 58 | } else { 59 | request.continue({ headers: requestHeaders }); 60 | } 61 | }); 62 | 63 | const client = await page.target().createCDPSession(); 64 | await client.send('Network.setRequestInterception', { 65 | patterns: [{ 66 | urlPattern: '*', 67 | resourceType: 'Document', 68 | interceptionStage: 'HeadersReceived' 69 | }], 70 | }); 71 | 72 | await client.on('Network.requestIntercepted', async e => { 73 | let obj = { interceptionId: e.interceptionId }; 74 | if (e.isDownload) { 75 | await client.send('Network.getResponseBodyForInterception', { 76 | interceptionId: e.interceptionId 77 | }).then((result) => { 78 | if (result.base64Encoded) { 79 | responseData = Buffer.from(result.body, 'base64'); 80 | } 81 | }); 82 | obj['errorReason'] = 'BlockedByClient'; 83 | responseHeaders = e.responseHeaders; 84 | } 85 | await client.send('Network.continueInterceptedRequest', obj); 86 | if (e.isDownload) 87 | await page.close(); 88 | }); 89 | try { 90 | let response; 91 | let tryCount = 0; 92 | response = await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' }); 93 | ctx.status = response.status(); 94 | responseBody = await response.text(); 95 | responseData = await response.buffer(); 96 | while (responseBody.includes(process.env.CHALLENGE_MATCH || "challenge-platform") && tryCount <= 10) { 97 | newResponse = await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' }); 98 | if (newResponse) response = newResponse; 99 | responseBody = await response.text(); 100 | responseData = await response.buffer(); 101 | tryCount++; 102 | } 103 | responseHeaders = await response.headers(); 104 | const cookies = await page.cookies(); 105 | if (cookies) 106 | cookies.forEach(cookie => { 107 | const { name, value, secure, expires, domain, ...options } = cookie; 108 | ctx.cookies.set(cookie.name, cookie.value, options); 109 | }); 110 | } catch (error) { 111 | if (!error.toString().includes("ERR_BLOCKED_BY_CLIENT")) { 112 | ctx.status = 500; 113 | ctx.body = error; 114 | } 115 | } 116 | 117 | await page.close(); 118 | if (responseHeaders) { 119 | responseHeadersToRemove.forEach(header => delete responseHeaders[header]); 120 | Object.keys(responseHeaders).forEach(header => ctx.set(header, jsesc(responseHeaders[header]))); 121 | } 122 | if (process.env.DEBUG) { 123 | console.log(`[DEBUG] response headers: \n${JSON.stringify(responseHeaders)}`); 124 | } 125 | if (process.env.DEBUG_BODY) { 126 | console.log(`[DEBUG] body: \n${responseData}`); 127 | } 128 | ctx.body = responseData; 129 | } 130 | else { 131 | ctx.body = "Please specify the URL in the 'url' query string."; 132 | } 133 | }); 134 | app.listen(process.env.PORT || 3000, process.env.ADDRESS || "::"); 135 | })(); 136 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pupflare", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "node index.js" 9 | }, 10 | "keywords": [], 11 | "author": "", 12 | "license": "ISC", 13 | "dependencies": { 14 | "jsesc": "^3.1.0", 15 | "koa": "^3.0.0", 16 | "koa-bodyparser": "^4.4.1", 17 | "puppeteer": "^24.8.2", 18 | "puppeteer-extra": "^3.3.6", 19 | "puppeteer-extra-plugin-stealth": "^2.11.2" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require('puppeteer'); 2 | 3 | (async () => { 4 | // Launch the browser and open a new blank page 5 | const browser = await puppeteer.launch({headless: false}); 6 | const page = await browser.newPage(); 7 | 8 | // Navigate the page to a URL 9 | await page.goto('https://abrahamjuliot.github.io/creepjs/'); 10 | })(); --------------------------------------------------------------------------------