├── .editorconfig ├── .eslintrc ├── .gitattributes ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug.md │ ├── config.yml │ └── feature_request.md ├── labeler.yml └── workflows │ ├── extract-stealth.yml │ ├── label.yml │ └── test.yml ├── .gitignore ├── .prettierrc.js ├── .travis.yml ├── LICENSE ├── README.md ├── lerna.json ├── package.json ├── packages ├── extract-stealth-evasions │ ├── .gitignore │ ├── index.js │ ├── package.json │ └── readme.md ├── playwright-extra │ ├── .prettierrc.js │ ├── package.json │ ├── readme.md │ ├── rollup.config.ts │ ├── src │ │ ├── extra.ts │ │ ├── helper │ │ │ └── loader.ts │ │ ├── index.ts │ │ ├── plugins.ts │ │ ├── puppeteer-compatiblity-shim │ │ │ ├── index.ts │ │ │ └── playwright-shim.d.ts │ │ └── types │ │ │ └── index.ts │ ├── test │ │ ├── exports.spec.ts │ │ ├── fixtures │ │ │ ├── dummyplugin.ts │ │ │ └── extra.ts │ │ ├── playwright.config.ts │ │ ├── plugin-events.spec.ts │ │ └── puppeteer-plugins │ │ │ ├── anonymize-ua.spec.ts │ │ │ ├── recaptcha.spec.ts │ │ │ └── stealth.spec.ts │ └── tsconfig.json ├── plugin-proxy-router │ ├── package.json │ ├── readme.md │ ├── rollup.config.ts │ ├── src │ │ ├── index.ts │ │ ├── plugin.ts │ │ ├── router.ts │ │ ├── stats.ts │ │ └── utils │ │ │ └── port.ts │ ├── tsconfig.json │ └── tslint.json ├── puppeteer-extra-plugin-adblocker │ ├── ava.config-ts.js │ ├── ava.config.js │ ├── build_version_check.js │ ├── package.json │ ├── readme.md │ ├── rollup.config.ts │ ├── src │ │ ├── ambient.d.ts │ │ ├── index.test.ts │ │ └── index.ts │ ├── tsconfig.json │ └── tslint.json ├── puppeteer-extra-plugin-anonymize-ua │ ├── index.d.ts │ ├── index.js │ ├── index.test.js │ ├── package.json │ ├── readme.md │ └── test │ │ ├── headless.js │ │ ├── headless_off.js │ │ ├── popup.js │ │ └── stresstest.js ├── puppeteer-extra-plugin-block-resources │ ├── example.js │ ├── index.d.ts │ ├── index.js │ ├── index.test.js │ ├── package.json │ └── readme.md ├── puppeteer-extra-plugin-click-and-wait │ ├── example.js │ ├── index.js │ ├── package.json │ └── readme.md ├── puppeteer-extra-plugin-devtools │ ├── index.js │ ├── index.test.js │ ├── lib │ │ ├── RemoteDevTools.js │ │ └── RemoteDevTools.test.js │ ├── package.json │ ├── readme.md │ └── test │ │ └── headless.js ├── puppeteer-extra-plugin-flash │ ├── example.js │ ├── index.js │ ├── package.json │ └── readme.md ├── puppeteer-extra-plugin-font-size │ ├── index.js │ ├── package.json │ └── readme.md ├── puppeteer-extra-plugin-recaptcha │ ├── ava.config-ts.js │ ├── ava.config.js │ ├── package.json │ ├── readme.md │ ├── rollup.config.ts │ ├── src │ │ ├── ambient.d.ts │ │ ├── content-hcaptcha.ts │ │ ├── content.ts │ │ ├── detection.test.ts │ │ ├── index.test.ts │ │ ├── index.ts │ │ ├── playwright-mods.d.ts │ │ ├── provider │ │ │ ├── 2captcha-api.ts │ │ │ └── 2captcha.ts │ │ ├── puppeteer-mods.d.ts │ │ ├── solve.test.ts │ │ └── types.ts │ ├── tsconfig.json │ └── tslint.json ├── puppeteer-extra-plugin-repl │ ├── index.d.ts │ ├── index.js │ ├── index.test.js │ ├── lib │ │ ├── REPLSession.js │ │ ├── REPLSession.test.js │ │ ├── super-readline.js │ │ └── super-readline.test.js │ ├── package.json │ ├── readme.md │ └── test │ │ └── headless.js ├── puppeteer-extra-plugin-stealth │ ├── .npmignore │ ├── evasions │ │ ├── _template │ │ │ ├── index.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── _utils │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── readme.md │ │ │ └── withUtils.js │ │ ├── chrome.app │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── chrome.csi │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── chrome.loadTimes │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── chrome.runtime │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ ├── readme.md │ │ │ └── staticData.json │ │ ├── defaultArgs │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── iframe.contentWindow │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── media.codecs │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── navigator.hardwareConcurrency │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── navigator.languages │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── navigator.permissions │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── navigator.plugins │ │ │ ├── data.json │ │ │ ├── functionMocks.js │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── magicArray.js │ │ │ ├── mimeTypes.js │ │ │ ├── mimeTypes.test.js │ │ │ ├── package.json │ │ │ ├── plugins.js │ │ │ ├── plugins.test.js │ │ │ └── readme.md │ │ ├── navigator.vendor │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── navigator.webdriver │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── readme.md │ │ ├── sourceurl │ │ │ ├── _fixtures │ │ │ │ └── test.html │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── user-agent-override │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ ├── webgl.vendor │ │ │ ├── index.js │ │ │ ├── index.test.js │ │ │ ├── package.json │ │ │ └── readme.md │ │ └── window.outerdimensions │ │ │ ├── index.js │ │ │ ├── package.json │ │ │ └── readme.md │ ├── examples │ │ ├── detect-headless.js │ │ ├── test1.js │ │ └── test2.js │ ├── index.d.ts │ ├── index.js │ ├── index.test.js │ ├── package.json │ ├── readme.md │ ├── runall_stealthtests.sh │ ├── stealthtests │ │ ├── _results │ │ │ ├── _thumbs │ │ │ │ ├── headful-chrome-stealth.js.png │ │ │ │ ├── headful-chrome-vanilla.js.png │ │ │ │ ├── headful-chromium-stealth.js.png │ │ │ │ ├── headful-chromium-vanilla.js.png │ │ │ │ ├── headless-chrome-stealth.js.png │ │ │ │ ├── headless-chrome-vanilla.js.png │ │ │ │ ├── headless-chromium-stealth.js.png │ │ │ │ └── headless-chromium-vanilla.js.png │ │ │ ├── headful-chrome-stealth.js.png │ │ │ ├── headful-chrome-vanilla.js.png │ │ │ ├── headful-chromium-stealth.js.png │ │ │ ├── headful-chromium-vanilla.js.png │ │ │ ├── headless-chrome-stealth.js.png │ │ │ ├── headless-chrome-vanilla.js.png │ │ │ ├── headless-chromium-stealth.js.png │ │ │ └── headless-chromium-vanilla.js.png │ │ ├── headful-chrome-stealth.js │ │ ├── headful-chrome-vanilla.js │ │ ├── headful-chromium-stealth.js │ │ ├── headful-chromium-vanilla.js │ │ ├── headless-chrome-stealth.js │ │ ├── headless-chrome-vanilla.js │ │ ├── headless-chromium-stealth.js │ │ └── headless-chromium-vanilla.js │ └── test │ │ ├── cat-and-mouse.test.js │ │ ├── fixtures │ │ ├── dummy-with-service-worker.html │ │ ├── dummy.html │ │ └── sw.js │ │ ├── fpscanner.test.js │ │ ├── service-worker.test.js │ │ └── util.js ├── puppeteer-extra-plugin-user-data-dir │ ├── index.js │ ├── package.json │ └── readme.md ├── puppeteer-extra-plugin-user-preferences │ ├── index.js │ ├── package.json │ └── readme.md ├── puppeteer-extra-plugin │ ├── ava.config-ts.js │ ├── ava.config.js │ ├── package.json │ ├── readme.md │ ├── rollup.config.ts │ ├── src │ │ ├── ambient.d.ts │ │ ├── index.test.ts │ │ ├── index.ts │ │ └── puppeteer.ts │ ├── tsconfig.json │ └── tslint.json └── puppeteer-extra │ ├── ava.config-ts.js │ ├── ava.config.js │ ├── package.json │ ├── readme.md │ ├── rollup.config.ts │ ├── src │ ├── ambient.d.ts │ ├── index.ts │ └── puppeteer-legacy.d.ts │ ├── test │ ├── addExtra.ts │ ├── basic.ts │ ├── connect.js │ ├── events.js │ ├── options.js │ └── plugin-support.js │ ├── tsconfig.json │ └── tslint.json └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig helps developers define and maintain 2 | # consistent coding styles between different editors and IDEs. 3 | 4 | root = true 5 | 6 | [*] 7 | charset = utf-8 8 | indent_style = space 9 | indent_size = 2 10 | end_of_line = lf 11 | insert_final_newline = true 12 | trim_trailing_whitespace = true 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false 16 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["prettier-standard"], 3 | "rules": { 4 | "lines-between-class-members": "off" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | custom: https://www.buymeacoffee.com/brstnd 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a bug report 4 | title: '[Bug] ' 5 | labels: 'issue: bug report, needs triage' 6 | --- 7 | 8 | 21 | 22 | **Describe the bug** 23 | 24 | 29 | 30 | **Code Snippet** 31 | 32 | 37 | 38 | ```javascript 39 | const puppeteer = require('puppeteer-extra') 40 | 41 | ;(async () => { 42 | const browser = await puppeteer.launch() 43 | // ... 44 | })() 45 | ``` 46 | 47 | **Versions** 48 | 49 | 54 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | contact_links: 2 | - name: Questions and Help 3 | url: https://github.com/berstend/puppeteer-extra/wiki/Scraping-Chat 4 | about: This issue tracker is not for support questions. You can join our Discord server and ask the community for help. 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea or feature 4 | title: '[Feature] ' 5 | labels: 'issue: proposal' 6 | --- 7 | 8 | **Feature request** 9 | 10 | 22 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | # This is used with the label workflow which 2 | # will triage pull requests and apply a label based on the 3 | # paths that are modified in the pull request. 4 | # 5 | # For more information, see: 6 | # https://github.com/actions/labeler 7 | 8 | 'package: core': 9 | - packages/automation-extra/**/* 10 | - packages/playright-extra/**/* 11 | - packages/puppeteer-extra/**/* 12 | - packages/automation-extra-plugin/**/* 13 | - packages/puppeteer-extra-plugin/**/* 14 | 15 | 'plugin: automation-extra': 16 | - packages/plugin-*/**/* 17 | 18 | 'plugin: puppeteer-extra': 19 | - packages/puppeteer-extra-plugin-*/**/* 20 | 21 | 'plugin: recaptcha 🏴': 22 | - packages/*recaptcha*/**/* 23 | 24 | 'plugin: stealth ㊙️': 25 | - packages/*stealth*/**/* 26 | -------------------------------------------------------------------------------- /.github/workflows/extract-stealth.yml: -------------------------------------------------------------------------------- 1 | name: Extract stealth.min.js 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | paths: 8 | - 'packages/puppeteer-extra-plugin-stealth/**' 9 | - 'packages/extract-stealth-evasions/**' 10 | - '.github/workflows/extract-stealth.yml' 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Sleep for 190 seconds 17 | uses: jakejarvis/wait-action@master 18 | with: 19 | time: '190s' 20 | 21 | - name: Checkout 22 | uses: actions/checkout@v2 23 | 24 | - name: 'Fix for: error fsevents@2.1.2: The platform "linux" is incompatible with this module.' 25 | run: npx json -I -f package.json -e 'this.resolutions={}' 26 | 27 | - name: Build packages 28 | run: | 29 | yarn install 30 | yarn bootstrap 31 | yarn build 32 | 33 | - name: Extract stealth.min.js 34 | run: | 35 | cd packages/extract-stealth-evasions 36 | node index.js 37 | cp stealth.min.js ../../ 38 | 39 | - name: Commit stealth.min.js 40 | uses: EndBug/add-and-commit@v4 41 | with: 42 | add: 'stealth.min.js' 43 | force: true 44 | ref: 'stealth-js' 45 | message: 'Auto-updated stealth.min.js with newest evasions' 46 | env: 47 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 48 | -------------------------------------------------------------------------------- /.github/workflows/label.yml: -------------------------------------------------------------------------------- 1 | # This workflow will triage pull requests and apply a label based on the 2 | # paths that are modified in the pull request. 3 | # 4 | # To use this workflow, you will need to set up a .github/labeler.yml 5 | # file with configuration. For more information, see: 6 | # https://github.com/actions/labeler 7 | 8 | name: "Pull Request Labeler" 9 | on: 10 | - pull_request_target 11 | 12 | jobs: 13 | triage: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/labeler@main 17 | with: 18 | repo-token: "${{ secrets.GITHUB_TOKEN }}" 19 | sync-labels: true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | node_modules 5 | 6 | # builds 7 | build 8 | dist 9 | 10 | # misc 11 | .DS_Store 12 | .env 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | .cache 18 | .rpt2_cache 19 | 20 | lerna-debug.log* 21 | lerna-error.log* 22 | npm-debug.log* 23 | yarn-debug.log* 24 | yarn-error.log* 25 | 26 | TODO.md 27 | packages/testing/ 28 | packages/plugin-stealth/ 29 | packages/puppeteer-extra2/ 30 | packages/puppeteer-extra-old/ 31 | packages/test-* 32 | packages/internal-testing-* 33 | testing/ 34 | 35 | *.tgz* 36 | 37 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('prettier-config-standard'), 3 | 4 | // override for Windows 5 | endOfLine: 'lf', 6 | } 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | dist: trusty 3 | addons: 4 | apt: 5 | packages: 6 | # This is required to run new chrome on old trusty 7 | - libnss3 8 | 9 | language: node_js 10 | 11 | # allow headful tests 12 | before_install: 13 | - "export DISPLAY=:99.0" 14 | - "sh -e /etc/init.d/xvfb start" 15 | 16 | # test against multiple node versions 17 | node_js: 18 | - '13' 19 | - '10' 20 | 21 | # Fix for: error fsevents@2.1.2: The platform "linux" is incompatible with this module. 22 | install: skip 23 | 24 | # Prevent potential issues 25 | cache: 26 | npm: false 27 | yarn: false 28 | 29 | # test against multiple puppeteer versions 30 | env: 31 | - PUPPETEER_VERSION=5.0.0 32 | - PUPPETEER_VERSION=2.1.1 # Chromium 79.0.3942.0, Oct 24 2019 33 | # - PUPPETEER_VERSION=2.0.0 # Chromium 79.0.3942.0, Oct 24 2019 34 | # - PUPPETEER_VERSION=1.20.0 # Chromium 78.0.3882.0, Sep 13 2019 35 | # - PUPPETEER_VERSION=1.15.0 # Chromium 75.0.3765.0, Apr 26 2019 36 | # - PUPPETEER_VERSION=1.9.0 # Chromium 71.0.3563.0, Oct 4, 2018 37 | # - PUPPETEER_VERSION=1.6.2 # Chromium 69.0.3494.0, Aug 1, 2018 38 | 39 | script: 40 | # Make sure to use latest @next package 41 | # https://github.com/yarnpkg/yarn/issues/4731 42 | 43 | # Fix for: error fsevents@2.1.2: The platform "linux" is incompatible with this module. 44 | - npx json -I -f package.json -e 'this.resolutions={}' 45 | # - npx json -I -f package.json -e 'this.resolutions={"**/puppeteer":"'${PUPPETEER_VERSION}'"}' 46 | 47 | # Install older version when required 48 | - rm -rf ./node_modules/ 49 | # - 'yarn; echo 0' 50 | # - 'yarn lerna exec "rm -f yarn.lock; rm -rf node_modules; echo 0"' 51 | # - 'rm -rf yarn.lock && yarn cache clean && rm -rf ./node_modules/puppeteer && yarn lerna add puppeteer@${PUPPETEER_VERSION}' 52 | # - "yarn lerna exec --concurrency 1 'yarn set resolution --save puppeteer@* 5.0.0; echo 0'" 53 | 54 | - 'yarn' 55 | - 'yarn bootstrap' 56 | - yarn lerna add puppeteer@${PUPPETEER_VERSION} 57 | - 'yarn lerna link' 58 | - 'yarn lerna run build --concurrency 1' 59 | 60 | # For debugging 61 | - yarn list puppeteer 62 | - yarn list puppeteer-extra 63 | - file node_modules/puppeteer-extra/dist/index.cjs.js 64 | 65 | # Run tests 66 | - yarn test-ci 67 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 berstend 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": ["packages/*"], 3 | "version": "independent", 4 | "useWorkspaces": true, 5 | "npmClient": "yarn" 6 | } 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "description": "Modular framework to teach Puppeteer new tricks.", 4 | "repository": "berstend/puppeteer-extra", 5 | "author": "berstend", 6 | "license": "MIT", 7 | "main": "packages/puppeteer-extra/index.js", 8 | "engines": { 9 | "node": ">=8" 10 | }, 11 | "scripts": { 12 | "bootstrap": "lerna bootstrap", 13 | "build": "yarn lerna exec --concurrency 1 'yarn build; echo 0'", 14 | "docs": "lerna run docs", 15 | "test": "lerna run test --concurrency 1 --stream", 16 | "test-ci": "lerna run test-ci --concurrency 1 --stream", 17 | "prepare": "lerna run prepare", 18 | "release": "lerna publish --npm-client npm" 19 | }, 20 | "workspaces": { 21 | "packages": [ 22 | "packages/*" 23 | ], 24 | "nohoist": [ 25 | "**/@types", 26 | "**/@types/**", 27 | "**/typescript", 28 | "**/typescript/**" 29 | ] 30 | }, 31 | "devDependencies": { 32 | "eslint": "^6.7.1", 33 | "eslint-config-prettier": "^6.7.0", 34 | "eslint-config-prettier-standard": "^3.0.1", 35 | "eslint-config-standard": "^14.1.0", 36 | "eslint-plugin-import": "^2.18.2", 37 | "eslint-plugin-node": "^10.0.0", 38 | "eslint-plugin-prettier": "^3.1.1", 39 | "eslint-plugin-promise": "^4.2.1", 40 | "eslint-plugin-standard": "^4.0.1", 41 | "lerna": "^3.19.0", 42 | "lerna-update-wizard": "^0.17.5", 43 | "prettier": "^1.19.1", 44 | "prettier-config-standard": "^1.0.1" 45 | }, 46 | "optionalDependencies": { 47 | "fsevents": "^2.1.2" 48 | }, 49 | "resolutions": { 50 | "**/fsevents": "^2.1.2" 51 | }, 52 | "dependencies": {}, 53 | "version": "0.0.0" 54 | } 55 | -------------------------------------------------------------------------------- /packages/extract-stealth-evasions/.gitignore: -------------------------------------------------------------------------------- 1 | stealth.min.js 2 | stealth.js -------------------------------------------------------------------------------- /packages/extract-stealth-evasions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "extract-stealth-evasions", 3 | "version": "2.7.3", 4 | "description": "Extract stealth evasions from puppeteer-extra-plugin-stealth", 5 | "main": "index.js", 6 | "bin": { 7 | "extract-stealth-evasions": "./index.js" 8 | }, 9 | "homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/extract-stealth-evasions#readme", 10 | "repository": "berstend/puppeteer-extra", 11 | "author": "berstend", 12 | "license": "MIT", 13 | "engines": { 14 | "node": ">=8" 15 | }, 16 | "keywords": [ 17 | "puppeteer", 18 | "puppeteer-extra", 19 | "puppeteer-extra-plugin", 20 | "chrome", 21 | "headless", 22 | "pupeteer" 23 | ], 24 | "dependencies": { 25 | "puppeteer": "*", 26 | "puppeteer-extra": "^3.3.6", 27 | "puppeteer-extra-plugin-stealth": "^2.11.2", 28 | "terser": "^5.1.0", 29 | "yargs": "^15.4.1" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /packages/extract-stealth-evasions/readme.md: -------------------------------------------------------------------------------- 1 | # extract-stealth-evasions 2 | 3 | This script offers a quick way to extract the latest stealth evasions from [puppeteer-extra-stealth](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth) to (minified) JavaScript. The resulting JS file can be used in pure [CDP](https://chromedevtools.github.io/devtools-protocol/tot/) implementations or to test the evasions in your devtools. 4 | 5 | #### Usage with `npx` 6 | 7 | You don't need to install anything, `npx` runs wherever NodeJS is installed. :-) 8 | 9 | ```bash 10 | npx extract-stealth-evasions 11 | ``` 12 | 13 | Will create a `stealth.min.js` file in the current folder. 14 | 15 | #### Using the CDN version 16 | 17 | You can also fetch the latest version from [gitCDN](https://gitcdn.xyz/repo/berstend/puppeteer-extra/stealth-js/stealth.min.js). For example, paste this one-liner in your browser devtools console: 18 | 19 | ```js 20 | document.body.appendChild(Object.assign(document.createElement('script'), {src: 'https://gitcdn.xyz/repo/berstend/puppeteer-extra/stealth-js/stealth.min.js'})) 21 | ``` 22 | 23 | #### How to use locally 24 | 25 | ```bash 26 | yarn install 27 | node index.js 28 | ``` 29 | 30 | Use the resulting `stealth.min.js` file however you like. 31 | 32 | #### Options 33 | 34 | ```bash 35 | $ npx extract-stealth-evasions -h 36 | Usage: extract-stealth-evasions [options] 37 | 38 | Options: 39 | --version Show version number [boolean] 40 | -e, --exclude Exclude evasion (repeat for multiple) 41 | -i, --include Include evasion (repeat for multiple) 42 | -l, --list List available evasions 43 | -h, --help Show help [boolean] 44 | -m, --minify Minify the output [boolean] [default: true] 45 | ``` -------------------------------------------------------------------------------- /packages/playwright-extra/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = 'prettier-config-standard' 2 | -------------------------------------------------------------------------------- /packages/playwright-extra/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "playwright-extra", 3 | "version": "4.3.6", 4 | "description": "Teach playwright new tricks through plugins.", 5 | "repository": "berstend/puppeteer-extra", 6 | "homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/playwright-extra#readme", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "typings": "dist/index.d.ts", 10 | "main": "dist/index.cjs.js", 11 | "module": "dist/index.esm.js", 12 | "files": [ 13 | "dist" 14 | ], 15 | "scripts": { 16 | "clean": "rimraf dist/*", 17 | "prebuild": "run-s clean", 18 | "build": "run-s build:tsc build:rollup ambient-dts", 19 | "build:tsc": "tsc --module commonjs", 20 | "build:rollup": "rollup -c rollup.config.ts", 21 | "docs": "echo \"No docs\"", 22 | "test": "yarn playwright test --config test/playwright.config.ts", 23 | "test-ci": "run-s test", 24 | "ambient-dts": "run-s ambient-dts-copy ambient-dts-fix-path", 25 | "ambient-dts-copy": "copyfiles -u 1 \"src/**/*.d.ts\" dist", 26 | "ambient-dts-fix-path": "replace-in-files --string='/// =12" 40 | }, 41 | "devDependencies": { 42 | "@playwright/test": "^1.23.1", 43 | "@types/debug": "^4.1.7", 44 | "@types/node": "^18.0.0", 45 | "esbuild": "^0.14.47", 46 | "esbuild-register": "^3.3.3", 47 | "npm-run-all": "^4.1.5", 48 | "playwright": "1.24.2", 49 | "prettier": "^2.7.1", 50 | "puppeteer-extra-plugin": "^3.2.3", 51 | "puppeteer-extra-plugin-anonymize-ua": "^2.4.5", 52 | "rimraf": "^3.0.0", 53 | "rollup": "^1.27.5", 54 | "rollup-plugin-commonjs": "^10.1.0", 55 | "rollup-plugin-node-resolve": "^5.2.0", 56 | "rollup-plugin-sourcemaps": "^0.4.2", 57 | "rollup-plugin-typescript2": "^0.25.2", 58 | "typescript": "4.4.3" 59 | }, 60 | "dependencies": { 61 | "debug": "^4.3.4" 62 | }, 63 | "peerDependencies": { 64 | "playwright": "*", 65 | "playwright-core": "*" 66 | }, 67 | "peerDependenciesMeta": { 68 | "playwright": { 69 | "optional": true 70 | }, 71 | "playwright-core": { 72 | "optional": true 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /packages/playwright-extra/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import commonjs from 'rollup-plugin-commonjs' 2 | import resolve from 'rollup-plugin-node-resolve' 3 | import sourceMaps from 'rollup-plugin-sourcemaps' 4 | import typescript from 'rollup-plugin-typescript2' 5 | 6 | const pkg = require('./package.json') 7 | 8 | const entryFile = 'index' 9 | const banner = ` 10 | /*! 11 | * ${pkg.name} v${pkg.version} by ${pkg.author} 12 | * ${pkg.homepage || `https://github.com/${pkg.repository}`} 13 | * @license ${pkg.license} 14 | */ 15 | `.trim() 16 | 17 | const defaultExportOutro = ` 18 | module.exports = exports.default || {} 19 | Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value }) 20 | ` 21 | 22 | export default { 23 | input: `src/${entryFile}.ts`, 24 | output: [ 25 | { 26 | file: pkg.main, 27 | format: 'cjs', 28 | sourcemap: true, 29 | exports: 'named', 30 | outro: defaultExportOutro, 31 | banner 32 | }, 33 | { 34 | file: pkg.module, 35 | format: 'es', 36 | sourcemap: true, 37 | exports: 'named', 38 | banner 39 | } 40 | ], 41 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 42 | external: [ 43 | ...Object.keys(pkg.dependencies || {}), 44 | ...Object.keys(pkg.peerDependencies || {}) 45 | ], 46 | watch: { 47 | include: 'src/**' 48 | }, 49 | plugins: [ 50 | // Compile TypeScript files 51 | typescript({ useTsconfigDeclarationDir: true }), 52 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 53 | commonjs(), 54 | // Allow node_modules resolution, so you can use 'external' to control 55 | // which external modules to include in the bundle 56 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 57 | resolve(), 58 | // Resolve source maps to the original source 59 | sourceMaps() 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /packages/playwright-extra/src/puppeteer-compatiblity-shim/playwright-shim.d.ts: -------------------------------------------------------------------------------- 1 | // Playwright objects extended with puppeteer compatiblity shims 2 | 3 | import type {} from 'playwright-core' 4 | 5 | import type { PuppeteerPageShim, PuppeteerBrowserShim } from '.' 6 | 7 | declare module 'playwright-core' { 8 | interface Page extends PuppeteerPageShim {} 9 | interface Frame extends PuppeteerPageShim {} 10 | interface Browser extends PuppeteerBrowserShim {} 11 | } 12 | -------------------------------------------------------------------------------- /packages/playwright-extra/test/fixtures/dummyplugin.ts: -------------------------------------------------------------------------------- 1 | import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin' 2 | 3 | export class DummyPlugin extends PuppeteerExtraPlugin { 4 | public pluginEventList: string[] = [] 5 | public pluginEventMap: Map = new Map() 6 | 7 | constructor(opts = {}) { 8 | super(opts) 9 | } 10 | get name() { 11 | return 'dummy' 12 | } 13 | 14 | async onPluginRegistered(...args: any[]) { 15 | this.pluginEventList.push('onPluginRegistered') 16 | } 17 | async beforeLaunch(...args: any[]) { 18 | this.pluginEventList.push('beforeLaunch') 19 | } 20 | async afterLaunch(...args: any[]) { 21 | this.pluginEventList.push('afterLaunch') 22 | } 23 | async beforeConnect(...args: any[]) { 24 | this.pluginEventList.push('beforeConnect') 25 | } 26 | async afterConnect(...args: any[]) { 27 | this.pluginEventList.push('afterConnect') 28 | } 29 | async onBrowser(...args: any[]) { 30 | this.pluginEventList.push('onBrowser') 31 | } 32 | async onTargetCreated(...args: any[]) { 33 | this.pluginEventList.push('onTargetCreated') 34 | } 35 | async onPageCreated(...args: any[]) { 36 | this.pluginEventList.push('onPageCreated') 37 | } 38 | async onTargetChanged(...args: any[]) { 39 | this.pluginEventList.push('onTargetChanged') 40 | } 41 | async onTargetDestroyed(...args: any[]) { 42 | this.pluginEventList.push('onTargetDestroyed') 43 | } 44 | async onDisconnected(...args: any[]) { 45 | this.pluginEventList.push('onDisconnected') 46 | } 47 | async onClose(...args: any[]) { 48 | this.pluginEventList.push('onClose') 49 | } 50 | 51 | // playwright only at the moment 52 | async beforeContext(...args: any[]) { 53 | this.pluginEventList.push('beforeContext') 54 | } 55 | async onContextCreated(...args: any[]) { 56 | this.pluginEventList.push('onContextCreated') 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /packages/playwright-extra/test/playwright.config.ts: -------------------------------------------------------------------------------- 1 | import { type PlaywrightTestConfig } from '@playwright/test' 2 | 3 | const config: PlaywrightTestConfig = { 4 | retries: 3, 5 | workers: 3, 6 | 7 | use: { 8 | browserName: 'chromium' 9 | }, 10 | 11 | projects: [ 12 | { 13 | name: 'chromium', 14 | use: { 15 | browserName: 'chromium', 16 | launchOptions: { 17 | chromiumSandbox: process.env.CI ? false : true, 18 | args: process.env.CI 19 | ? ['--no-sandbox', '--disable-setuid-sandbox'] 20 | : [] 21 | } 22 | } 23 | }, 24 | { 25 | name: 'firefox', 26 | use: { 27 | browserName: 'firefox' 28 | } 29 | }, 30 | { 31 | name: 'webkit', 32 | use: { 33 | browserName: 'webkit' 34 | // Note: webkit doesn't support --no-sandbox 35 | } 36 | } 37 | ] 38 | } 39 | 40 | export default config 41 | -------------------------------------------------------------------------------- /packages/playwright-extra/test/puppeteer-plugins/anonymize-ua.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from '../fixtures/extra' 2 | 3 | import AnonymizeUAPlugin from 'puppeteer-extra-plugin-anonymize-ua' 4 | 5 | test('puppeteer-extra-plugin-anonymize-ua will remove headless', async ({ 6 | browserName, 7 | extraLauncher, 8 | _browserOptions 9 | }) => { 10 | test.skip(browserName !== 'chromium', 'Chromium only') 11 | 12 | const pluginErrors = [] 13 | extraLauncher.plugins.onPluginError = (plugin, method, err) => { 14 | pluginErrors.push(err) 15 | } 16 | 17 | extraLauncher.use(AnonymizeUAPlugin()) 18 | expect(extraLauncher.plugins.list.length).toEqual(1) 19 | expect(extraLauncher.plugins.list[0].name).toEqual('anonymize-ua') 20 | 21 | const browser = await extraLauncher.launch(_browserOptions) 22 | const context = await browser.newContext() 23 | const page = await context.newPage() 24 | await page.goto('https://example.com') 25 | 26 | const ua = await page.evaluate(() => navigator.userAgent) 27 | expect(ua.includes('Headless')).toBeFalsy() 28 | expect(pluginErrors).toStrictEqual([]) 29 | 30 | await browser.close() 31 | }) 32 | 33 | test('puppeteer-extra-plugin-anonymize-ua will allow a custom UA', async ({ 34 | browserName, 35 | extraLauncher, 36 | _browserOptions 37 | }) => { 38 | test.skip(browserName !== 'chromium', 'Chromium only') 39 | 40 | const pluginErrors = [] 41 | extraLauncher.plugins.onPluginError = (plugin, method, err) => { 42 | pluginErrors.push(err) 43 | } 44 | extraLauncher.use( 45 | AnonymizeUAPlugin({ 46 | customFn: ua => 'MyCoolUserAgent' 47 | }) 48 | ) 49 | expect(extraLauncher.plugins.list.length).toEqual(1) 50 | expect(extraLauncher.plugins.list[0].name).toEqual('anonymize-ua') 51 | 52 | const browser = await extraLauncher.launch(_browserOptions) 53 | const context = await browser.newContext() 54 | const page = await context.newPage() 55 | await page.goto('https://example.com') 56 | 57 | const ua = await page.evaluate(() => navigator.userAgent) 58 | expect(ua).toBe('MyCoolUserAgent') 59 | expect(pluginErrors).toStrictEqual([]) 60 | 61 | await browser.close() 62 | }) 63 | -------------------------------------------------------------------------------- /packages/playwright-extra/test/puppeteer-plugins/stealth.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from '../fixtures/extra' 2 | 3 | import StealthPlugin from 'puppeteer-extra-plugin-stealth' 4 | 5 | test('puppeteer-extra-plugin-stealth will work', async ({ 6 | browserName, 7 | extraLauncher, 8 | _browserOptions 9 | }) => { 10 | test.skip(browserName !== 'chromium', 'Chromium only') 11 | 12 | const pluginErrors = [] 13 | extraLauncher.plugins.onPluginError = (plugin, method, err) => { 14 | pluginErrors.push(err) 15 | } 16 | 17 | extraLauncher.use(StealthPlugin()) 18 | expect(extraLauncher.plugins.list.length).toEqual(1) 19 | expect(extraLauncher.plugins.list[0].name).toEqual('stealth') 20 | 21 | extraLauncher.plugins.setDependencyDefaults('stealth/evasions/webgl.vendor', { 22 | vendor: 'Bob', 23 | renderer: 'Alice' 24 | }) 25 | 26 | const browser = await extraLauncher.launch(_browserOptions) 27 | const context = await browser.newContext() 28 | const page = await context.newPage() 29 | await page.goto('https://example.com') 30 | 31 | const webgl = await page.evaluate(getWebglUnmasked) 32 | expect(webgl).toStrictEqual({ renderer: 'Alice', vendor: 'Bob' }) 33 | 34 | expect(pluginErrors).toStrictEqual([]) 35 | 36 | await browser.close() 37 | }) 38 | 39 | function getWebglUnmasked() { 40 | const gl = document.createElement('canvas').getContext('webgl') as any 41 | if (!gl) { 42 | return { 43 | error: 'no webgl' 44 | } 45 | } 46 | const debugInfo = gl.getExtension('WEBGL_debug_renderer_info') 47 | if (debugInfo) { 48 | return { 49 | vendor: gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL), 50 | renderer: gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) 51 | } 52 | } 53 | return { 54 | error: 'no WEBGL_debug_renderer_info' 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /packages/playwright-extra/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./dist", 4 | "target": "es2017", 5 | "module": "es2015", 6 | "moduleResolution": "node", 7 | "lib": ["es2015", "es2016", "es2017", "dom"], 8 | "sourceMap": true, 9 | "declaration": true, 10 | "allowSyntheticDefaultImports": true, 11 | "esModuleInterop": true, 12 | "emitDecoratorMetadata": true, 13 | "experimentalDecorators": true, 14 | "strict": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "noImplicitReturns": false, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": false, 19 | "pretty": true, 20 | "stripInternal": true, 21 | "types": ["node"] 22 | }, 23 | "include": ["./src/**/*.tsx", "./src/**/*.ts"], 24 | "exclude": ["node_modules", "dist", "./test/**/*.spec.ts"] 25 | } 26 | -------------------------------------------------------------------------------- /packages/plugin-proxy-router/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@extra/proxy-router", 3 | "version": "3.1.6", 4 | "description": "A plugin for playwright & puppeteer to route proxies dynamically.", 5 | "repository": "berstend/puppeteer-extra", 6 | "homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/plugin-proxy-router", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "main": "dist/index.cjs.js", 10 | "module": "dist/index.esm.js", 11 | "typings": "dist/index.d.ts", 12 | "files": [ 13 | "dist" 14 | ], 15 | "publishConfig": { 16 | "access": "public" 17 | }, 18 | "scripts": { 19 | "clean": "rimraf dist/*", 20 | "tscheck": "tsc --pretty --noEmit", 21 | "prebuild": "run-s clean", 22 | "build": "run-s build:tsc build:rollup", 23 | "build:tsc": "tsc --project tsconfig.json --module commonjs", 24 | "build:rollup": "rollup -c rollup.config.ts", 25 | "docs": "node -e 0", 26 | "test": "run-s build", 27 | "pretest-ci": "run-s build", 28 | "test-ci": "run-s build" 29 | }, 30 | "engines": { 31 | "node": ">=14" 32 | }, 33 | "prettier": { 34 | "printWidth": 80, 35 | "semi": false, 36 | "singleQuote": true 37 | }, 38 | "keywords": [ 39 | "puppeteer", 40 | "playwright", 41 | "puppeteer-extra", 42 | "playwright-extra", 43 | "proxy", 44 | "proxy-router", 45 | "headless", 46 | "luminati" 47 | ], 48 | "devDependencies": { 49 | "@types/debug": "^4.1.5", 50 | "@types/node": "14.17.6", 51 | "@types/puppeteer": "*", 52 | "ava": "2.4.0", 53 | "copyfiles": "^2.1.1", 54 | "npm-run-all": "^4.1.5", 55 | "playwright-core": "1.24.2", 56 | "prettier": "^2.7.1", 57 | "puppeteer": "^15.5.0", 58 | "puppeteer-extra": "^3.3.6", 59 | "replace-in-files-cli": "^0.3.1", 60 | "rimraf": "^3.0.0", 61 | "rollup-plugin-commonjs": "^10.1.0", 62 | "rollup-plugin-node-resolve": "^5.2.0", 63 | "rollup-plugin-sourcemaps": "^0.4.2", 64 | "rollup-plugin-typescript2": "^0.25.2", 65 | "ts-node": "^8.5.4", 66 | "typescript": "^4.7.4" 67 | }, 68 | "dependencies": { 69 | "debug": "^4.1.1", 70 | "merge-deep": "^3.0.2", 71 | "proxy-chain": "^2.0.6", 72 | "puppeteer-extra-plugin": "^3.2.3" 73 | }, 74 | "peerDependencies": { 75 | "playwright-extra": "*", 76 | "puppeteer-extra": "*" 77 | }, 78 | "peerDependenciesMeta": { 79 | "puppeteer-extra": { 80 | "optional": true 81 | }, 82 | "playwright-extra": { 83 | "optional": true 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /packages/plugin-proxy-router/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import resolve from 'rollup-plugin-node-resolve' 2 | import sourceMaps from 'rollup-plugin-sourcemaps' 3 | import typescript from 'rollup-plugin-typescript2' 4 | 5 | const pkg = require('./package.json') 6 | 7 | const entryFile = 'index' 8 | const banner = ` 9 | /*! 10 | * ${pkg.name} v${pkg.version} by ${pkg.author} 11 | * ${pkg.homepage || `https://github.com/${pkg.repository}`} 12 | * @license ${pkg.license} 13 | */ 14 | `.trim() 15 | 16 | const defaultExportOutro = ` 17 | module.exports = exports.default || {} 18 | Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value }) 19 | ` 20 | 21 | export default { 22 | input: `src/${entryFile}.ts`, 23 | output: [ 24 | { 25 | file: pkg.main, 26 | format: 'cjs', 27 | sourcemap: true, 28 | exports: 'named', 29 | outro: defaultExportOutro, 30 | banner 31 | }, 32 | { 33 | file: pkg.module, 34 | format: 'es', 35 | sourcemap: true, 36 | exports: 'named', 37 | banner 38 | } 39 | ], 40 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 41 | external: [ 42 | ...Object.keys(pkg.dependencies || {}), 43 | ...Object.keys(pkg.peerDependencies || {}) 44 | ], 45 | watch: { 46 | include: 'src/**' 47 | }, 48 | plugins: [ 49 | // Compile TypeScript files 50 | typescript({ useTsconfigDeclarationDir: true }), 51 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 52 | // commonjs(), 53 | // Allow node_modules resolution, so you can use 'external' to control 54 | // which external modules to include in the bundle 55 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 56 | resolve(), 57 | // Resolve source maps to the original source 58 | sourceMaps() 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /packages/plugin-proxy-router/src/index.ts: -------------------------------------------------------------------------------- 1 | import { ExtraPluginProxyRouter, ExtraPluginProxyRouterOptions } from './plugin' 2 | 3 | export * from './plugin' 4 | export * from './router' 5 | export * from './stats' 6 | 7 | /** Default export, ExtraPluginProxyRouter */ 8 | const defaultExport = (options?: Partial) => { 9 | return new ExtraPluginProxyRouter(options || {}) 10 | } 11 | 12 | export default defaultExport 13 | -------------------------------------------------------------------------------- /packages/plugin-proxy-router/src/utils/port.ts: -------------------------------------------------------------------------------- 1 | import net from 'net' 2 | 3 | export interface Options { 4 | /** 5 | * A preferred port or an array of preferred ports to use. 6 | */ 7 | port?: number | ReadonlyArray 8 | 9 | /** 10 | * The host on which port resolution should be performed. Can be either an IPv4 or IPv6 address. 11 | */ 12 | host?: string 13 | } 14 | 15 | const isAvailable = (options: Options): Promise => 16 | new Promise((resolve, reject) => { 17 | const server = net.createServer() 18 | server.unref() 19 | server.on('error', reject) 20 | server.listen(options, () => { 21 | const { port } = server.address() as any 22 | server.close(() => { 23 | resolve(port as number) 24 | }) 25 | }) 26 | }) 27 | 28 | const getPort = (options: Options) => { 29 | options = Object.assign({}, options) 30 | 31 | if (typeof options.port === 'number') { 32 | options.port = [options.port] 33 | } 34 | 35 | return (options.port || []).reduce( 36 | (seq, port) => 37 | seq.catch(() => isAvailable(Object.assign({}, options, { port }))), 38 | Promise.reject() 39 | ) 40 | } 41 | 42 | export default (options?: Options) => 43 | options 44 | ? getPort(options).catch(() => getPort(Object.assign(options, { port: 0 }))) 45 | : getPort({ port: 0 }) 46 | -------------------------------------------------------------------------------- /packages/plugin-proxy-router/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./dist", 4 | "target": "es2017", 5 | "module": "es2015", 6 | "moduleResolution": "node", 7 | "lib": ["es2015", "es2016", "es2017", "es2019", "dom"], 8 | // "noResolve": true, // Important: Otherwise TS would rewrite our ambient d.ts file locations (see: yarn copy-dts) :( 9 | "sourceMap": true, 10 | "declaration": true, 11 | "allowSyntheticDefaultImports": true, 12 | "esModuleInterop": true, 13 | "emitDecoratorMetadata": true, 14 | "experimentalDecorators": true, 15 | "strict": false, 16 | "noFallthroughCasesInSwitch": true, 17 | "noImplicitReturns": false, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": false, 20 | "pretty": true, 21 | "stripInternal": true, 22 | "types": ["node"] 23 | }, 24 | "include": [ 25 | "./src/**/*.tsx", 26 | "./src/**/*.ts", 27 | "./src/**/*.d.ts", 28 | "./src/**/*.test.ts", 29 | "./test/**/*.ts" 30 | ], 31 | "exclude": ["node_modules", "dist", "./test/**/*.spec.ts"] 32 | } 33 | -------------------------------------------------------------------------------- /packages/plugin-proxy-router/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint-config-standard", "tslint-config-prettier"], 3 | "rules": { 4 | "ordered-imports": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/ava.config-ts.js: -------------------------------------------------------------------------------- 1 | export default { 2 | compileEnhancements: false, 3 | environmentVariables: { 4 | TS_NODE_COMPILER_OPTIONS: '{"module":"commonjs"}' 5 | }, 6 | files: ['src/**.test.ts'], 7 | extensions: ['ts'], 8 | require: ['ts-node/register'] 9 | } 10 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/ava.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | files: ['dist/*.test.js'] 3 | } 4 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/build_version_check.js: -------------------------------------------------------------------------------- 1 | const pkg = require('./package.json') 2 | 3 | const isIncompatiblePuppeteerVersion = () => { 4 | const version = pkg.devDependencies.puppeteer 5 | const majorVersion = parseInt(version.split('.')[0]) 6 | if (majorVersion >= 6) { 7 | return true 8 | } else { 9 | return false 10 | } 11 | } 12 | 13 | const incompatible = isIncompatiblePuppeteerVersion() 14 | if (incompatible) { 15 | console.warn( 16 | 'ERR: The adblocker plugin requires pptr >= 6', 17 | process.env.PUPPETEER_VERSION 18 | ) 19 | } 20 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import resolve from 'rollup-plugin-node-resolve' 2 | import sourceMaps from 'rollup-plugin-sourcemaps' 3 | import typescript from 'rollup-plugin-typescript2' 4 | 5 | const pkg = require('./package.json') 6 | 7 | const entryFile = 'index' 8 | const banner = ` 9 | /*! 10 | * ${pkg.name} v${pkg.version} by ${pkg.author} 11 | * ${pkg.homepage || `https://github.com/${pkg.repository}`} 12 | * @license ${pkg.license} 13 | */ 14 | `.trim() 15 | 16 | const defaultExportOutro = ` 17 | module.exports = exports.default || {} 18 | Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value }) 19 | ` 20 | 21 | export default { 22 | input: `src/${entryFile}.ts`, 23 | output: [ 24 | { 25 | file: pkg.main, 26 | format: 'cjs', 27 | sourcemap: true, 28 | exports: 'named', 29 | outro: defaultExportOutro, 30 | banner 31 | }, 32 | { 33 | file: pkg.module, 34 | format: 'es', 35 | sourcemap: true, 36 | exports: 'named', 37 | banner 38 | } 39 | ], 40 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 41 | external: [ 42 | ...Object.keys(pkg.dependencies || {}), 43 | ...Object.keys(pkg.peerDependencies || {}), 44 | 'fs', 45 | 'os', 46 | 'path' 47 | ], 48 | watch: { 49 | include: 'src/**' 50 | }, 51 | plugins: [ 52 | // Compile TypeScript files 53 | typescript({ useTsconfigDeclarationDir: true }), 54 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 55 | // commonjs(), 56 | // Allow node_modules resolution, so you can use 'external' to control 57 | // which external modules to include in the bundle 58 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 59 | resolve({ 60 | preferBuiltins: true 61 | }), 62 | // Resolve source maps to the original source 63 | sourceMaps() 64 | ] 65 | } 66 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/src/ambient.d.ts: -------------------------------------------------------------------------------- 1 | export {} 2 | 3 | // https://github.com/sindresorhus/type-fest/issues/19 4 | declare global { 5 | interface SymbolConstructor { 6 | readonly observable: symbol 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/src/index.test.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import AdblockerPlugin from './index' 4 | 5 | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'] 6 | 7 | test('will block ads', async t => { 8 | const puppeteer = require('puppeteer-extra') 9 | const adblockerPlugin = AdblockerPlugin({ 10 | blockTrackers: true 11 | }) 12 | puppeteer.use(adblockerPlugin) 13 | 14 | const browser = await puppeteer.launch({ 15 | args: PUPPETEER_ARGS, 16 | headless: true 17 | }) 18 | 19 | const blocker = await adblockerPlugin.getBlocker() 20 | 21 | const page = await browser.newPage() 22 | 23 | let blockedRequests = 0 24 | blocker.on('request-blocked', () => { 25 | blockedRequests += 1 26 | }) 27 | 28 | let hiddenAds = 0 29 | blocker.on('style-injected', () => { 30 | hiddenAds += 1 31 | }) 32 | 33 | const url = 'https://www.google.com/search?q=rent%20a%20car' 34 | await page.goto(url, { waitUntil: 'networkidle0' }) 35 | 36 | t.not(hiddenAds, 0) 37 | t.not(blockedRequests, 0) 38 | 39 | await browser.close() 40 | }) 41 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./dist", 4 | "target": "es2017", 5 | "module": "es2015", 6 | "moduleResolution": "node", 7 | "lib": ["es2015", "es2016", "es2017", "dom"], 8 | "sourceMap": true, 9 | "declaration": true, 10 | "allowSyntheticDefaultImports": true, 11 | "esModuleInterop": true, 12 | "emitDecoratorMetadata": true, 13 | "experimentalDecorators": true, 14 | "strict": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "noImplicitReturns": false, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": false, 19 | "pretty": true, 20 | "stripInternal": true, 21 | "types": ["node"] 22 | }, 23 | "include": [ 24 | "./src/**/*.tsx", 25 | "./src/**/*.ts", 26 | "./src/**/*.test.ts", 27 | "./test/**/*.ts" 28 | ], 29 | "exclude": ["node_modules", "dist", "./test/**/*.spec.ts"] 30 | } 31 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-adblocker/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint-config-standard", "tslint-config-prettier"], 3 | "rules": { 4 | "ordered-imports": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-anonymize-ua/index.d.ts: -------------------------------------------------------------------------------- 1 | declare const PuppeteerExtraPlugin: typeof import("puppeteer-extra-plugin").PuppeteerExtraPlugin; 2 | declare const Page: typeof import("puppeteer").Page; 3 | type CustomFn = ((ua: string) => string | null) | null; 4 | declare class Plugin extends PuppeteerExtraPlugin { 5 | get name(): string; 6 | get defaults(): { 7 | stripHeadless: boolean; 8 | makeWindows: boolean; 9 | customFn: CustomFn; 10 | }; 11 | async onPageCreated(page: Page): void; 12 | } 13 | export default function (options?: { 14 | stripHeadless?: true; 15 | makeWindows?: true; 16 | customFn?: CustomFn; 17 | }): Plugin; 18 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-anonymize-ua/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | /** 6 | * Anonymize the User-Agent on all pages. 7 | * 8 | * Supports dynamic replacing, so the Chrome version stays intact and recent. 9 | * 10 | * @param {Object} opts - Options 11 | * @param {boolean} [opts.stripHeadless=true] - Replace `HeadlessChrome` with `Chrome`. 12 | * @param {boolean} [opts.makeWindows=true] - Sets the platform to Windows 10, 64bit (most common). 13 | * @param {Function} [opts.customFn=null] - A custom UA replacer function. 14 | * 15 | * @example 16 | * const puppeteer = require('puppeteer-extra') 17 | * puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')()) 18 | * // or 19 | * puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')({ 20 | * customFn: (ua) => 'MyCoolAgent/' + ua.replace('Chrome', 'Beer')}) 21 | * ) 22 | * const browser = await puppeteer.launch() 23 | */ 24 | class Plugin extends PuppeteerExtraPlugin { 25 | constructor(opts = {}) { 26 | super(opts) 27 | } 28 | 29 | get name() { 30 | return 'anonymize-ua' 31 | } 32 | 33 | get defaults() { 34 | return { 35 | stripHeadless: true, 36 | makeWindows: true, 37 | customFn: null 38 | } 39 | } 40 | 41 | async onPageCreated(page) { 42 | let ua = await page.browser().userAgent() 43 | if (this.opts.stripHeadless) { 44 | ua = ua.replace('HeadlessChrome/', 'Chrome/') 45 | } 46 | if (this.opts.makeWindows) { 47 | ua = ua.replace(/\(([^)]+)\)/, '(Windows NT 10.0; Win64; x64)') 48 | } 49 | if (this.opts.customFn) { 50 | ua = this.opts.customFn(ua) 51 | } 52 | this.debug('new ua', ua) 53 | await page.setUserAgent(ua) 54 | } 55 | } 56 | 57 | module.exports = function(pluginConfig) { 58 | return new Plugin(pluginConfig) 59 | } 60 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-anonymize-ua/index.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const PLUGIN_NAME = 'anonymize-ua' 4 | 5 | const test = require('ava') 6 | 7 | const Plugin = require('.') 8 | 9 | test('is a function', async t => { 10 | t.is(typeof Plugin, 'function') 11 | }) 12 | 13 | test('should have the basic class members', async t => { 14 | const instance = new Plugin() 15 | 16 | t.is(instance.name, PLUGIN_NAME) 17 | t.true(instance._isPuppeteerExtraPlugin) 18 | }) 19 | 20 | test('should have the public child class members', async t => { 21 | const instance = new Plugin() 22 | const prototype = Object.getPrototypeOf(instance) 23 | const childClassMembers = Object.getOwnPropertyNames(prototype) 24 | 25 | t.true(childClassMembers.includes('constructor')) 26 | t.true(childClassMembers.includes('name')) 27 | t.true(childClassMembers.includes('defaults')) 28 | t.true(childClassMembers.includes('onPageCreated')) 29 | t.true(childClassMembers.length === 4) 30 | }) 31 | 32 | test('should have opts with default values', async t => { 33 | const instance = new Plugin() 34 | const opts = instance.opts 35 | 36 | t.is(opts.stripHeadless, true) 37 | t.is(opts.makeWindows, true) 38 | t.is(opts.customFn, null) 39 | }) 40 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-anonymize-ua/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-anonymize-ua", 3 | "version": "2.4.6", 4 | "description": "Anonymize User-Agent in puppeteer.", 5 | "main": "index.js", 6 | "types": "index.d.ts", 7 | "repository": "berstend/puppeteer-extra", 8 | "author": "berstend", 9 | "license": "MIT", 10 | "scripts": { 11 | "docs": "node -e 0", 12 | "lint": "eslint --ext .js .", 13 | "test:js": "ava -v --serial --concurrency 1 --fail-fast", 14 | "test": "run-p test:js lint", 15 | "test-ci": "run-s test" 16 | }, 17 | "engines": { 18 | "node": ">=8" 19 | }, 20 | "keywords": [ 21 | "puppeteer", 22 | "puppeteer-extra", 23 | "puppeteer-extra-plugin", 24 | "ua", 25 | "user-agent", 26 | "chrome", 27 | "headless", 28 | "pupeteer" 29 | ], 30 | "devDependencies": { 31 | "ava": "2.4.0", 32 | "npm-run-all": "^4.1.5", 33 | "puppeteer": "^2.0.0", 34 | "puppeteer-extra": "^3.3.6", 35 | "puppeteer-extra-plugin-devtools": "^2.4.6" 36 | }, 37 | "dependencies": { 38 | "debug": "^4.1.1", 39 | "puppeteer-extra-plugin": "^3.2.3" 40 | }, 41 | "peerDependencies": { 42 | "playwright-extra": "*", 43 | "puppeteer-extra": "*" 44 | }, 45 | "peerDependenciesMeta": { 46 | "puppeteer-extra": { 47 | "optional": true 48 | }, 49 | "playwright-extra": { 50 | "optional": true 51 | } 52 | }, 53 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 54 | } 55 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-anonymize-ua/readme.md: -------------------------------------------------------------------------------- 1 | # puppeteer-extra-plugin-anonymize-ua 2 | 3 | > A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra). 4 | 5 | ### Install 6 | 7 | ```bash 8 | yarn add puppeteer-extra-plugin-anonymize-ua 9 | ``` 10 | 11 | ## API 12 | 13 | 14 | 15 | #### Table of Contents 16 | 17 | - [Plugin](#plugin) 18 | 19 | ### [Plugin](https://github.com/berstend/puppeteer-extra/blob/db57ea66cf10d407cf63af387892492e495a84f2/packages/puppeteer-extra-plugin-anonymize-ua/index.js#L24-L51) 20 | 21 | **Extends: PuppeteerExtraPlugin** 22 | 23 | Anonymize the User-Agent on all pages. 24 | 25 | Supports dynamic replacing, so the Chrome version stays intact and recent. 26 | 27 | Type: `function (opts)` 28 | 29 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** Options (optional, default `{}`) 30 | - `opts.stripHeadless` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Replace `HeadlessChrome` with `Chrome`. (optional, default `true`) 31 | - `opts.makeWindows` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Sets the platform to Windows 10, 64bit (most common). (optional, default `true`) 32 | - `opts.customFn` **[Function](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function)** A custom UA replacer function. (optional, default `null`) 33 | 34 | Example: 35 | 36 | ```javascript 37 | const puppeteer = require('puppeteer-extra') 38 | puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')()) 39 | // or 40 | puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')({ 41 | customFn: (ua) => 'MyCoolAgent/' + ua.replace('Chrome', 'Beer')}) 42 | ) 43 | const browser = await puppeteer.launch() 44 | ``` 45 | 46 | * * * 47 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-anonymize-ua/test/headless_off.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'] 6 | 7 | test.beforeEach(t => { 8 | // Make sure we work with pristine modules 9 | delete require.cache[require.resolve('puppeteer-extra')] 10 | delete require.cache[require.resolve('puppeteer-extra-plugin-anonymize-ua')] 11 | }) 12 | 13 | test('will not modify the user-agent when disabled', async t => { 14 | const puppeteer = require('puppeteer-extra') 15 | const AnonymizeUA = require('puppeteer-extra-plugin-anonymize-ua')({ 16 | stripHeadless: false, 17 | makeWindows: false, 18 | customFn: null 19 | }) 20 | puppeteer.use(AnonymizeUA) 21 | 22 | const browser = await puppeteer.launch({ args: PUPPETEER_ARGS }) 23 | const page = await browser.newPage() 24 | await page.goto('https://httpbin.org/headers', { 25 | waitUntil: 'domcontentloaded' 26 | }) 27 | 28 | const content = await page.content() 29 | t.true(content.includes('HeadlessChrome')) 30 | t.true(!content.includes('MyCoolAgent/Mozilla')) 31 | t.true(!content.includes('Beer/')) 32 | 33 | await browser.close() 34 | t.true(true) 35 | }) 36 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-anonymize-ua/test/popup.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'] 6 | 7 | const waitEvent = function(emitter, eventName) { 8 | return new Promise(resolve => emitter.once(eventName, resolve)) 9 | } 10 | 11 | test.beforeEach(t => { 12 | // Make sure we work with pristine modules 13 | delete require.cache[require.resolve('puppeteer-extra')] 14 | delete require.cache[require.resolve('puppeteer-extra-plugin-anonymize-ua')] 15 | }) 16 | 17 | test('known issue: will not remove headless from implicitly created popup pages', async t => { 18 | const puppeteer = require('puppeteer-extra') 19 | puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')()) 20 | const browser = await puppeteer.launch({ args: PUPPETEER_ARGS }) 21 | 22 | const pages = await Promise.all([...Array(10)].map(slot => browser.newPage())) 23 | for (const page of pages) { 24 | // Works 25 | const ua = await page.evaluate(() => window.navigator.userAgent) 26 | t.true(!ua.includes('HeadlessChrome')) 27 | 28 | // Works 29 | await page.goto('about:blank') 30 | const ua2 = await page.evaluate(() => window.navigator.userAgent) 31 | t.true(!ua2.includes('HeadlessChrome')) 32 | 33 | // Does NOT work: 34 | // https://github.com/GoogleChrome/puppeteer/issues/2669 35 | page.evaluate(url => window.open(url), 'about:blank') 36 | const popupTarget = await waitEvent(browser, 'targetcreated') 37 | const popupPage = await popupTarget.page() 38 | const ua3 = await popupPage.evaluate(() => window.navigator.userAgent) 39 | // Test against the problem until it's fixed 40 | t.true(ua3.includes('HeadlessChrome')) // should be: !ua3.includes('HeadlessChrome') 41 | 42 | // Works: The bug only affects newly created popups, subsequent page navigations are fine. 43 | await popupPage.goto('about:blank') 44 | const ua4 = await page.evaluate(() => window.navigator.userAgent) 45 | t.true(!ua4.includes('HeadlessChrome')) 46 | } 47 | 48 | await browser.close() 49 | t.true(true) 50 | }) 51 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-block-resources/example.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // 4 | // With debug logs: 5 | // DEBUG=puppeteer-extra,puppeteer-extra-plugin,puppeteer-extra-plugin:* node example.js 6 | // 7 | 8 | // const puppeteer = require('puppeteer-extra') 9 | // puppeteer.use(require('puppeteer-extra-plugin-block-resources')({ 10 | // blockedTypes: new Set(['image', 'stylesheet']) 11 | // })) 12 | // ;(async () => { 13 | // const browser = await puppeteer.launch({ headless: false }) 14 | // const page = await browser.newPage() 15 | // await page.goto('http://www.msn.com/', {waitUntil: 'domcontentloaded'}) 16 | // console.log('all done') 17 | // })() 18 | 19 | const { DEFAULT_INTERCEPT_RESOLUTION_PRIORITY } = require('puppeteer') 20 | const puppeteer = require('puppeteer-extra') 21 | const blockResourcesPlugin = require('puppeteer-extra-plugin-block-resources')({ 22 | // Optionally enable Cooperative Mode for several request interceptors 23 | interceptResolutionPriority: DEFAULT_INTERCEPT_RESOLUTION_PRIORITY 24 | }) 25 | 26 | puppeteer.use(blockResourcesPlugin) 27 | ;(async () => { 28 | const browser = await puppeteer.launch({ headless: false }) 29 | const page = await browser.newPage() 30 | 31 | blockResourcesPlugin.blockedTypes.add('image') 32 | await page.goto('http://www.msn.com/', { waitUntil: 'domcontentloaded' }) 33 | 34 | blockResourcesPlugin.blockedTypes.add('stylesheet') 35 | blockResourcesPlugin.blockedTypes.add('other') // e.g. favicon 36 | await page.goto('http://news.ycombinator.com', { 37 | waitUntil: 'domcontentloaded' 38 | }) 39 | 40 | blockResourcesPlugin.blockedTypes.delete('stylesheet') 41 | blockResourcesPlugin.blockedTypes.delete('other') 42 | blockResourcesPlugin.blockedTypes.add('media') 43 | blockResourcesPlugin.blockedTypes.add('script') 44 | await page.goto('http://www.youtube.com', { waitUntil: 'domcontentloaded' }) 45 | 46 | console.log('all done') 47 | })() 48 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-block-resources/index.d.ts: -------------------------------------------------------------------------------- 1 | import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin' 2 | import { ResourceType } from 'puppeteer' 3 | 4 | declare interface PluginOptions { 5 | availableTypes?: Set 6 | blockedTypes?: Set 7 | interceptResolutionPriority?: number 8 | } 9 | 10 | declare class Plugin extends PuppeteerExtraPlugin { 11 | constructor(opts: Partial) 12 | get name(): string 13 | get defaults(): PluginOptions 14 | get engineCacheFile(): string 15 | get availableTypes(): Set 16 | get blockedTypes(): Set 17 | get interceptResolutionPriority(): number 18 | } 19 | 20 | declare const _default: (options?: Partial) => Plugin 21 | 22 | export default _default 23 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-block-resources/index.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const PLUGIN_NAME = 'block-resources' 4 | 5 | const test = require('ava') 6 | 7 | const Plugin = require('.') 8 | 9 | test('is a function', async t => { 10 | t.is(typeof Plugin, 'function') 11 | }) 12 | 13 | test('should have the basic class members', async t => { 14 | const instance = new Plugin() 15 | t.is(instance.name, PLUGIN_NAME) 16 | t.true(instance._isPuppeteerExtraPlugin) 17 | }) 18 | 19 | test('should have the public child class members', async t => { 20 | const instance = new Plugin() 21 | const prototype = Object.getPrototypeOf(instance) 22 | const childClassMembers = Object.getOwnPropertyNames(prototype) 23 | 24 | t.true(childClassMembers.includes('constructor')) 25 | t.true(childClassMembers.includes('name')) 26 | t.true(childClassMembers.includes('defaults')) 27 | t.true(childClassMembers.includes('availableTypes')) 28 | t.true(childClassMembers.includes('blockedTypes')) 29 | t.true(childClassMembers.includes('interceptResolutionPriority')) 30 | t.true(childClassMembers.includes('onRequest')) 31 | t.true(childClassMembers.includes('onPageCreated')) 32 | t.true(childClassMembers.length === 8) 33 | }) 34 | 35 | test('should have opts with default values', async t => { 36 | const instance = new Plugin() 37 | t.deepEqual(instance.opts.blockedTypes, new Set([])) 38 | t.is(instance.opts.availableTypes.size, 13) 39 | t.is(instance.opts.interceptResolutionPriority, undefined) 40 | }) 41 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-block-resources/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-block-resources", 3 | "version": "2.4.3", 4 | "description": "Block resources (images, media, etc.) in puppeteer.", 5 | "main": "index.js", 6 | "types": "index.d.ts", 7 | "repository": "berstend/puppeteer-extra", 8 | "author": "berstend", 9 | "license": "MIT", 10 | "scripts": { 11 | "docs": "node -e 0", 12 | "lint": "eslint --ext .js .", 13 | "test:js": "ava -v", 14 | "test": "run-p test:js lint", 15 | "test-ci": "run-s test" 16 | }, 17 | "engines": { 18 | "node": ">=8" 19 | }, 20 | "keywords": [ 21 | "puppeteer", 22 | "puppeteer-extra", 23 | "puppeteer-extra-plugin", 24 | "block-resources", 25 | "datasaver", 26 | "chrome", 27 | "headless", 28 | "pupeteer" 29 | ], 30 | "devDependencies": { 31 | "ava": "2.4.0", 32 | "npm-run-all": "^4.1.5", 33 | "puppeteer": "^2.0.0" 34 | }, 35 | "dependencies": { 36 | "debug": "^4.1.1", 37 | "puppeteer-extra-plugin": "^3.2.3" 38 | }, 39 | "peerDependencies": { 40 | "puppeteer-extra": "*" 41 | }, 42 | "peerDependenciesMeta": { 43 | "puppeteer-extra": { 44 | "optional": true 45 | } 46 | }, 47 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 48 | } 49 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-click-and-wait/example.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const puppeteer = require('puppeteer-extra') 4 | puppeteer.use(require('puppeteer-extra-plugin-click-and-wait')()) 5 | ;(async () => { 6 | const browser = await puppeteer.launch({ headless: false }) 7 | const page = await browser.newPage() 8 | await page.goto('https://example.com/', { waitUntil: 'domcontentloaded' }) 9 | console.log('clicking on first link') 10 | await page.clickAndWaitForNavigation('a') 11 | console.log('all done') 12 | })() 13 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-click-and-wait/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | /** 6 | * Convenience function to wait for navigation to complete after clicking on an element. 7 | * 8 | * Adds a new `page.clickAndWaitForNavigation(selector, clickOptions, waitOptions)` method. 9 | * 10 | * See this issue for more context: https://github.com/GoogleChrome/puppeteer/issues/1421 11 | * 12 | * > Note: Be wary of ajax powered pages where the navigation event is not triggered. 13 | * 14 | * @example 15 | * await page.clickAndWaitForNavigation('input#submitData') 16 | * 17 | * // as opposed to: 18 | * 19 | * await Promise.all([ 20 | * page.waitForNavigation(waitOptions), 21 | * page.click('input#submitData', clickOptions), 22 | * ]) 23 | */ 24 | class Plugin extends PuppeteerExtraPlugin { 25 | constructor(opts = {}) { 26 | super(opts) 27 | } 28 | 29 | get name() { 30 | return 'click-and-wait' 31 | } 32 | 33 | async clickAndWaitForNavigation(selector, clickOptions, waitOptions) { 34 | return Promise.all([ 35 | this.waitForNavigation(waitOptions), 36 | this.click(selector, clickOptions) 37 | ]).then(values => { 38 | return values[0] 39 | }) 40 | } 41 | 42 | async onPageCreated(page) { 43 | page.clickAndWaitForNavigation = this.clickAndWaitForNavigation.bind(page) 44 | } 45 | } 46 | 47 | module.exports = function(pluginConfig) { 48 | return new Plugin(pluginConfig) 49 | } 50 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-click-and-wait/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-click-and-wait", 3 | "version": "2.3.3", 4 | "description": "Convenience function to wait for navigation to complete after clicking on an element.", 5 | "main": "index.js", 6 | "repository": "berstend/puppeteer-extra", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "scripts": { 10 | "docs": "node -e 0", 11 | "lint": "eslint --ext .js .", 12 | "test": "run-p lint", 13 | "test-ci": "run-s test" 14 | }, 15 | "engines": { 16 | "node": ">=8" 17 | }, 18 | "keywords": [ 19 | "puppeteer", 20 | "puppeteer-extra", 21 | "puppeteer-extra-plugin", 22 | "clickAndWaitForNavigation", 23 | "chrome", 24 | "headless", 25 | "pupeteer" 26 | ], 27 | "devDependencies": { 28 | "ava": "2.4.0", 29 | "npm-run-all": "^4.1.5", 30 | "puppeteer": "^2.0.0" 31 | }, 32 | "dependencies": { 33 | "debug": "^4.1.1", 34 | "puppeteer-extra-plugin": "^3.2.3" 35 | }, 36 | "peerDependencies": { 37 | "playwright-extra": "*", 38 | "puppeteer-extra": "*" 39 | }, 40 | "peerDependenciesMeta": { 41 | "puppeteer-extra": { 42 | "optional": true 43 | }, 44 | "playwright-extra": { 45 | "optional": true 46 | } 47 | }, 48 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 49 | } 50 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-click-and-wait/readme.md: -------------------------------------------------------------------------------- 1 | # puppeteer-extra-plugin-click-and-wait 2 | 3 | > A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra). 4 | 5 | ### Install 6 | 7 | ```bash 8 | yarn add puppeteer-extra-plugin-click-and-wait 9 | ``` 10 | 11 | ## API 12 | 13 | 14 | 15 | #### Table of Contents 16 | 17 | - [Plugin](#plugin) 18 | 19 | ### [Plugin](https://github.com/berstend/puppeteer-extra/blob/db57ea66cf10d407cf63af387892492e495a84f2/packages/puppeteer-extra-plugin-click-and-wait/index.js#L24-L39) 20 | 21 | **Extends: PuppeteerExtraPlugin** 22 | 23 | Convenience function to wait for navigation to complete after clicking on an element. 24 | 25 | Adds a new `page.clickAndWaitForNavigation(selector, clickOptions, waitOptions)` method. 26 | 27 | See this issue for more context: 28 | 29 | > Note: Be wary of ajax powered pages where the navigation event is not triggered. 30 | 31 | Type: `function (opts)` 32 | 33 | - `opts` (optional, default `{}`) 34 | 35 | Example: 36 | 37 | ```javascript 38 | await page.clickAndWaitForNavigation('input#submitData') 39 | 40 | // as opposed to: 41 | 42 | await Promise.all([ 43 | page.waitForNavigation(waitOptions), 44 | page.click('input#submitData', clickOptions), 45 | ]) 46 | ``` 47 | 48 | * * * 49 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-devtools/index.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const PLUGIN_NAME = 'devtools' 4 | 5 | const test = require('ava') 6 | 7 | const Plugin = require('.') 8 | 9 | test('is a function', async t => { 10 | t.is(typeof Plugin, 'function') 11 | }) 12 | 13 | test('should have the basic class members', async t => { 14 | const instance = new Plugin() 15 | t.is(instance.name, PLUGIN_NAME) 16 | t.true(instance._isPuppeteerExtraPlugin) 17 | }) 18 | 19 | test('should have opts with default values', async t => { 20 | const instance = new Plugin() 21 | t.is(instance.opts.prefix, 'devtools-tunnel') 22 | t.is(instance.opts.auth.user, 'user') 23 | t.is(instance.opts.auth.pass.length, 40) 24 | }) 25 | 26 | test('will throw without browser when creating a tunnel', async t => { 27 | const instance = new Plugin() 28 | let error = null 29 | try { 30 | await instance.createTunnel() 31 | } catch (err) { 32 | error = err 33 | } 34 | t.is(error.name, `ArgumentError`) 35 | }) 36 | 37 | // test('will accept a browser when creating a tunnel', async t => { 38 | // const instance = new Plugin({ auth: { user: 'bob', pass: 'yup' } }) 39 | // const fakeBrowser = { wsEndpoint: () => 'ws://foobar:1337' } 40 | // await instance.createTunnel(fakeBrowser) 41 | // t.is(true, true) 42 | // }) 43 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-devtools/lib/RemoteDevTools.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | const { 6 | DevToolsCommon, 7 | DevToolsLocal, 8 | DevToolsTunnel 9 | } = require('./RemoteDevTools') 10 | 11 | const webSocketDebuggerUrl = 12 | 'ws://127.0.0.1:9222/devtools/browser/ec78d039-2f19-4c6f-a08e-bcaf88e34b69' 13 | 14 | test('is a function', async t => { 15 | t.is(typeof DevToolsCommon, 'function') 16 | t.is(typeof DevToolsLocal, 'function') 17 | t.is(typeof DevToolsTunnel, 'function') 18 | }) 19 | 20 | test('will throw when missing webSocketDebuggerUrl', async t => { 21 | const error = await t.throws(() => new DevToolsCommon()) 22 | t.is( 23 | error.message, 24 | 'Expected argument to be of type `string` but received type `undefined`' 25 | ) // eslint-disable-line 26 | }) 27 | 28 | test('DevToolsLocal: has basic functionality', async t => { 29 | const instance = new DevToolsLocal(webSocketDebuggerUrl) 30 | t.is(instance.url, 'http://localhost:9222') 31 | t.is( 32 | instance.getUrlForPageId('foobar'), 33 | 'http://localhost:9222/devtools/inspector.html?ws=localhost:9222/devtools/page/foobar' 34 | ) 35 | }) 36 | 37 | test('DevToolsTunnel: has basic functionality', async t => { 38 | const instance = new DevToolsTunnel(webSocketDebuggerUrl) 39 | instance.tunnel = { url: 'https://faketunnel.com' } 40 | instance.tunnelHost = 'faketunnel.com' 41 | t.is(instance.url, instance.tunnel.url) 42 | t.is( 43 | instance.getUrlForPageId('foobar'), 44 | 'https://faketunnel.com/devtools/inspector.html?wss=faketunnel.com/devtools/page/foobar' 45 | ) 46 | }) 47 | 48 | test('DevToolsTunnel: has defaults', async t => { 49 | const instance = new DevToolsTunnel(webSocketDebuggerUrl) 50 | 51 | t.is(instance.opts.prefix, 'devtools-tunnel') 52 | t.is(instance.opts.subdomain, null) 53 | t.deepEqual(instance.opts.auth, { user: null, pass: null }) 54 | }) 55 | 56 | test('DevToolsTunnel: has public members', async t => { 57 | const instance = new DevToolsTunnel(webSocketDebuggerUrl) 58 | 59 | t.true(instance.create instanceof Function) 60 | t.true(instance.close instanceof Function) 61 | }) 62 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-devtools/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-devtools", 3 | "version": "2.4.6", 4 | "description": "Make puppeteer browser debugging possible from anywhere (devtools with screencasting on the internet).", 5 | "main": "index.js", 6 | "repository": "berstend/puppeteer-extra", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "scripts": { 10 | "docs": "node -e 0", 11 | "lint": "eslint --ext .js .", 12 | "test-ava": "ava --fail-fast -v", 13 | "test": "run-p lint test-ava", 14 | "test-ci": "run-s test" 15 | }, 16 | "engines": { 17 | "node": ">=8" 18 | }, 19 | "keywords": [ 20 | "puppeteer", 21 | "puppeteer-extra", 22 | "puppeteer-extra-plugin", 23 | "devtools", 24 | "devtools-tunnel", 25 | "localtunnel", 26 | "remote-debugging", 27 | "chrome", 28 | "headless", 29 | "pupeteer" 30 | ], 31 | "devDependencies": { 32 | "ava": "2.4.0", 33 | "npm-run-all": "^4.1.5", 34 | "puppeteer": "^2.0.0", 35 | "puppeteer-extra": "^3.3.6" 36 | }, 37 | "dependencies": { 38 | "debug": "^4.1.1", 39 | "get-port": "^3.2.0", 40 | "got": "^8.3.1", 41 | "http-auth": "^3.2.3", 42 | "http-proxy": "^1.17.0", 43 | "http-proxy-response-rewrite": "^0.0.1", 44 | "localtunnel": "^2.0.0", 45 | "ow": "^0.4.0", 46 | "puppeteer-extra-plugin": "^3.2.3", 47 | "randomstring": "^1.1.5", 48 | "url-parse": "^1.5.3" 49 | }, 50 | "peerDependencies": { 51 | "playwright-extra": "*", 52 | "puppeteer-extra": "*" 53 | }, 54 | "peerDependenciesMeta": { 55 | "puppeteer-extra": { 56 | "optional": true 57 | }, 58 | "playwright-extra": { 59 | "optional": true 60 | } 61 | }, 62 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 63 | } 64 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-devtools/test/headless.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | // const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'] 6 | 7 | // test.beforeEach(t => { 8 | // // Make sure we work with pristine modules 9 | // delete require.cache[require.resolve('puppeteer-extra')] 10 | // delete require.cache[require.resolve('puppeteer-extra-plugin-devtools')] 11 | // }) 12 | 13 | test('will create a tunnel', async t => { 14 | // const puppeteer = require('puppeteer-extra') 15 | // const devtools = require('puppeteer-extra-plugin-devtools')() 16 | // puppeteer.use(devtools) 17 | // devtools.setAuthCredentials('bob', 'swordfish') 18 | 19 | // await puppeteer.launch({ args: PUPPETEER_ARGS }).then(async browser => { 20 | // const tunnel = await devtools.createTunnel(browser) 21 | // t.true(tunnel.url.includes('https://devtools-tunnel-')) 22 | // await browser.close() 23 | // }) 24 | t.true(true) 25 | }) 26 | 27 | // Note: https://tunnel.datahub.at is gone and I don't have an alternative currently 28 | // test('will create a tunnel with custom localtunnel options', async t => { 29 | // const puppeteer = require('puppeteer-extra') 30 | // const devtools = require('puppeteer-extra-plugin-devtools')({ 31 | // auth: { user: 'francis', pass: 'president' }, 32 | // localtunnel: { 33 | // host: 'https://tunnel.datahub.at' 34 | // } 35 | // }) 36 | // puppeteer.use(devtools) 37 | 38 | // await puppeteer.launch({ args: PUPPETEER_ARGS }).then(async browser => { 39 | // const tunnel = await devtools.createTunnel(browser) 40 | // t.true(tunnel.url.includes('.tunnel.datahub.at')) 41 | // browser.close() 42 | // }) 43 | // t.true(true) 44 | // }) 45 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-flash/example.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const puppeteer = require('puppeteer-extra') 4 | 5 | // This might not be the flashPath you're looking for. ;-) 6 | const userName = require('os').userInfo().username 7 | const pluginPath = ` 8 | /Users/${userName}/Library/Application Support/Google/Chrome/PepperFlash/29.0.0.171/PepperFlashPlayer.plugin 9 | `.trim() 10 | const pluginVersion = '29.0.0.171' 11 | 12 | // Will implicitely require 'user-preferences' which will require 'user-data-dir' 13 | // When using default Chromium the pluginPath/pluginVersion need to be specified 14 | puppeteer.use( 15 | require('puppeteer-extra-plugin-flash')({ 16 | pluginPath, 17 | pluginVersion 18 | }) 19 | ) 20 | ;(async () => { 21 | const browser = await puppeteer.launch({ headless: false }) 22 | const page = await browser.newPage() 23 | await page.goto('http://ultrasounds.com', { waitUntil: 'domcontentloaded' }) 24 | })() 25 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-flash/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-flash", 3 | "version": "2.3.3", 4 | "description": "Allow flash on all sites without user interaction.", 5 | "main": "index.js", 6 | "repository": "berstend/puppeteer-extra", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "scripts": { 10 | "docs": "node -e 0", 11 | "lint": "eslint --ext .js .", 12 | "test": "run-p lint", 13 | "test-ci": "run-s test" 14 | }, 15 | "engines": { 16 | "node": ">=8" 17 | }, 18 | "keywords": [ 19 | "puppeteer", 20 | "puppeteer-extra", 21 | "puppeteer-extra-plugin", 22 | "chrome", 23 | "flash", 24 | "allow-flash", 25 | "headless", 26 | "pupeteer" 27 | ], 28 | "devDependencies": { 29 | "ava": "2.4.0", 30 | "npm-run-all": "^4.1.5", 31 | "puppeteer": "^2.0.0" 32 | }, 33 | "dependencies": { 34 | "debug": "^4.1.1", 35 | "puppeteer-extra-plugin": "^3.2.3" 36 | }, 37 | "peerDependencies": { 38 | "playwright-extra": "*", 39 | "puppeteer-extra": "*" 40 | }, 41 | "peerDependenciesMeta": { 42 | "puppeteer-extra": { 43 | "optional": true 44 | }, 45 | "playwright-extra": { 46 | "optional": true 47 | } 48 | }, 49 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 50 | } 51 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-flash/readme.md: -------------------------------------------------------------------------------- 1 | # puppeteer-extra-plugin-flash 2 | 3 | > A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra). 4 | 5 | ## Install 6 | 7 | ```bash 8 | yarn add puppeteer-extra-plugin-flash 9 | ``` 10 | 11 | ## Changelog 12 | 13 | #### `v2.2.5` 14 | 15 | - Improved: Fixes flash content in newer Chrome versions (76+) ([#133](https://github.com/berstend/puppeteer-extra/pull/133), thanks [@Niek](https://github.com/Niek)) 16 | 17 | ## API 18 | 19 | 20 | 21 | #### Table of Contents 22 | 23 | - [Plugin](#plugin) 24 | 25 | ### [Plugin](https://github.com/berstend/puppeteer-extra/blob/db57ea66cf10d407cf63af387892492e495a84f2/packages/puppeteer-extra-plugin-flash/index.js#L31-L100) 26 | 27 | **Extends: PuppeteerExtraPlugin** 28 | 29 | Allow flash on all sites without user interaction. 30 | 31 | Note: The flash plugin is not working in headless mode. 32 | 33 | Note: When using the default Chromium browser 34 | `pluginPath` and `pluginVersion` must be specified (stated in `chrome://version/`). 35 | 36 | Note: Unfortunately this doesn't seem to enable flash on incognito pages, 37 | see [this gist] for a workaround using management policies. 38 | 39 | [this gist]: https://gist.github.com/berstend/bcd64a4a2db28afbd6486daf69f4e787 40 | 41 | Type: `function (opts)` 42 | 43 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** Options (optional, default `{}`) 44 | - `opts.allowFlash` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Whether to allow flash content or not (optional, default `true`) 45 | - `opts.pluginPath` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Flash plugin path (optional, default `null`) 46 | - `opts.pluginVersion` **[boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Flash plugin version (9000 is high enough for Chrome not to complain) (optional, default `9000`) 47 | 48 | Example: 49 | 50 | ```javascript 51 | const puppeteer = require('puppeteer-extra') 52 | puppeteer.use(require('puppeteer-extra-plugin-flash')()) 53 | ;(async () => { 54 | const browser = await puppeteer.launch({ headless: false }) 55 | const page = await browser.newPage() 56 | await page.goto('http://ultrasounds.com', { waitUntil: 'domcontentloaded' }) 57 | })() 58 | ``` 59 | 60 | --- 61 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-font-size/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | /** 6 | * Modify/increase the default font size in puppeteer. 7 | * 8 | * @param {Object} opts - Options 9 | * @param {Number} [opts.defaultFontSize=20] - Default browser font size 10 | * 11 | * @example 12 | * const puppeteer = require('puppeteer-extra') 13 | * puppeteer.use(require('puppeteer-extra-plugin-font-size')()) 14 | * // or 15 | * puppeteer.use(require('puppeteer-extra-plugin-font-size')({defaultFontSize: 18})) 16 | * const browser = await puppeteer.launch() 17 | */ 18 | class Plugin extends PuppeteerExtraPlugin { 19 | constructor(opts = {}) { 20 | super(opts) 21 | } 22 | 23 | get name() { 24 | return 'font-size' 25 | } 26 | 27 | get defaults() { 28 | return { defaultFontSize: 20 } 29 | } 30 | 31 | get requirements() { 32 | return new Set(['launch', 'headful']) 33 | } 34 | 35 | get dependencies() { 36 | return new Set(['user-preferences']) 37 | } 38 | 39 | get data() { 40 | const userPreferences = { 41 | webkit: { 42 | webprefs: { 43 | default_font_size: this.opts.defaultFontSize 44 | } 45 | } 46 | } 47 | return [ 48 | { 49 | name: 'userPreferences', 50 | value: userPreferences 51 | } 52 | ] 53 | } 54 | } 55 | 56 | module.exports = function(pluginConfig) { 57 | return new Plugin(pluginConfig) 58 | } 59 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-font-size/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-font-size", 3 | "version": "2.3.3", 4 | "description": "Adjust font sizes in puppeteer.", 5 | "main": "index.js", 6 | "repository": "berstend/puppeteer-extra", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "scripts": { 10 | "docs": "node -e 0", 11 | "lint": "eslint --ext .js .", 12 | "test": "run-p lint", 13 | "test-ci": "run-s test" 14 | }, 15 | "engines": { 16 | "node": ">=8" 17 | }, 18 | "keywords": [ 19 | "puppeteer", 20 | "puppeteer-extra", 21 | "puppeteer-extra-plugin", 22 | "chrome", 23 | "headless", 24 | "pupeteer" 25 | ], 26 | "devDependencies": { 27 | "ava": "2.4.0", 28 | "npm-run-all": "^4.1.5", 29 | "puppeteer": "^2.0.0" 30 | }, 31 | "dependencies": { 32 | "debug": "^4.1.1", 33 | "puppeteer-extra-plugin": "^3.2.3" 34 | }, 35 | "peerDependencies": { 36 | "playwright-extra": "*", 37 | "puppeteer-extra": "*" 38 | }, 39 | "peerDependenciesMeta": { 40 | "puppeteer-extra": { 41 | "optional": true 42 | }, 43 | "playwright-extra": { 44 | "optional": true 45 | } 46 | }, 47 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 48 | } 49 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-font-size/readme.md: -------------------------------------------------------------------------------- 1 | # puppeteer-extra-plugin-font-size 2 | 3 | > A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra). 4 | 5 | ### Install 6 | 7 | ```bash 8 | yarn add puppeteer-extra-plugin-font-size 9 | ``` 10 | 11 | ## API 12 | 13 | 14 | 15 | #### Table of Contents 16 | 17 | - [Plugin](#plugin) 18 | 19 | ### [Plugin](https://github.com/berstend/puppeteer-extra/blob/db57ea66cf10d407cf63af387892492e495a84f2/packages/puppeteer-extra-plugin-font-size/index.js#L18-L44) 20 | 21 | **Extends: PuppeteerExtraPlugin** 22 | 23 | Modify/increase the default font size in puppeteer. 24 | 25 | Type: `function (opts)` 26 | 27 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** Options (optional, default `{}`) 28 | - `opts.defaultFontSize` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Default browser font size (optional, default `20`) 29 | 30 | Example: 31 | 32 | ```javascript 33 | const puppeteer = require('puppeteer-extra') 34 | puppeteer.use(require('puppeteer-extra-plugin-font-size')()) 35 | // or 36 | puppeteer.use(require('puppeteer-extra-plugin-font-size')({defaultFontSize: 18})) 37 | const browser = await puppeteer.launch() 38 | ``` 39 | 40 | * * * 41 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/ava.config-ts.js: -------------------------------------------------------------------------------- 1 | export default { 2 | compileEnhancements: false, 3 | environmentVariables: { 4 | TS_NODE_COMPILER_OPTIONS: '{"module":"commonjs"}' 5 | }, 6 | files: ['src/**.test.ts'], 7 | extensions: ['ts'], 8 | require: ['ts-node/register'] 9 | } 10 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/ava.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | files: ['dist/*.test.js'] 3 | } 4 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import resolve from 'rollup-plugin-node-resolve' 2 | import sourceMaps from 'rollup-plugin-sourcemaps' 3 | import typescript from 'rollup-plugin-typescript2' 4 | 5 | const pkg = require('./package.json') 6 | 7 | const entryFile = 'index' 8 | const banner = ` 9 | /*! 10 | * ${pkg.name} v${pkg.version} by ${pkg.author} 11 | * ${pkg.homepage || `https://github.com/${pkg.repository}`} 12 | * @license ${pkg.license} 13 | */ 14 | `.trim() 15 | 16 | const defaultExportOutro = ` 17 | module.exports = exports.default || {} 18 | Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value }) 19 | ` 20 | 21 | export default { 22 | input: `src/${entryFile}.ts`, 23 | output: [ 24 | { 25 | file: pkg.main, 26 | format: 'cjs', 27 | sourcemap: true, 28 | exports: 'named', 29 | outro: defaultExportOutro, 30 | banner 31 | }, 32 | { 33 | file: pkg.module, 34 | format: 'es', 35 | sourcemap: true, 36 | exports: 'named', 37 | banner 38 | } 39 | ], 40 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 41 | external: [ 42 | ...Object.keys(pkg.dependencies || {}), 43 | ...Object.keys(pkg.peerDependencies || {}) 44 | ], 45 | watch: { 46 | include: 'src/**' 47 | }, 48 | plugins: [ 49 | // Compile TypeScript files 50 | typescript({ useTsconfigDeclarationDir: true }), 51 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 52 | // commonjs(), 53 | // Allow node_modules resolution, so you can use 'external' to control 54 | // which external modules to include in the bundle 55 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 56 | resolve(), 57 | // Resolve source maps to the original source 58 | sourceMaps() 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/src/ambient.d.ts: -------------------------------------------------------------------------------- 1 | export {} 2 | 3 | // https://github.com/sindresorhus/type-fest/issues/19 4 | declare global { 5 | interface SymbolConstructor { 6 | readonly observable: symbol 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/src/playwright-mods.d.ts: -------------------------------------------------------------------------------- 1 | // Extend Playwright interfaces transparently to the end user. 2 | import {} from 'playwright-core' 3 | 4 | import { RecaptchaPluginPageAdditions } from './types' 5 | 6 | declare module 'playwright-core' { 7 | interface Page extends RecaptchaPluginPageAdditions {} 8 | interface Frame extends RecaptchaPluginPageAdditions {} 9 | } 10 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/src/puppeteer-mods.d.ts: -------------------------------------------------------------------------------- 1 | // Extend Puppeteer interfaces transparently to the end user. 2 | 3 | // Note, we need to manually copy this file into the build dir (yarn ambient-dts): https://stackoverflow.com/questions/56018167 4 | // Note2: It's not sufficient to just copy over this d.ts file, it needs to be referenced by another .ts file! 5 | // Note3: To make it even more urgh the TS compiler will change the reference import path, hence we need to fix that in the end as well 6 | 7 | // This import statement is important for all this to work, otherwise we don't extend but replace the puppeteer module definition. 8 | // https://github.com/microsoft/TypeScript/issues/10859 9 | import {} from 'puppeteer' 10 | 11 | import { RecaptchaPluginPageAdditions } from './types' 12 | 13 | declare module 'puppeteer' { 14 | interface Page extends RecaptchaPluginPageAdditions {} 15 | interface Frame extends RecaptchaPluginPageAdditions {} 16 | } 17 | 18 | declare module 'puppeteer-core' { 19 | interface Page extends RecaptchaPluginPageAdditions {} 20 | interface Frame extends RecaptchaPluginPageAdditions {} 21 | } 22 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./dist", 4 | "target": "es2017", 5 | "module": "es2015", 6 | "moduleResolution": "node", 7 | "lib": ["es2015", "es2016", "es2017", "es2019", "dom"], 8 | // "noResolve": true, // Important: Otherwise TS would rewrite our ambient d.ts file locations (see: yarn copy-dts) :( 9 | "sourceMap": true, 10 | "declaration": true, 11 | "allowSyntheticDefaultImports": true, 12 | "esModuleInterop": true, 13 | "emitDecoratorMetadata": true, 14 | "experimentalDecorators": true, 15 | "strict": false, 16 | "noFallthroughCasesInSwitch": true, 17 | "noImplicitReturns": false, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": false, 20 | "pretty": true, 21 | "stripInternal": true, 22 | "types": ["node"] 23 | }, 24 | "include": [ 25 | "./src/**/*.tsx", 26 | "./src/**/*.ts", 27 | "./src/**/*.d.ts", 28 | "./src/**/*.test.ts", 29 | "./test/**/*.ts", 30 | "src/provider/2captcha-api.js" 31 | ], 32 | "exclude": ["node_modules", "dist", "./test/**/*.spec.ts"] 33 | } 34 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-recaptcha/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint-config-standard", "tslint-config-prettier"], 3 | "rules": { 4 | "ordered-imports": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-repl/index.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import { EventEmitter } from 'events'; 4 | import 'puppeteer'; 5 | import { PuppeteerExtraPlugin, PluginOptions } from 'puppeteer-extra-plugin'; 6 | 7 | // augment repl() for Page/Browser 8 | 9 | declare module 'puppeteer' { 10 | export interface Page extends EventEmitter, FrameBase { 11 | repl(): Promise; 12 | } 13 | 14 | export interface Browser extends EventEmitter, TargetAwaiter { 15 | repl(): Promise; 16 | } 17 | } 18 | 19 | /** 20 | * Create an interactive REPL for the provided object. 21 | * Uses an extended (colorized) readline interface under the hood. Will resolve the returned Promise when the readline interface is closed. 22 | * If opts.addToPuppeteerClass is true (default) then page.repl()/browser.repl() will point to this method, for convenience. 23 | * Can be used standalone as well, to inspect an arbitrary class instance or object. 24 | */ 25 | declare function repl(config?: Options): Plugin; 26 | 27 | declare interface Options extends DefaultOptions, PluginOptions {} 28 | 29 | declare interface DefaultOptions { 30 | /** 31 | * If a .repl() method should be attached to Puppeteer Page and Browser instances 32 | * @default true 33 | */ 34 | addToPuppeteerClass?: boolean; 35 | } 36 | 37 | declare class Plugin extends PuppeteerExtraPlugin { 38 | get name(): 'repl'; 39 | get defaults(): DefaultOptions; 40 | repl(obj: any): Promise; 41 | } 42 | 43 | export = repl; 44 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-repl/index.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const PLUGIN_NAME = 'repl' 4 | 5 | const test = require('ava') 6 | 7 | const Plugin = require('.') 8 | 9 | test('is a function', async t => { 10 | t.is(typeof Plugin, 'function') 11 | }) 12 | 13 | test('should have the basic class members', async t => { 14 | const instance = new Plugin() 15 | t.is(instance.name, PLUGIN_NAME) 16 | t.true(instance._isPuppeteerExtraPlugin) 17 | }) 18 | 19 | test('should have the public child class members', async t => { 20 | const instance = new Plugin() 21 | const prototype = Object.getPrototypeOf(instance) 22 | const childClassMembers = Object.getOwnPropertyNames(prototype) 23 | 24 | t.true(childClassMembers.includes('constructor')) 25 | t.true(childClassMembers.includes('name')) 26 | t.true(childClassMembers.includes('defaults')) 27 | t.true(childClassMembers.includes('requirements')) 28 | t.true(childClassMembers.includes('repl')) 29 | t.true(childClassMembers.includes('onPageCreated')) 30 | t.true(childClassMembers.length === 6) 31 | }) 32 | 33 | test('should have opts with default values', async t => { 34 | const instance = new Plugin() 35 | const opts = instance.opts 36 | 37 | t.is(opts.addToPuppeteerClass, true) 38 | }) 39 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-repl/lib/REPLSession.js: -------------------------------------------------------------------------------- 1 | const ow = require('ow') 2 | const readline = require('./super-readline') 3 | 4 | class REPLSession { 5 | constructor(opts) { 6 | ow(opts, ow.object.hasKeys('obj')) 7 | ow(opts.obj, ow.object.hasKeys('constructor')) 8 | 9 | this._obj = opts.obj 10 | this._meta = { 11 | type: typeof this._obj, 12 | name: this._obj.constructor.name, 13 | members: 14 | Object.getOwnPropertyNames(Object.getPrototypeOf(this._obj)) || [] 15 | } 16 | this._completions = [].concat(this.extraMethods, this._meta.members) 17 | } 18 | 19 | get extraMethods() { 20 | return ['inspect', 'exit'] 21 | } 22 | 23 | async start() { 24 | this._createInterface() 25 | this._showIntro() 26 | this._rl.prompt() 27 | return this._closePromise 28 | } 29 | 30 | _createInterface() { 31 | this._rl = readline.createInterface({ 32 | input: process.stdin, 33 | output: process.stdout, 34 | prompt: this._meta.name ? `> ${this._meta.name.toLowerCase()}.` : `> `, 35 | completer: readline.defaultCompleter(this._completions), 36 | colors: { 37 | prompt: readline.chalk.cyan, 38 | completer: readline.chalk.yellow 39 | } 40 | }) 41 | this._rl.on('line', this._onLineInput.bind(this)) 42 | this._closePromise = new Promise(resolve => 43 | this._rl.once('close', () => resolve()) 44 | ) 45 | } 46 | 47 | _showIntro() { 48 | console.log(` 49 | Started puppeteer-extra repl for ${this._meta.type} '${this._meta.name}' with ${this._meta.members.length} properties. 50 | 51 | - Type 'inspect' to return the current ${this._meta.type}. 52 | - Type 'exit' to leave the repl. 53 | 54 | Tab auto-completion available: 55 | `) 56 | this._rl.showTabCompletions() 57 | } 58 | 59 | async _onLineInput(line) { 60 | if (!line) { 61 | return this._rl.prompt() 62 | } 63 | if (line === 'exit') { 64 | return this._rl.close() 65 | } 66 | 67 | const cmd = line === 'inspect' ? this._obj : `this._obj.${line}` 68 | await this._evalAsync(cmd) 69 | this._rl.prompt() 70 | } 71 | 72 | async _evalAsync(cmd) { 73 | try { 74 | // eslint-disable-next-line no-eval 75 | const out = await eval(cmd) 76 | console.log(out) 77 | } catch (err) { 78 | console.warn(err) 79 | } 80 | } 81 | } 82 | 83 | module.exports = REPLSession 84 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-repl/lib/REPLSession.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | const REPLSession = require('./REPLSession') 6 | 7 | test('is a function', async t => { 8 | t.is(typeof REPLSession, 'function') 9 | }) 10 | 11 | test('is a class', async t => { 12 | t.is(REPLSession.constructor.name, 'Function') 13 | }) 14 | 15 | test('will throw without opts', async t => { 16 | const error = await t.throws(() => new REPLSession()) 17 | t.is( 18 | error.message, 19 | 'Expected argument to be of type `object` but received type `undefined`' 20 | ) 21 | }) 22 | 23 | test('will throw when opts.obj is not a class derivative', async t => { 24 | const error = await t.throws(() => new REPLSession({ obj: 'foobar' })) 25 | t.is( 26 | error.message, 27 | 'Expected argument to be of type `object` but received type `string`' 28 | ) 29 | }) 30 | 31 | test('should have the expected class members', async t => { 32 | const FakeClass = class Foo {} 33 | const opts = { obj: new FakeClass() } 34 | const instance = new REPLSession(opts) 35 | const prototype = Object.getPrototypeOf(instance) 36 | const childClassMembers = Object.getOwnPropertyNames(prototype) 37 | 38 | t.true(childClassMembers.includes('constructor')) 39 | t.true(childClassMembers.includes('extraMethods')) 40 | t.true(childClassMembers.includes('start')) 41 | t.true(childClassMembers.includes('_createInterface')) 42 | t.true(childClassMembers.includes('_showIntro')) 43 | t.true(childClassMembers.includes('_onLineInput')) 44 | t.true(childClassMembers.includes('_evalAsync')) 45 | t.true(childClassMembers.length === 7) 46 | }) 47 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-repl/lib/super-readline.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | const readline = require('./super-readline') 6 | 7 | test('is an object', async t => { 8 | t.is(typeof readline, 'object') 9 | }) 10 | 11 | test('should have the expected number of exports', async t => { 12 | const exportedKeys = Object.keys(readline) 13 | 14 | t.true(exportedKeys.includes('chalk')) 15 | t.true(exportedKeys.includes('Interface')) 16 | t.true(exportedKeys.includes('createInterface')) 17 | t.true(exportedKeys.includes('defaultCompleter')) 18 | t.true(exportedKeys.includes('clearLine')) 19 | t.true(exportedKeys.includes('clearScreenDown')) 20 | t.true(exportedKeys.includes('cursorTo')) 21 | t.true(exportedKeys.includes('emitKeypressEvents')) 22 | t.true(exportedKeys.includes('moveCursor')) 23 | t.is(exportedKeys.length, 9) 24 | }) 25 | 26 | test('can create an interface', async t => { 27 | const instance = readline.createInterface({ 28 | input: process.stdin, 29 | output: process.stdout, 30 | prompt: '> ', 31 | completer: readline.defaultCompleter(['bob', 'yolk']), 32 | colors: { 33 | prompt: readline.chalk.cyan, 34 | completer: readline.chalk.yellow 35 | } 36 | }) 37 | t.is(instance.constructor.name, 'SuperInterface') 38 | t.is(typeof instance, 'object') 39 | }) 40 | 41 | test('should have the extended class members', async t => { 42 | const instance = readline.createInterface({ 43 | input: process.stdin, 44 | output: process.stdout, 45 | prompt: '> ', 46 | completer: readline.defaultCompleter(['bob', 'yolk']), 47 | colors: { 48 | prompt: readline.chalk.cyan, 49 | completer: readline.chalk.yellow 50 | } 51 | }) 52 | const prototype = Object.getPrototypeOf(instance) 53 | const childClassMembers = Object.getOwnPropertyNames(prototype) 54 | 55 | t.true(childClassMembers.includes('constructor')) 56 | t.true(childClassMembers.includes('_tabComplete')) 57 | t.true(childClassMembers.includes('_writeToOutput')) 58 | t.true(childClassMembers.includes('showTabCompletions')) 59 | }) 60 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-repl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-repl", 3 | "version": "2.3.3", 4 | "description": "Start an interactive REPL in your puppeteer code.", 5 | "main": "index.js", 6 | "types": "index.d.ts", 7 | "repository": "berstend/puppeteer-extra", 8 | "author": "nswbmw & berstend", 9 | "license": "MIT", 10 | "scripts": { 11 | "docs": "node -e 0", 12 | "lint": "eslint --ext .js .", 13 | "test": "run-p lint", 14 | "test-ci": "run-s test" 15 | }, 16 | "engines": { 17 | "node": ">=8" 18 | }, 19 | "keywords": [ 20 | "puppeteer", 21 | "puppeteer-extra", 22 | "puppeteer-extra-plugin", 23 | "repl", 24 | "debug", 25 | "interactive", 26 | "puppeteer-debug", 27 | "puppeteer-repl", 28 | "chrome", 29 | "headless", 30 | "pupeteer" 31 | ], 32 | "devDependencies": { 33 | "ava": "2.4.0", 34 | "mock-stdin": "^0.3.1", 35 | "npm-run-all": "^4.1.5", 36 | "puppeteer": "^2.0.0" 37 | }, 38 | "dependencies": { 39 | "chalk": "^3.0.0", 40 | "debug": "^4.1.1", 41 | "ow": "^0.4.0", 42 | "puppeteer-extra-plugin": "^3.2.3" 43 | }, 44 | "peerDependencies": { 45 | "puppeteer-extra": "*" 46 | }, 47 | "peerDependenciesMeta": { 48 | "puppeteer-extra": { 49 | "optional": true 50 | } 51 | }, 52 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 53 | } 54 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-repl/test/headless.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | // const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'] 6 | 7 | test.beforeEach(t => { 8 | // Make sure we work with pristine modules 9 | // delete require.cache[require.resolve('puppeteer-extra')] 10 | // delete require.cache[require.resolve('puppeteer-extra-plugin-repl')] 11 | }) 12 | 13 | test('will create a repl', async t => { 14 | t.pass() 15 | // @TODO: This test is a little brittle and fails in CI sometimes. 16 | 17 | // const stdin = require('mock-stdin').stdin() 18 | 19 | // const puppeteer = require('puppeteer-extra') 20 | // const repl = require('puppeteer-extra-plugin-repl')() 21 | // puppeteer.use(repl) 22 | 23 | // await puppeteer.launch({ args: PUPPETEER_ARGS }).then(async browser => { 24 | // const page = await browser.newPage() 25 | 26 | // // Mock stdout, there might be cleaner ways to do this :-) 27 | // let stdoutOutput = '' 28 | // const origStdout = process.stdout.write 29 | // process.stdout.write = (string, encoding, fd) => { stdoutOutput += string } 30 | // await Promise.all([ 31 | // page.repl(), 32 | // stdin.send('url()'), 33 | // stdin.end() 34 | // ]) 35 | // process.stdout.write = origStdout 36 | 37 | // t.true(stdoutOutput.includes(`Started puppeteer-extra repl for object 'Page' with`)) 38 | // t.true(stdoutOutput.includes(`> page.`)) 39 | // t.true(stdoutOutput.includes(`url()`)) 40 | // t.true(stdoutOutput.includes(`about:blank`)) 41 | 42 | // browser.close() 43 | // }) 44 | // t.true(true) 45 | }) 46 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/.npmignore: -------------------------------------------------------------------------------- 1 | stealthtests/ 2 | runall_stealthtests.sh -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/_template/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | /** 6 | * Minimal stealth plugin template, not being used. :-) 7 | * 8 | * Feel free to copy this folder as the basis for additional detection evasion plugins. 9 | */ 10 | class Plugin extends PuppeteerExtraPlugin { 11 | constructor(opts = {}) { 12 | super(opts) 13 | } 14 | 15 | get name() { 16 | return 'stealth/evasions/_template' 17 | } 18 | 19 | async onPageCreated(page) { 20 | await page.evaluateOnNewDocument(() => { 21 | console.debug('hello world') 22 | }) 23 | } 24 | } 25 | 26 | module.exports = function(pluginConfig) { 27 | return new Plugin(pluginConfig) 28 | } 29 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/_template/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/_template/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/_template/index.js#L10-L24) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Minimal stealth plugin template, not being used. :-) 16 | 17 | Feel free to copy this folder as the basis for additional detection evasion plugins. 18 | 19 | --- 20 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/_utils/withUtils.js: -------------------------------------------------------------------------------- 1 | const utils = require('./index') 2 | 3 | /** 4 | * Wrap a page with utilities. 5 | * 6 | * @param {Puppeteer.Page} page 7 | */ 8 | module.exports = page => ({ 9 | /** 10 | * Simple `page.evaluate` replacement to preload utils 11 | */ 12 | evaluate: async function (mainFunction, ...args) { 13 | return page.evaluate( 14 | ({ _utilsFns, _mainFunction, _args }) => { 15 | // Add this point we cannot use our utililty functions as they're just strings, we need to materialize them first 16 | const utils = Object.fromEntries( 17 | Object.entries(_utilsFns).map(([key, value]) => [key, eval(value)]) // eslint-disable-line no-eval 18 | ) 19 | utils.init() 20 | return eval(_mainFunction)(utils, ..._args) // eslint-disable-line no-eval 21 | }, 22 | { 23 | _utilsFns: utils.stringifyFns(utils), 24 | _mainFunction: mainFunction.toString(), 25 | _args: args || [] 26 | } 27 | ) 28 | }, 29 | /** 30 | * Simple `page.evaluateOnNewDocument` replacement to preload utils 31 | */ 32 | evaluateOnNewDocument: async function (mainFunction, ...args) { 33 | return page.evaluateOnNewDocument( 34 | ({ _utilsFns, _mainFunction, _args }) => { 35 | // Add this point we cannot use our utililty functions as they're just strings, we need to materialize them first 36 | const utils = Object.fromEntries( 37 | Object.entries(_utilsFns).map(([key, value]) => [key, eval(value)]) // eslint-disable-line no-eval 38 | ) 39 | utils.init() 40 | return eval(_mainFunction)(utils, ..._args) // eslint-disable-line no-eval 41 | }, 42 | { 43 | _utilsFns: utils.stringifyFns(utils), 44 | _mainFunction: mainFunction.toString(), 45 | _args: args || [] 46 | } 47 | ) 48 | } 49 | }) 50 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 4 | 5 | const Plugin = require('.') 6 | 7 | /* global chrome */ 8 | 9 | test('stealth: will add convincing chrome.app object', async t => { 10 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin({})) 11 | const browser = await puppeteer.launch({ headless: true }) 12 | const page = await browser.newPage() 13 | 14 | const results = await page.evaluate(() => { 15 | const catchErr = (fn, ...args) => { 16 | try { 17 | return fn.apply(this, args) 18 | } catch ({ name, message, stack }) { 19 | return { name, message, stack } 20 | } 21 | } 22 | 23 | return { 24 | app: { 25 | exists: window.chrome && 'app' in window.chrome, 26 | toString: chrome.app.toString(), 27 | deepToString: chrome.app.runningState.toString() 28 | }, 29 | data: { 30 | getIsInstalled: chrome.app.getIsInstalled(), 31 | runningState: chrome.app.runningState(), 32 | getDetails: chrome.app.getDetails(), 33 | InstallState: chrome.app.InstallState, 34 | RunningState: chrome.app.RunningState 35 | }, 36 | errors: { 37 | getIsInstalled: catchErr(chrome.app.getDetails, 'foo').message, 38 | stackOK: !catchErr(chrome.app.getDetails, 'foo').stack.includes( 39 | 'at getDetails' 40 | ) 41 | } 42 | } 43 | }) 44 | 45 | t.deepEqual(results, { 46 | app: { 47 | exists: true, 48 | toString: '[object Object]', 49 | deepToString: 'function getDetails() { [native code] }' 50 | }, 51 | data: { 52 | InstallState: { 53 | DISABLED: 'disabled', 54 | INSTALLED: 'installed', 55 | NOT_INSTALLED: 'not_installed' 56 | }, 57 | RunningState: { 58 | CANNOT_RUN: 'cannot_run', 59 | READY_TO_RUN: 'ready_to_run', 60 | RUNNING: 'running' 61 | }, 62 | getDetails: null, 63 | getIsInstalled: false, 64 | runningState: 'cannot_run' 65 | }, 66 | errors: { 67 | getIsInstalled: 'Error in invocation of app.getDetails()', 68 | stackOK: true 69 | } 70 | }) 71 | }) 72 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.js#L11-L97) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Mock the `chrome.app` object if not available (e.g. when running headless). 16 | 17 | --- 18 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 4 | 5 | const Plugin = require('.') 6 | 7 | /* global chrome */ 8 | 9 | test('stealth: will add functional chrome.csi function mock', async t => { 10 | const puppeteer = addExtra(vanillaPuppeteer).use( 11 | Plugin({ 12 | runOnInsecureOrigins: true // for testing 13 | }) 14 | ) 15 | const browser = await puppeteer.launch({ headless: true }) 16 | const page = await browser.newPage() 17 | 18 | const results = await page.evaluate(() => { 19 | const { timing } = window.performance 20 | const csi = window.chrome.csi() 21 | 22 | return { 23 | csi: { 24 | exists: window.chrome && 'csi' in window.chrome, 25 | toString: chrome.csi.toString() 26 | }, 27 | dataOK: { 28 | onloadT: csi.onloadT === timing.domContentLoadedEventEnd, 29 | startE: csi.startE === timing.navigationStart, 30 | pageT: Number.isInteger(csi.pageT), 31 | tran: Number.isInteger(csi.tran) 32 | } 33 | } 34 | }) 35 | 36 | t.deepEqual(results, { 37 | csi: { 38 | exists: true, 39 | toString: 'function () { [native code] }' 40 | }, 41 | dataOK: { 42 | onloadT: true, 43 | pageT: true, 44 | startE: true, 45 | tran: true 46 | } 47 | }) 48 | }) 49 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/index.js#L25-L70) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Mock the `chrome.csi` function if not available (e.g. when running headless). 16 | It's a deprecated (but unfortunately still existing) chrome specific API to fetch browser timings. 17 | 18 | Internally chromium switched the implementation to use the WebPerformance API, 19 | so we can do the same to create a fully functional mock. :-) 20 | 21 | Note: We're using the deprecated PerformanceTiming API instead of the new Navigation Timing Level 2 API on purpopse. 22 | 23 | - **See: ** 24 | - **See: ** 25 | - **See: ** 26 | - **See: ** 27 | - **See: ** 28 | - **See: `chrome.loadTimes` evasion** 29 | 30 | --- 31 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 4 | 5 | const Plugin = require('.') 6 | 7 | /* global chrome */ 8 | 9 | test('stealth: will add functional chrome.loadTimes function mock', async t => { 10 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin({})) 11 | const browser = await puppeteer.launch({ headless: true }) 12 | const page = await browser.newPage() 13 | 14 | const results = await page.evaluate(() => { 15 | const loadTimes = window.chrome.loadTimes() 16 | 17 | return { 18 | loadTimes: { 19 | exists: window.chrome && 'loadTimes' in window.chrome, 20 | toString: chrome.loadTimes.toString() 21 | }, 22 | dataOK: { 23 | connectionInfo: 'connectionInfo' in loadTimes, 24 | npnNegotiatedProtocol: 'npnNegotiatedProtocol' in loadTimes, 25 | navigationType: 'navigationType' in loadTimes, 26 | wasAlternateProtocolAvailable: 27 | 'wasAlternateProtocolAvailable' in loadTimes, 28 | wasFetchedViaSpdy: 'wasFetchedViaSpdy' in loadTimes, 29 | wasNpnNegotiated: 'wasNpnNegotiated' in loadTimes, 30 | 31 | firstPaintAfterLoadTime: 'firstPaintAfterLoadTime' in loadTimes, 32 | requestTime: 'requestTime' in loadTimes, 33 | startLoadTime: 'startLoadTime' in loadTimes, 34 | commitLoadTime: 'commitLoadTime' in loadTimes, 35 | finishDocumentLoadTime: 'finishDocumentLoadTime' in loadTimes, 36 | finishLoadTime: 'finishLoadTime' in loadTimes, 37 | firstPaintTime: 'firstPaintTime' in loadTimes 38 | } 39 | } 40 | }) 41 | 42 | t.deepEqual(results, { 43 | loadTimes: { 44 | exists: true, 45 | toString: 'function () { [native code] }' 46 | }, 47 | dataOK: { 48 | commitLoadTime: true, 49 | connectionInfo: true, 50 | finishDocumentLoadTime: true, 51 | finishLoadTime: true, 52 | firstPaintAfterLoadTime: true, 53 | firstPaintTime: true, 54 | navigationType: true, 55 | npnNegotiatedProtocol: true, 56 | requestTime: true, 57 | startLoadTime: true, 58 | wasAlternateProtocolAvailable: true, 59 | wasFetchedViaSpdy: true, 60 | wasNpnNegotiated: true 61 | } 62 | }) 63 | }) 64 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/index.js#L23-L164) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Mock the `chrome.loadTimes` function if not available (e.g. when running headless). 16 | It's a deprecated (but unfortunately still existing) chrome specific API to fetch browser timings and connection info. 17 | 18 | Internally chromium switched the implementation to use the WebPerformance API, 19 | so we can do the same to create a fully functional mock. :-) 20 | 21 | Note: We're using the deprecated PerformanceTiming API instead of the new Navigation Timing Level 2 API on purpopse. 22 | 23 | - **See: ** 24 | - **See: ** 25 | - **See: ** 26 | - **See: `chrome.csi` evasion** 27 | 28 | --- 29 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | - [sendMessageHandler()](#sendmessagehandler) 9 | - [connectHandler()](#connecthandler) 10 | 11 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js#L13-L251) 12 | 13 | - `opts` (optional, default `{}`) 14 | 15 | **Extends: PuppeteerExtraPlugin** 16 | 17 | Mock the `chrome.runtime` object if not available (e.g. when running headless) and on a secure site. 18 | 19 | --- 20 | 21 | ### [sendMessageHandler()](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js#L80-L123) 22 | 23 | Mock `chrome.runtime.sendMessage` 24 | 25 | --- 26 | 27 | ### [connectHandler()](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js#L136-L210) 28 | 29 | Mock `chrome.runtime.connect` 30 | 31 | - **See: ** 32 | 33 | --- 34 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/staticData.json: -------------------------------------------------------------------------------- 1 | { 2 | "OnInstalledReason": { 3 | "CHROME_UPDATE": "chrome_update", 4 | "INSTALL": "install", 5 | "SHARED_MODULE_UPDATE": "shared_module_update", 6 | "UPDATE": "update" 7 | }, 8 | "OnRestartRequiredReason": { 9 | "APP_UPDATE": "app_update", 10 | "OS_UPDATE": "os_update", 11 | "PERIODIC": "periodic" 12 | }, 13 | "PlatformArch": { 14 | "ARM": "arm", 15 | "ARM64": "arm64", 16 | "MIPS": "mips", 17 | "MIPS64": "mips64", 18 | "X86_32": "x86-32", 19 | "X86_64": "x86-64" 20 | }, 21 | "PlatformNaclArch": { 22 | "ARM": "arm", 23 | "MIPS": "mips", 24 | "MIPS64": "mips64", 25 | "X86_32": "x86-32", 26 | "X86_64": "x86-64" 27 | }, 28 | "PlatformOs": { 29 | "ANDROID": "android", 30 | "CROS": "cros", 31 | "LINUX": "linux", 32 | "MAC": "mac", 33 | "OPENBSD": "openbsd", 34 | "WIN": "win" 35 | }, 36 | "RequestUpdateCheckStatus": { 37 | "NO_UPDATE": "no_update", 38 | "THROTTLED": "throttled", 39 | "UPDATE_AVAILABLE": "update_available" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | const argsToIgnore = [ 6 | '--disable-extensions', 7 | '--disable-default-apps', 8 | '--disable-component-extensions-with-background-pages' 9 | ] 10 | 11 | /** 12 | * A CDP driver like puppeteer can make use of various browser launch arguments that are 13 | * adversarial to mimicking a regular browser and need to be stripped when launching the browser. 14 | */ 15 | class Plugin extends PuppeteerExtraPlugin { 16 | constructor(opts = {}) { 17 | super(opts) 18 | } 19 | 20 | get name() { 21 | return 'stealth/evasions/defaultArgs' 22 | } 23 | 24 | get requirements() { 25 | return new Set(['runLast']) // So other plugins can modify launch options before 26 | } 27 | 28 | async beforeLaunch(options = {}) { 29 | options.ignoreDefaultArgs = options.ignoreDefaultArgs || [] 30 | if (options.ignoreDefaultArgs === true) { 31 | // that means the user explicitly wants to disable all default arguments 32 | return 33 | } 34 | argsToIgnore.forEach(arg => { 35 | if (options.ignoreDefaultArgs.includes(arg)) { 36 | return 37 | } 38 | options.ignoreDefaultArgs.push(arg) 39 | }) 40 | } 41 | } 42 | 43 | module.exports = function (pluginConfig) { 44 | return new Plugin(pluginConfig) 45 | } 46 | 47 | module.exports.argsToIgnore = argsToIgnore 48 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 4 | const Plugin = require('.') 5 | const { argsToIgnore } = require('.') 6 | 7 | test('vanilla: uses args to ignore', async t => { 8 | const browser = await vanillaPuppeteer.launch({ headless: true }) 9 | const page = await browser.newPage() 10 | const client = 11 | typeof page._client === 'function' ? page._client() : page._client 12 | const { arguments: launchArgs } = await client.send( 13 | 'Browser.getBrowserCommandLine' 14 | ) 15 | const ok = argsToIgnore.every(arg => launchArgs.includes(arg)) 16 | if (!ok) { 17 | console.log({ argsToIgnore, launchArgs }) 18 | } 19 | t.is(ok, true) 20 | }) 21 | 22 | test('stealth: does not use args to ignore', async t => { 23 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 24 | const browser = await puppeteer.launch({ headless: true }) 25 | const page = await browser.newPage() 26 | const client = 27 | typeof page._client === 'function' ? page._client() : page._client 28 | const { arguments: launchArgs } = await client.send( 29 | 'Browser.getBrowserCommandLine' 30 | ) 31 | const ok = argsToIgnore.every(arg => !launchArgs.includes(arg)) 32 | if (!ok) { 33 | console.log({ argsToIgnore, launchArgs }) 34 | } 35 | t.is(ok, true) 36 | }) 37 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/358246d5cc56bbb8800624128503482b8d7b426a/packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/index.js#L15-L41) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | A CDP driver like puppeteer can make use of various browser launch arguments that are 16 | adversarial to mimicking a regular browser and need to be stripped when launching the browser. 17 | 18 | --- 19 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/index.js#L11-L125) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Fix for the HEADCHR_IFRAME detection (iframe.contentWindow.chrome), hopefully this time without breaking iframes. 16 | Note: Only `srcdoc` powered iframes cause issues due to a chromium bug: 17 | 18 | 19 | 20 | --- 21 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | - [parseInput(arg)](#parseinputarg) 9 | 10 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.js#L12-L88) 11 | 12 | - `opts` (optional, default `{}`) 13 | 14 | **Extends: PuppeteerExtraPlugin** 15 | 16 | Fix Chromium not reporting "probably" to codecs like `videoEl.canPlayType('video/mp4; codecs="avc1.42E01E"')`. 17 | (Chromium doesn't support proprietary codecs, only Chrome does) 18 | 19 | --- 20 | 21 | ### [parseInput(arg)](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.js#L33-L51) 22 | 23 | - `arg` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 24 | 25 | Input might look funky, we need to normalize it so e.g. whitespace isn't an issue for our spoofing. 26 | 27 | Example: 28 | 29 | ```javascript 30 | video / webm 31 | codecs = 'vp8, vorbis' 32 | video / mp4 33 | codecs = 'avc1.42E01E' 34 | audio / x - m4a 35 | audio / ogg 36 | codecs = 'vorbis' 37 | ``` 38 | 39 | --- 40 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | const withUtils = require('../_utils/withUtils') 6 | 7 | /** 8 | * Set the hardwareConcurrency to 4 (optionally configurable with `hardwareConcurrency`) 9 | * 10 | * @see https://arh.antoinevastel.com/reports/stats/osName_hardwareConcurrency_report.html 11 | * 12 | * @param {Object} [opts] - Options 13 | * @param {number} [opts.hardwareConcurrency] - The value to use in `navigator.hardwareConcurrency` (default: `4`) 14 | */ 15 | 16 | class Plugin extends PuppeteerExtraPlugin { 17 | constructor(opts = {}) { 18 | super(opts) 19 | } 20 | 21 | get name() { 22 | return 'stealth/evasions/navigator.hardwareConcurrency' 23 | } 24 | 25 | get defaults() { 26 | return { 27 | hardwareConcurrency: 4 28 | } 29 | } 30 | 31 | async onPageCreated(page) { 32 | await withUtils(page).evaluateOnNewDocument( 33 | (utils, { opts }) => { 34 | utils.replaceGetterWithProxy( 35 | Object.getPrototypeOf(navigator), 36 | 'hardwareConcurrency', 37 | utils.makeHandler().getterValue(opts.hardwareConcurrency) 38 | ) 39 | }, 40 | { 41 | opts: this.opts 42 | } 43 | ) 44 | } 45 | } 46 | 47 | module.exports = function (pluginConfig) { 48 | return new Plugin(pluginConfig) 49 | } 50 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const os = require('os') 3 | 4 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 5 | 6 | const { 7 | getVanillaFingerPrint, 8 | getStealthFingerPrint 9 | } = require('../../test/util') 10 | const Plugin = require('.') 11 | 12 | const fingerprintFn = page => page.evaluate('navigator.hardwareConcurrency') 13 | 14 | test('vanilla: matches real core count', async t => { 15 | const { pageFnResult } = await getVanillaFingerPrint(fingerprintFn) 16 | t.is(pageFnResult, os.cpus().length) 17 | }) 18 | 19 | test('stealth: default is set to 4', async t => { 20 | const { pageFnResult } = await getStealthFingerPrint(Plugin, fingerprintFn) 21 | t.is(pageFnResult, 4) 22 | }) 23 | 24 | test('stealth: will override value correctly', async t => { 25 | const { pageFnResult } = await getStealthFingerPrint(Plugin, fingerprintFn, { 26 | hardwareConcurrency: 8 27 | }) 28 | t.is(pageFnResult, 8) 29 | }) 30 | 31 | test('stealth: does patch getters properly', async t => { 32 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 33 | const browser = await puppeteer.launch({ headless: true }) 34 | const page = await browser.newPage() 35 | 36 | const results = await page.evaluate(() => { 37 | const hasInvocationError = (() => { 38 | try { 39 | // eslint-disable-next-line dot-notation 40 | Object['seal'](Object.getPrototypeOf(navigator)['hardwareConcurrency']) 41 | return false 42 | } catch (err) { 43 | return true 44 | } 45 | })() 46 | return { 47 | hasInvocationError, 48 | toString: Object.getOwnPropertyDescriptor( 49 | Object.getPrototypeOf(navigator), 50 | 'hardwareConcurrency' 51 | ).get.toString() 52 | } 53 | }) 54 | 55 | t.deepEqual(results, { 56 | hasInvocationError: true, 57 | toString: 'function get hardwareConcurrency() { [native code] }' 58 | }) 59 | }) 60 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/9534845cc95088e65c2d53bfb029263976fc9add/packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js#L16-L37) 10 | 11 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`) 12 | - `opts.hardwareConcurrency` **[number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)?** The value to use in `navigator.hardwareConcurrency` (default: `4`) 13 | 14 | **Extends: PuppeteerExtraPlugin** 15 | 16 | Set the hardwareConcurrency to 4 (optionally configurable with `hardwareConcurrency`) 17 | 18 | - **See: ** 19 | 20 | --- 21 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | const withUtils = require('../_utils/withUtils') 5 | 6 | /** 7 | * Pass the Languages Test. Allows setting custom languages. 8 | * 9 | * @param {Object} [opts] - Options 10 | * @param {Array} [opts.languages] - The languages to use (default: `['en-US', 'en']`) 11 | */ 12 | class Plugin extends PuppeteerExtraPlugin { 13 | constructor(opts = {}) { 14 | super(opts) 15 | } 16 | 17 | get name() { 18 | return 'stealth/evasions/navigator.languages' 19 | } 20 | 21 | get defaults() { 22 | return { 23 | languages: [] // Empty default, otherwise this would be merged with user defined array override 24 | } 25 | } 26 | 27 | async onPageCreated(page) { 28 | await withUtils(page).evaluateOnNewDocument( 29 | (utils, { opts }) => { 30 | const languages = opts.languages.length 31 | ? opts.languages 32 | : ['en-US', 'en'] 33 | utils.replaceGetterWithProxy( 34 | Object.getPrototypeOf(navigator), 35 | 'languages', 36 | utils.makeHandler().getterValue(Object.freeze([...languages])) 37 | ) 38 | }, 39 | { 40 | opts: this.opts 41 | } 42 | ) 43 | } 44 | } 45 | 46 | module.exports = function (pluginConfig) { 47 | return new Plugin(pluginConfig) 48 | } 49 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js#L11-L28) 10 | 11 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`) 12 | - `opts.languages` **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>?** The languages to use (default: `['en-US', 'en']`) 13 | 14 | **Extends: PuppeteerExtraPlugin** 15 | 16 | Pass the Languages Test. Allows setting custom languages. 17 | 18 | --- 19 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | const withUtils = require('../_utils/withUtils') 6 | 7 | /** 8 | * Fix `Notification.permission` behaving weirdly in headless mode 9 | * 10 | * @see https://bugs.chromium.org/p/chromium/issues/detail?id=1052332 11 | */ 12 | 13 | class Plugin extends PuppeteerExtraPlugin { 14 | constructor(opts = {}) { 15 | super(opts) 16 | } 17 | 18 | get name() { 19 | return 'stealth/evasions/navigator.permissions' 20 | } 21 | 22 | /* global Notification Permissions PermissionStatus */ 23 | async onPageCreated(page) { 24 | await withUtils(page).evaluateOnNewDocument((utils, opts) => { 25 | const isSecure = document.location.protocol.startsWith('https') 26 | 27 | // In headful on secure origins the permission should be "default", not "denied" 28 | if (isSecure) { 29 | utils.replaceGetterWithProxy(Notification, 'permission', { 30 | apply() { 31 | return 'default' 32 | } 33 | }) 34 | } 35 | 36 | // Another weird behavior: 37 | // On insecure origins in headful the state is "denied", 38 | // whereas in headless it's "prompt" 39 | if (!isSecure) { 40 | const handler = { 41 | apply(target, ctx, args) { 42 | const param = (args || [])[0] 43 | 44 | const isNotifications = 45 | param && param.name && param.name === 'notifications' 46 | if (!isNotifications) { 47 | return utils.cache.Reflect.apply(...arguments) 48 | } 49 | 50 | return Promise.resolve( 51 | Object.setPrototypeOf( 52 | { 53 | state: 'denied', 54 | onchange: null 55 | }, 56 | PermissionStatus.prototype 57 | ) 58 | ) 59 | } 60 | } 61 | // Note: Don't use `Object.getPrototypeOf` here 62 | utils.replaceWithProxy(Permissions.prototype, 'query', handler) 63 | } 64 | }, this.opts) 65 | } 66 | } 67 | 68 | module.exports = function (pluginConfig) { 69 | return new Plugin(pluginConfig) 70 | } 71 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js#L12-L45) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Pass the Permissions Test. 16 | 17 | --- 18 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "mimeTypes": [ 3 | { 4 | "type": "application/pdf", 5 | "suffixes": "pdf", 6 | "description": "", 7 | "__pluginName": "Chrome PDF Viewer" 8 | }, 9 | { 10 | "type": "application/x-google-chrome-pdf", 11 | "suffixes": "pdf", 12 | "description": "Portable Document Format", 13 | "__pluginName": "Chrome PDF Plugin" 14 | }, 15 | { 16 | "type": "application/x-nacl", 17 | "suffixes": "", 18 | "description": "Native Client Executable", 19 | "__pluginName": "Native Client" 20 | }, 21 | { 22 | "type": "application/x-pnacl", 23 | "suffixes": "", 24 | "description": "Portable Native Client Executable", 25 | "__pluginName": "Native Client" 26 | } 27 | ], 28 | "plugins": [ 29 | { 30 | "name": "Chrome PDF Plugin", 31 | "filename": "internal-pdf-viewer", 32 | "description": "Portable Document Format", 33 | "__mimeTypes": ["application/x-google-chrome-pdf"] 34 | }, 35 | { 36 | "name": "Chrome PDF Viewer", 37 | "filename": "mhjfbmdgcfjbbpaeojofohoefgiehjai", 38 | "description": "", 39 | "__mimeTypes": ["application/pdf"] 40 | }, 41 | { 42 | "name": "Native Client", 43 | "filename": "internal-nacl-plugin", 44 | "description": "", 45 | "__mimeTypes": ["application/x-nacl", "application/x-pnacl"] 46 | } 47 | ] 48 | } 49 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/functionMocks.js: -------------------------------------------------------------------------------- 1 | /** 2 | * `navigator.{plugins,mimeTypes}` share similar custom functions to look up properties 3 | * 4 | * Note: This is meant to be run in the context of the page. 5 | */ 6 | module.exports.generateFunctionMocks = utils => ( 7 | proto, 8 | itemMainProp, 9 | dataArray 10 | ) => ({ 11 | /** Returns the MimeType object with the specified index. */ 12 | item: utils.createProxy(proto.item, { 13 | apply(target, ctx, args) { 14 | if (!args.length) { 15 | throw new TypeError( 16 | `Failed to execute 'item' on '${ 17 | proto[Symbol.toStringTag] 18 | }': 1 argument required, but only 0 present.` 19 | ) 20 | } 21 | // Special behavior alert: 22 | // - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup 23 | // - If anything else than an integer (including as string) is provided it will return the first entry 24 | const isInteger = args[0] && Number.isInteger(Number(args[0])) // Cast potential string to number first, then check for integer 25 | // Note: Vanilla never returns `undefined` 26 | return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null 27 | } 28 | }), 29 | /** Returns the MimeType object with the specified name. */ 30 | namedItem: utils.createProxy(proto.namedItem, { 31 | apply(target, ctx, args) { 32 | if (!args.length) { 33 | throw new TypeError( 34 | `Failed to execute 'namedItem' on '${ 35 | proto[Symbol.toStringTag] 36 | }': 1 argument required, but only 0 present.` 37 | ) 38 | } 39 | return dataArray.find(mt => mt[itemMainProp] === args[0]) || null // Not `undefined`! 40 | } 41 | }), 42 | /** Does nothing and shall return nothing */ 43 | refresh: proto.refresh 44 | ? utils.createProxy(proto.refresh, { 45 | apply(target, ctx, args) { 46 | return undefined 47 | } 48 | }) 49 | : undefined 50 | }) 51 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { 4 | getVanillaFingerPrint, 5 | getStealthFingerPrint 6 | } = require('../../test/util') 7 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 8 | 9 | const Plugin = require('.') 10 | 11 | test('vanilla: empty plugins, empty mimetypes', async t => { 12 | const { plugins, mimeTypes } = await getVanillaFingerPrint() 13 | t.is(plugins.length, 0) 14 | t.is(mimeTypes.length, 0) 15 | }) 16 | 17 | test('vanilla: will not have modifications', async t => { 18 | const browser = await vanillaPuppeteer.launch({ headless: true }) 19 | const page = await browser.newPage() 20 | 21 | const test1 = await page.evaluate(() => ({ 22 | mimeTypes: Object.getOwnPropertyDescriptor(navigator, 'mimeTypes'), // Must be undefined if native 23 | plugins: Object.getOwnPropertyDescriptor(navigator, 'plugins') // Must be undefined if native 24 | })) 25 | t.is(test1.mimeTypes, undefined) 26 | t.is(test1.plugins, undefined) 27 | 28 | const test2 = await page.evaluate( 29 | () => Object.getOwnPropertyNames(navigator) // Must be an empty array if native 30 | ) 31 | t.false(test2.includes('plugins')) 32 | }) 33 | 34 | test('stealth: has plugin, has mimetypes', async t => { 35 | const { plugins, mimeTypes } = await getStealthFingerPrint(Plugin) 36 | t.is(plugins.length, 3) 37 | t.is(mimeTypes.length, 4) 38 | }) 39 | 40 | test('stealth: will not leak modifications', async t => { 41 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 42 | const browser = await puppeteer.launch({ headless: true }) 43 | const page = await browser.newPage() 44 | 45 | const test1 = await page.evaluate(() => ({ 46 | mimeTypes: Object.getOwnPropertyDescriptor(navigator, 'mimeTypes'), // Must be undefined if native 47 | plugins: Object.getOwnPropertyDescriptor(navigator, 'plugins') // Must be undefined if native 48 | })) 49 | t.is(test1.mimeTypes, undefined) 50 | t.is(test1.plugins, undefined) 51 | 52 | const test2 = await page.evaluate( 53 | () => Object.getOwnPropertyNames(navigator) // Must be an empty array if native 54 | ) 55 | t.false(test2.includes('plugins')) 56 | }) 57 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/mimeTypes.js: -------------------------------------------------------------------------------- 1 | /* global MimeType MimeTypeArray */ 2 | 3 | /** 4 | * Generate a convincing and functional MimeTypeArray (with mime types) from scratch. 5 | * 6 | * Note: This is meant to be run in the context of the page. 7 | * 8 | * @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/mimeTypes 9 | * @see https://developer.mozilla.org/en-US/docs/Web/API/MimeTypeArray 10 | */ 11 | module.exports.generateMimeTypeArray = (utils, fns) => mimeTypesData => { 12 | return fns.generateMagicArray(utils, fns)( 13 | mimeTypesData, 14 | MimeTypeArray.prototype, 15 | MimeType.prototype, 16 | 'type' 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/plugins.js: -------------------------------------------------------------------------------- 1 | /* global Plugin PluginArray */ 2 | 3 | /** 4 | * Generate a convincing and functional PluginArray (with plugins) from scratch. 5 | * 6 | * Note: This is meant to be run in the context of the page. 7 | * 8 | * @see https://developer.mozilla.org/en-US/docs/Web/API/NavigatorPlugins/plugins 9 | * @see https://developer.mozilla.org/en-US/docs/Web/API/PluginArray 10 | */ 11 | module.exports.generatePluginArray = (utils, fns) => pluginsData => { 12 | return fns.generateMagicArray(utils, fns)( 13 | pluginsData, 14 | PluginArray.prototype, 15 | Plugin.prototype, 16 | 'name' 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js#L26-L88) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | In headless mode `navigator.mimeTypes` and `navigator.plugins` are empty. 16 | This plugin emulates both of these with functional mocks to match regular headful Chrome. 17 | 18 | Note: mimeTypes and plugins cross-reference each other, so it makes sense to do them at the same time. 19 | 20 | - **See: ** 21 | - **See: ** 22 | - **See: ** 23 | - **See: ** 24 | 25 | --- 26 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | const withUtils = require('../_utils/withUtils') 6 | 7 | /** 8 | * By default puppeteer will have a fixed `navigator.vendor` property. 9 | * 10 | * This plugin makes it possible to change this property. 11 | * 12 | * @example 13 | * const puppeteer = require("puppeteer-extra") 14 | * 15 | * const StealthPlugin = require("puppeteer-extra-plugin-stealth") 16 | * const stealth = StealthPlugin() 17 | * // Remove this specific stealth plugin from the default set 18 | * stealth.enabledEvasions.delete("navigator.vendor") 19 | * puppeteer.use(stealth) 20 | * 21 | * // Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such 22 | * const NavigatorVendorPlugin = require("puppeteer-extra-plugin-stealth/evasions/navigator.vendor") 23 | * const nvp = NavigatorVendorPlugin({ vendor: 'Apple Computer, Inc.' }) // Custom vendor 24 | * puppeteer.use(nvp) 25 | * 26 | * @param {Object} [opts] - Options 27 | * @param {string} [opts.vendor] - The vendor to use in `navigator.vendor` (default: `Google Inc.`) 28 | * 29 | */ 30 | class Plugin extends PuppeteerExtraPlugin { 31 | constructor(opts = {}) { 32 | super(opts) 33 | } 34 | 35 | get name() { 36 | return 'stealth/evasions/navigator.vendor' 37 | } 38 | 39 | get defaults() { 40 | return { 41 | vendor: 'Google Inc.' 42 | } 43 | } 44 | 45 | async onPageCreated(page) { 46 | this.debug('onPageCreated', { 47 | opts: this.opts 48 | }) 49 | 50 | await withUtils(page).evaluateOnNewDocument( 51 | (utils, { opts }) => { 52 | utils.replaceGetterWithProxy( 53 | Object.getPrototypeOf(navigator), 54 | 'vendor', 55 | utils.makeHandler().getterValue(opts.vendor) 56 | ) 57 | }, 58 | { 59 | opts: this.opts 60 | } 61 | ) 62 | } // onPageCreated 63 | } 64 | 65 | const defaultExport = opts => new Plugin(opts) 66 | module.exports = defaultExport 67 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 4 | const Plugin = require('.') 5 | 6 | test('vanilla: navigator.vendor is always Google Inc.', async t => { 7 | const browser = await vanillaPuppeteer.launch({ headless: true }) 8 | const page = await browser.newPage() 9 | 10 | const vendor = await page.evaluate(() => navigator.vendor) 11 | t.is(vendor, 'Google Inc.') 12 | }) 13 | 14 | test('stealth: navigator.vendor set to custom value', async t => { 15 | const puppeteer = addExtra(vanillaPuppeteer).use( 16 | Plugin({ vendor: 'Apple Computer, Inc.' }) 17 | ) 18 | const browser = await puppeteer.launch({ headless: true }) 19 | const page = await browser.newPage() 20 | 21 | const vendor = await page.evaluate(() => navigator.vendor) 22 | t.is(vendor, 'Apple Computer, Inc.') 23 | }) 24 | 25 | test('stealth: will not leak modifications', async t => { 26 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 27 | const browser = await puppeteer.launch({ headless: true }) 28 | const page = await browser.newPage() 29 | 30 | const test1 = await page.evaluate( 31 | () => Object.getOwnPropertyDescriptor(navigator, 'vendor') // Must be undefined if native 32 | ) 33 | t.is(test1, undefined) 34 | 35 | const test2 = await page.evaluate( 36 | () => Object.getOwnPropertyNames(navigator) // Must be an empty array if native 37 | ) 38 | t.false(test2.includes('vendor')) 39 | }) 40 | 41 | test('stealth: does patch getters properly', async t => { 42 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 43 | const browser = await puppeteer.launch({ headless: true }) 44 | const page = await browser.newPage() 45 | 46 | const results = await page.evaluate(() => { 47 | const hasInvocationError = (() => { 48 | try { 49 | // eslint-disable-next-line dot-notation 50 | Object['seal'](Object.getPrototypeOf(navigator)['vendor']) 51 | return false 52 | } catch (err) { 53 | return true 54 | } 55 | })() 56 | return { 57 | hasInvocationError, 58 | toString: Object.getOwnPropertyDescriptor( 59 | Object.getPrototypeOf(navigator), 60 | 'vendor' 61 | ).get.toString() 62 | } 63 | }) 64 | 65 | t.deepEqual(results, { 66 | hasInvocationError: true, 67 | toString: 'function get vendor() { [native code] }' 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/index.js#L28-L55) 10 | 11 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`) 12 | - `opts.vendor` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The vendor to use in `navigator.vendor` (default: `Google Inc.`) 13 | 14 | **Extends: PuppeteerExtraPlugin** 15 | 16 | By default puppeteer will have a fixed `navigator.vendor` property. 17 | 18 | This plugin makes it possible to change this property. 19 | 20 | Example: 21 | 22 | ```javascript 23 | const puppeteer = require('puppeteer-extra') 24 | 25 | const StealthPlugin = require('puppeteer-extra-plugin-stealth') 26 | const stealth = StealthPlugin() 27 | // Remove this specific stealth plugin from the default set 28 | stealth.enabledEvasions.delete('navigator.vendor') 29 | puppeteer.use(stealth) 30 | 31 | // Stealth plugins are just regular `puppeteer-extra` plugins and can be added as such 32 | const NavigatorVendorPlugin = require('puppeteer-extra-plugin-stealth/evasions/navigator.vendor') 33 | const nvp = NavigatorVendorPlugin({ vendor: 'Apple Computer, Inc.' }) // Custom vendor 34 | puppeteer.use(nvp) 35 | ``` 36 | 37 | --- 38 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | /** 6 | * Pass the Webdriver Test. 7 | * Will delete `navigator.webdriver` property. 8 | */ 9 | class Plugin extends PuppeteerExtraPlugin { 10 | constructor(opts = {}) { 11 | super(opts) 12 | } 13 | 14 | get name() { 15 | return 'stealth/evasions/navigator.webdriver' 16 | } 17 | 18 | async onPageCreated(page) { 19 | await page.evaluateOnNewDocument(() => { 20 | if (navigator.webdriver === false) { 21 | // Post Chrome 89.0.4339.0 and already good 22 | } else if (navigator.webdriver === undefined) { 23 | // Pre Chrome 89.0.4339.0 and already good 24 | } else { 25 | // Pre Chrome 88.0.4291.0 and needs patching 26 | delete Object.getPrototypeOf(navigator).webdriver 27 | } 28 | }) 29 | } 30 | 31 | // Post Chrome 88.0.4291.0 32 | // Note: this will add an infobar to Chrome with a warning that an unsupported flag is set 33 | // To remove this bar on Linux, run: mkdir -p /etc/opt/chrome/policies/managed && echo '{ "CommandLineFlagSecurityWarningsEnabled": false }' > /etc/opt/chrome/policies/managed/managed_policies.json 34 | async beforeLaunch(options) { 35 | // If disable-blink-features is already passed, append the AutomationControlled switch 36 | const idx = options.args.findIndex((arg) => arg.startsWith('--disable-blink-features=')); 37 | if (idx !== -1) { 38 | const arg = options.args[idx]; 39 | options.args[idx] = `${arg},AutomationControlled`; 40 | } else { 41 | options.args.push('--disable-blink-features=AutomationControlled'); 42 | } 43 | } 44 | } 45 | 46 | module.exports = function(pluginConfig) { 47 | return new Plugin(pluginConfig) 48 | } 49 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { vanillaPuppeteer, addExtra, compareLooseVersionStrings } = require('../../test/util') 4 | const Plugin = require('.') 5 | 6 | function getExpectedValue(looseVersionString) { 7 | if (compareLooseVersionStrings(looseVersionString, '89.0.4339.0') >= 0) { 8 | return false 9 | } else { 10 | return undefined 11 | } 12 | } 13 | 14 | test('vanilla: navigator.webdriver is defined', async t => { 15 | const browser = await vanillaPuppeteer.launch({ headless: true }) 16 | const page = await browser.newPage() 17 | 18 | const data = await page.evaluate(() => navigator.webdriver) 19 | t.is(data, true) 20 | }) 21 | 22 | test('stealth: navigator.webdriver is undefined', async t => { 23 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 24 | const browser = await puppeteer.launch({ headless: true }) 25 | const page = await browser.newPage() 26 | 27 | const data = await page.evaluate(() => navigator.webdriver) 28 | // XXX: launch this test multiple times with browsers of different versions? 29 | t.is(data, getExpectedValue(await browser.version())) 30 | }) 31 | 32 | // https://github.com/berstend/puppeteer-extra/pull/130 33 | test('stealth: regression: wont kill other navigator methods', async t => { 34 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 35 | const browser = await puppeteer.launch({ headless: true }) 36 | const page = await browser.newPage() 37 | 38 | try { 39 | const data = await page.evaluate(() => navigator.javaEnabled()) 40 | t.is(data, false) 41 | } catch (err) { 42 | t.is(err, undefined) 43 | } 44 | }) 45 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/index.js#L9-L23) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Pass the Webdriver Test. 16 | Will delete `navigator.webdriver` property. 17 | 18 | --- 19 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/readme.md: -------------------------------------------------------------------------------- 1 | # puppeteer-extra-plugin-stealth/evasions 2 | 3 | Various detection evasion plugins for `puppeteer-extra-plugin-stealth`. 4 | 5 | You can bypass the main module and require specific evasion plugins yourself, if you wish to do so: 6 | 7 | ```es6 8 | puppeteer.use( 9 | require('puppeteer-extra-plugin-stealth/evasions/console.debug')() 10 | ) 11 | ``` 12 | 13 | If you want to add a new evasion technique I suggest you look at the [template](./_template/) to kickstart things. 14 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/_fixtures/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Page Title 6 | 7 | 8 | 9 |

Please use `document.querySelector`..

10 | 11 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { vanillaPuppeteer, addExtra } = require('../../test/util') 4 | const Plugin = require('.') 5 | 6 | const TEST_HTML_FILE = require('path').join(__dirname, './_fixtures/test.html') 7 | 8 | test('vanilla: sourceurl is leaking', async t => { 9 | const browser = await vanillaPuppeteer.launch({ headless: true }) 10 | const page = await browser.newPage() 11 | await page.goto('file://' + TEST_HTML_FILE, { waitUntil: 'load' }) 12 | 13 | // Trigger test 14 | await page.$('title') 15 | 16 | const result = await page.evaluate( 17 | () => document.querySelector('#result').innerText 18 | ) 19 | t.is(result, 'FAIL') 20 | 21 | const result2 = await page.evaluate(() => { 22 | try { 23 | Function.prototype.toString.apply({}) 24 | } catch (err) { 25 | return err.stack 26 | } 27 | }) 28 | t.true(result2.includes('__puppeteer_evaluation_script')) 29 | }) 30 | 31 | test('stealth: sourceurl is not leaking', async t => { 32 | const puppeteer = addExtra(vanillaPuppeteer).use(Plugin()) 33 | const browser = await puppeteer.launch({ headless: true }) 34 | const page = await browser.newPage() 35 | 36 | await page.goto('file://' + TEST_HTML_FILE, { waitUntil: 'load' }) 37 | 38 | // Trigger test 39 | await page.$('title') 40 | 41 | const result = await page.evaluate( 42 | () => document.querySelector('#result').innerText 43 | ) 44 | t.is(result, 'PASS') 45 | 46 | const result2 = await page.evaluate(() => { 47 | try { 48 | Function.prototype.toString.apply({}) 49 | } catch (err) { 50 | return err.stack 51 | } 52 | }) 53 | t.false(result2.includes('__puppeteer_evaluation_script')) 54 | }) 55 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.js#L9-L58) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Strip sourceURL from scripts injected by puppeteer. 16 | It can be used to identify the presence of pptr via stacktraces. 17 | 18 | --- 19 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/user-agent-override/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | const withUtils = require('../_utils/withUtils') 6 | 7 | /** 8 | * Fix WebGL Vendor/Renderer being set to Google in headless mode 9 | * 10 | * Example data (Apple Retina MBP 13): {vendor: "Intel Inc.", renderer: "Intel(R) Iris(TM) Graphics 6100"} 11 | * 12 | * @param {Object} [opts] - Options 13 | * @param {string} [opts.vendor] - The vendor string to use (default: `Intel Inc.`) 14 | * @param {string} [opts.renderer] - The renderer string (default: `Intel Iris OpenGL Engine`) 15 | */ 16 | class Plugin extends PuppeteerExtraPlugin { 17 | constructor(opts = {}) { 18 | super(opts) 19 | } 20 | 21 | get name() { 22 | return 'stealth/evasions/webgl.vendor' 23 | } 24 | 25 | /* global WebGLRenderingContext WebGL2RenderingContext */ 26 | async onPageCreated(page) { 27 | await withUtils(page).evaluateOnNewDocument((utils, opts) => { 28 | const getParameterProxyHandler = { 29 | apply: function(target, ctx, args) { 30 | const param = (args || [])[0] 31 | const result = utils.cache.Reflect.apply(target, ctx, args) 32 | // UNMASKED_VENDOR_WEBGL 33 | if (param === 37445) { 34 | return opts.vendor || 'Intel Inc.' // default in headless: Google Inc. 35 | } 36 | // UNMASKED_RENDERER_WEBGL 37 | if (param === 37446) { 38 | return opts.renderer || 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader 39 | } 40 | return result 41 | } 42 | } 43 | 44 | // There's more than one WebGL rendering context 45 | // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility 46 | // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter) 47 | const addProxy = (obj, propName) => { 48 | utils.replaceWithProxy(obj, propName, getParameterProxyHandler) 49 | } 50 | // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing: 51 | addProxy(WebGLRenderingContext.prototype, 'getParameter') 52 | addProxy(WebGL2RenderingContext.prototype, 'getParameter') 53 | }, this.opts) 54 | } 55 | } 56 | 57 | module.exports = function(pluginConfig) { 58 | return new Plugin(pluginConfig) 59 | } 60 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/index.js#L17-L55) 10 | 11 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)?** Options (optional, default `{}`) 12 | - `opts.vendor` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The vendor string to use (default: `Intel Inc.`) 13 | - `opts.renderer` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** The renderer string (default: `Intel Iris OpenGL Engine`) 14 | 15 | **Extends: PuppeteerExtraPlugin** 16 | 17 | Fix WebGL Vendor/Renderer being set to Google in headless mode 18 | 19 | Example data (Apple Retina MBP 13): {vendor: "Intel Inc.", renderer: "Intel(R) Iris(TM) Graphics 6100"} 20 | 21 | --- 22 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/window.outerdimensions/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 4 | 5 | /** 6 | * Fix missing window.outerWidth/window.outerHeight in headless mode 7 | * Will also set the viewport to match window size, unless specified by user 8 | */ 9 | class Plugin extends PuppeteerExtraPlugin { 10 | constructor(opts = {}) { 11 | super(opts) 12 | } 13 | 14 | get name() { 15 | return 'stealth/evasions/window.outerdimensions' 16 | } 17 | 18 | async onPageCreated(page) { 19 | // Chrome returns undefined, Firefox false 20 | await page.evaluateOnNewDocument(() => { 21 | try { 22 | if (window.outerWidth && window.outerHeight) { 23 | return // nothing to do here 24 | } 25 | const windowFrame = 85 // probably OS and WM dependent 26 | window.outerWidth = window.innerWidth 27 | window.outerHeight = window.innerHeight + windowFrame 28 | } catch (err) {} 29 | }) 30 | } 31 | 32 | async beforeLaunch(options) { 33 | // Have viewport match window size, unless specified by user 34 | // https://github.com/GoogleChrome/puppeteer/issues/3688 35 | if (!('defaultViewport' in options)) { 36 | options.defaultViewport = null 37 | } 38 | return options 39 | } 40 | } 41 | 42 | module.exports = function(pluginConfig) { 43 | return new Plugin(pluginConfig) 44 | } 45 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/window.outerdimensions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "main": "index.js" 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/evasions/window.outerdimensions/readme.md: -------------------------------------------------------------------------------- 1 | ## API 2 | 3 | 4 | 5 | #### Table of Contents 6 | 7 | - [class: Plugin](#class-plugin) 8 | 9 | ### class: [Plugin](https://github.com/berstend/puppeteer-extra/blob/e6133619b051febed630ada35241664eba59b9fa/packages/puppeteer-extra-plugin-stealth/evasions/window.outerdimensions/index.js#L9-L40) 10 | 11 | - `opts` (optional, default `{}`) 12 | 13 | **Extends: PuppeteerExtraPlugin** 14 | 15 | Fix missing window.outerWidth/window.outerHeight in headless mode 16 | Will also set the viewport to match window size, unless specified by user 17 | 18 | --- 19 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/examples/test1.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const puppeteer = require('puppeteer-extra') 4 | puppeteer.use(require('puppeteer-extra-plugin-stealth')()) 5 | 6 | const detectHeadless = require('./detect-headless') 7 | 8 | ;(async () => { 9 | const browser = await puppeteer.launch({ args: ['--no-sandbox'] }) 10 | const page = await browser.newPage() 11 | page.on('console', msg => { 12 | console.log('Page console: ', msg.text()) 13 | }) 14 | 15 | await page.goto('about:blank') 16 | const detectionResults = await page.evaluate(detectHeadless) 17 | console.assert( 18 | Object.keys(detectionResults).length, 19 | 'No detection results returned.' 20 | ) 21 | 22 | await browser.close() 23 | })() 24 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/examples/test2.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const puppeteer = require('puppeteer-extra') 4 | // Enable stealth plugin 5 | puppeteer.use(require('puppeteer-extra-plugin-stealth')()) 6 | ;(async () => { 7 | // Launch the browser in headless mode and set up a page. 8 | const browser = await puppeteer.launch({ 9 | args: ['--no-sandbox'], 10 | headless: true 11 | }) 12 | const page = await browser.newPage() 13 | 14 | // Navigate to the page that will perform the tests. 15 | const testUrl = 16 | 'https://intoli.com/blog/' + 17 | 'not-possible-to-block-chrome-headless/chrome-headless-test.html' 18 | await page.goto(testUrl) 19 | 20 | // Save a screenshot of the results. 21 | const screenshotPath = '/tmp/headless-test-result.png' 22 | await page.screenshot({ path: screenshotPath }) 23 | console.log('have a look at the screenshot:', screenshotPath) 24 | 25 | await browser.close() 26 | })() 27 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/index.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const PLUGIN_NAME = 'stealth' 4 | 5 | const test = require('ava') 6 | 7 | const Plugin = require('.') 8 | 9 | test('is a function', async t => { 10 | t.is(typeof Plugin, 'function') 11 | }) 12 | 13 | test('should have the basic class members', async t => { 14 | const instance = Plugin() 15 | t.is(instance.name, PLUGIN_NAME) 16 | t.true(instance._isPuppeteerExtraPlugin) 17 | }) 18 | 19 | test('should have the public child class members', async t => { 20 | const instance = Plugin() 21 | const prototype = Object.getPrototypeOf(instance) 22 | const childClassMembers = Object.getOwnPropertyNames(prototype) 23 | 24 | t.true(childClassMembers.includes('constructor')) 25 | t.true(childClassMembers.includes('name')) 26 | t.true(childClassMembers.includes('name')) 27 | t.true(childClassMembers.includes('defaults')) 28 | t.true(childClassMembers.includes('availableEvasions')) 29 | t.true(childClassMembers.includes('enabledEvasions')) 30 | t.is(childClassMembers.length, 7) 31 | }) 32 | 33 | test('should have opts with default values', async t => { 34 | const instance = Plugin() 35 | t.deepEqual(instance.opts.enabledEvasions, instance.availableEvasions) 36 | }) 37 | 38 | test('should add all dependencies dynamically', async t => { 39 | const instance = Plugin() 40 | const deps = new Set( 41 | [...instance.opts.enabledEvasions].map(e => `${PLUGIN_NAME}/evasions/${e}`) 42 | ) 43 | t.deepEqual(instance.dependencies, deps) 44 | }) 45 | 46 | test('should add all dependencies dynamically including changes', async t => { 47 | const instance = Plugin() 48 | const fakeDep = 'foobar' 49 | instance.enabledEvasions = new Set([fakeDep]) 50 | t.deepEqual( 51 | instance.dependencies, 52 | new Set([`${PLUGIN_NAME}/evasions/${fakeDep}`]) 53 | ) 54 | }) 55 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-stealth", 3 | "version": "2.11.2", 4 | "description": "Stealth mode: Applies various techniques to make detection of headless puppeteer harder.", 5 | "main": "index.js", 6 | "typings": "index.d.ts", 7 | "repository": "berstend/puppeteer-extra", 8 | "homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth#readme", 9 | "author": "berstend", 10 | "license": "MIT", 11 | "scripts": { 12 | "docs": "run-s docs-for-plugin postdocs-for-plugin docs-for-evasions postdocs-for-evasions types", 13 | "docs-for-plugin": "documentation readme --quiet --shallow --github --markdown-theme transitivebs --readme-file readme.md --section API index.js", 14 | "postdocs-for-plugin": "npx prettier --write readme.md", 15 | "docs-for-evasions": "cd ./evasions && loop \"documentation readme --quiet --shallow --github --markdown-theme transitivebs --readme-file readme.md --section API index.js\"", 16 | "postdocs-for-evasions": "cd ./evasions && loop \"npx prettier --write readme.md\"", 17 | "lint": "eslint --ext .js .", 18 | "test:js": "ava --concurrency 2 -v", 19 | "test": "run-p test:js", 20 | "test-ci": "run-s test:js", 21 | "types": "npx --package typescript@3.7 tsc --emitDeclarationOnly --declaration --allowJs index.js" 22 | }, 23 | "engines": { 24 | "node": ">=8" 25 | }, 26 | "keywords": [ 27 | "puppeteer", 28 | "puppeteer-extra", 29 | "puppeteer-extra-plugin", 30 | "stealth", 31 | "stealth-mode", 32 | "detection-evasion", 33 | "crawler", 34 | "chrome", 35 | "headless", 36 | "pupeteer" 37 | ], 38 | "ava": { 39 | "files": [ 40 | "!test/util.js", 41 | "!test/fixtures/sw.js" 42 | ] 43 | }, 44 | "devDependencies": { 45 | "ava": "2.4.0", 46 | "documentation-markdown-themes": "^12.1.5", 47 | "fpcollect": "^1.0.4", 48 | "fpscanner": "^0.1.5", 49 | "loop": "^3.0.6", 50 | "npm-run-all": "^4.1.5", 51 | "puppeteer": "9" 52 | }, 53 | "dependencies": { 54 | "debug": "^4.1.1", 55 | "puppeteer-extra-plugin": "^3.2.3", 56 | "puppeteer-extra-plugin-user-preferences": "^2.4.1" 57 | }, 58 | "peerDependencies": { 59 | "playwright-extra": "*", 60 | "puppeteer-extra": "*" 61 | }, 62 | "peerDependenciesMeta": { 63 | "puppeteer-extra": { 64 | "optional": true 65 | }, 66 | "playwright-extra": { 67 | "optional": true 68 | } 69 | }, 70 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 71 | } 72 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/runall_stealthtests.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | echo "Cleanup.." 4 | rm -f ./stealthtests/_results/*.png 5 | rm -f ./stealthtests/_results/_thumbs/*.png 6 | 7 | echo "Running scripts.." 8 | FILES=`find ./stealthtests -type f -name '*.js'` 9 | for file in $FILES 10 | do 11 | node $file 12 | done 13 | 14 | echo "Making thumbnails.." 15 | cp ./stealthtests/_results/*.png ./stealthtests/_results/_thumbs 16 | # Note: MacOS specific image resizing command 17 | sips -Z 640 ./stealthtests/_results/_thumbs/*.png 18 | 19 | echo "All done." 20 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chrome-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chrome-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chrome-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chrome-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chromium-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chromium-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chromium-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headful-chromium-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chrome-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chrome-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chrome-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chrome-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chromium-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chromium-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chromium-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/_thumbs/headless-chromium-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chrome-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chrome-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chrome-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chrome-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chromium-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chromium-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chromium-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headful-chromium-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chrome-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chrome-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chrome-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chrome-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chromium-stealth.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chromium-stealth.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chromium-vanilla.js.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berstend/puppeteer-extra/39248f1f5deeb21b1e7eb6ae07b8ef73f1231ab9/packages/puppeteer-extra-plugin-stealth/stealthtests/_results/headless-chromium-vanilla.js.png -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chrome-stealth.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer-extra') 6 | const pluginStealth = require('puppeteer-extra-plugin-stealth')() 7 | puppeteer.use(pluginStealth) 8 | 9 | async function main() { 10 | console.log('start', scriptName) 11 | const browser = await puppeteer.launch({ 12 | headless: false, 13 | executablePath: `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome` // MacOS 14 | }) 15 | 16 | const page = await browser.newPage() 17 | await page.setViewport({ width: 800, height: 600 }) 18 | await page.goto('https://bot.sannysoft.com/') 19 | await page.waitForTimeout(5000) 20 | await page.screenshot({ path: screenshotPath, fullPage: true }) 21 | 22 | await browser.close() 23 | console.log('end', screenshotPath) 24 | } 25 | main() 26 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chrome-vanilla.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer') 6 | 7 | async function main() { 8 | console.log('start', scriptName) 9 | const browser = await puppeteer.launch({ 10 | headless: false, 11 | executablePath: `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome` // MacOS 12 | }) 13 | 14 | const page = await browser.newPage() 15 | await page.setViewport({ width: 800, height: 600 }) 16 | await page.goto('https://bot.sannysoft.com/') 17 | await page.waitForTimeout(5000) 18 | await page.screenshot({ path: screenshotPath, fullPage: true }) 19 | 20 | await browser.close() 21 | console.log('end', screenshotPath) 22 | } 23 | main() 24 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chromium-stealth.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer-extra') 6 | const pluginStealth = require('puppeteer-extra-plugin-stealth') 7 | 8 | async function main() { 9 | puppeteer.use(pluginStealth()) 10 | console.log('start', scriptName) 11 | const browser = await puppeteer.launch({ headless: false }) 12 | 13 | const page = await browser.newPage() 14 | await page.setViewport({ width: 800, height: 600 }) 15 | await page.goto('https://bot.sannysoft.com/') 16 | await page.waitForTimeout(5000) 17 | await page.screenshot({ path: screenshotPath, fullPage: true }) 18 | 19 | await browser.close() 20 | console.log('end', screenshotPath) 21 | } 22 | main() 23 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chromium-vanilla.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer') 6 | 7 | async function main() { 8 | console.log('start', scriptName) 9 | const browser = await puppeteer.launch({ headless: false }) 10 | 11 | const page = await browser.newPage() 12 | await page.setViewport({ width: 800, height: 600 }) 13 | await page.goto('https://bot.sannysoft.com/') 14 | await page.waitForTimeout(5000) 15 | await page.screenshot({ path: screenshotPath, fullPage: true }) 16 | 17 | await browser.close() 18 | console.log('end', screenshotPath) 19 | } 20 | main() 21 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chrome-stealth.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer-extra') 6 | const pluginStealth = require('puppeteer-extra-plugin-stealth')() 7 | puppeteer.use(pluginStealth) 8 | 9 | async function main() { 10 | console.log('start', scriptName) 11 | const browser = await puppeteer.launch({ 12 | headless: true, 13 | executablePath: `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome` // MacOS 14 | }) 15 | 16 | const page = await browser.newPage() 17 | await page.setViewport({ width: 800, height: 600 }) 18 | await page.goto('https://bot.sannysoft.com/') 19 | await page.waitForTimeout(5000) 20 | await page.screenshot({ path: screenshotPath, fullPage: true }) 21 | 22 | await browser.close() 23 | console.log('end', screenshotPath) 24 | } 25 | main() 26 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chrome-vanilla.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer') 6 | 7 | async function main() { 8 | console.log('start', scriptName) 9 | const browser = await puppeteer.launch({ 10 | headless: true, 11 | executablePath: `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome` // MacOS 12 | }) 13 | 14 | const page = await browser.newPage() 15 | await page.setViewport({ width: 800, height: 600 }) 16 | await page.goto('https://bot.sannysoft.com/') 17 | await page.waitForTimeout(5000) 18 | await page.screenshot({ path: screenshotPath, fullPage: true }) 19 | 20 | await browser.close() 21 | console.log('end', screenshotPath) 22 | } 23 | main() 24 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chromium-stealth.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer-extra') 6 | const pluginStealth = require('puppeteer-extra-plugin-stealth')() 7 | puppeteer.use(pluginStealth) 8 | 9 | async function main() { 10 | console.log('start', scriptName) 11 | const browser = await puppeteer.launch({ headless: true }) 12 | 13 | const page = await browser.newPage() 14 | await page.setViewport({ width: 800, height: 600 }) 15 | await page.goto('https://bot.sannysoft.com/') 16 | await page.waitForTimeout(5000) 17 | await page.screenshot({ path: screenshotPath, fullPage: true }) 18 | 19 | await browser.close() 20 | console.log('end', screenshotPath) 21 | } 22 | main() 23 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chromium-vanilla.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const scriptName = path.basename(__filename) 3 | const screenshotPath = path.join(__dirname, '_results', `${scriptName}.png`) 4 | 5 | const puppeteer = require('puppeteer') 6 | 7 | async function main() { 8 | console.log('start', scriptName) 9 | const browser = await puppeteer.launch({ headless: true }) 10 | 11 | const page = await browser.newPage() 12 | await page.setViewport({ width: 800, height: 600 }) 13 | await page.goto('https://bot.sannysoft.com/') 14 | await page.waitForTimeout(5000) 15 | await page.screenshot({ path: screenshotPath, fullPage: true }) 16 | 17 | await browser.close() 18 | console.log('end', screenshotPath) 19 | } 20 | main() 21 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/test/fixtures/dummy-with-service-worker.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | title foo 6 | 7 | 18 | 19 | 20 |

Test page with service worker

21 | 22 | 23 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/test/fixtures/dummy.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | title foo 6 | 7 | 8 | 9 |

Test page

10 | 11 | 12 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/test/fixtures/sw.js: -------------------------------------------------------------------------------- 1 | // Left empty 2 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/test/fpscanner.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const fpscanner = require('fpscanner') 4 | 5 | const { getVanillaFingerPrint, getStealthFingerPrint, compareLooseVersionStrings } = require('./util') 6 | const Plugin = require('../.') 7 | 8 | // Fix CI issues with old versions 9 | const isOldPuppeteerVersion = () => { 10 | const version = process.env.PUPPETEER_VERSION 11 | if (!version) { 12 | return false 13 | } 14 | if (version === '1.9.0' || version === '1.6.2') { 15 | return true 16 | } 17 | return false 18 | } 19 | 20 | test('vanilla: will fail multiple fpscanner tests', async t => { 21 | const fingerPrint = await getVanillaFingerPrint() 22 | const testedFingerPrints = fpscanner.analyseFingerprint(fingerPrint) 23 | const failedChecks = Object.values(testedFingerPrints).filter( 24 | val => val.consistent < 3 25 | ) 26 | 27 | if (isOldPuppeteerVersion()) { 28 | t.is(failedChecks.length, 8) 29 | } else { 30 | t.is(failedChecks.length, 7) 31 | } 32 | }) 33 | 34 | test('stealth: will not fail a single fpscanner test', async t => { 35 | const fingerPrint = await getStealthFingerPrint(Plugin) 36 | const testedFingerPrints = fpscanner.analyseFingerprint(fingerPrint) 37 | const failedChecks = Object.values(testedFingerPrints).filter( 38 | val => val.consistent < 3 39 | ) 40 | 41 | if (failedChecks.length) { 42 | console.warn('The following fingerprints failed:', failedChecks) 43 | } 44 | 45 | if (compareLooseVersionStrings(fingerPrint.userAgent, '89.0.4339.0') >= 0) { 46 | // Updated navigator.webdriver behavior breaks the fpscanner tests. 47 | t.is(failedChecks.length, 1) 48 | t.is(failedChecks[0].name, 'WEBDRIVER') 49 | } else { 50 | t.is(failedChecks.length, 0) 51 | } 52 | }) 53 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-stealth/test/util.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert') 2 | const vanillaPuppeteer = require('puppeteer') 3 | const { addExtra } = require('puppeteer-extra') 4 | 5 | const fpCollectPath = require.resolve('fpcollect/dist/fpCollect.min.js') 6 | 7 | const getFingerPrintFromPage = async page => { 8 | return page.evaluate(() => fpCollect.generateFingerprint()) // eslint-disable-line 9 | } 10 | 11 | const dummyHTMLPath = require('path').join(__dirname, './fixtures/dummy.html') 12 | 13 | const getFingerPrint = async (puppeteer, pageFn) => { 14 | const browser = await puppeteer.launch({ headless: true }) 15 | const page = await browser.newPage() 16 | await page.goto('file://' + dummyHTMLPath) 17 | await page.addScriptTag({ path: fpCollectPath }) 18 | const fingerPrint = await getFingerPrintFromPage(page) 19 | 20 | let pageFnResult = null 21 | if (pageFn) { 22 | pageFnResult = await pageFn(page) 23 | } 24 | 25 | await browser.close() 26 | return { ...fingerPrint, pageFnResult } 27 | } 28 | 29 | const getVanillaFingerPrint = async pageFn => 30 | getFingerPrint(vanillaPuppeteer, pageFn) 31 | const getStealthFingerPrint = async (Plugin, pageFn, pluginOptions = null) => 32 | getFingerPrint(addExtra(vanillaPuppeteer).use(Plugin(pluginOptions)), pageFn) 33 | 34 | // Expecting the input string to be in one of these formats: 35 | // - The UA string 36 | // - The shorter version string from Puppeteers browser.version() 37 | // - The shortest four-integer string 38 | const parseLooseVersionString = looseVersionString => looseVersionString 39 | .match(/(\d+\.){3}\d+/)[0] 40 | .split('.') 41 | .map(x => parseInt(x)) 42 | 43 | const compareLooseVersionStrings = (version0, version1) => { 44 | const parsed0 = parseLooseVersionString(version0) 45 | const parsed1 = parseLooseVersionString(version1) 46 | assert(parsed0.length == 4) 47 | assert(parsed1.length == 4) 48 | for (let i = 0; i < parsed0.length; i++) { 49 | if (parsed0[i] < parsed1[i]) { 50 | return -1 51 | } else if (parsed0[i] > parsed1[i]) { 52 | return 1 53 | } 54 | } 55 | return 0 56 | } 57 | 58 | module.exports = { 59 | getVanillaFingerPrint, 60 | getStealthFingerPrint, 61 | dummyHTMLPath, 62 | vanillaPuppeteer, 63 | addExtra, 64 | compareLooseVersionStrings 65 | } 66 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-user-data-dir/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-user-data-dir", 3 | "version": "2.4.1", 4 | "description": "Custom user data directory for puppeteer.", 5 | "main": "index.js", 6 | "repository": "berstend/puppeteer-extra", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "scripts": { 10 | "docs": "node -e 0", 11 | "lint": "eslint --ext .js .", 12 | "test": "run-p lint", 13 | "test-ci": "run-s test" 14 | }, 15 | "engines": { 16 | "node": ">=8" 17 | }, 18 | "keywords": [ 19 | "puppeteer", 20 | "puppeteer-extra", 21 | "puppeteer-extra-plugin", 22 | "user-data", 23 | "userDataDir", 24 | "profile", 25 | "chrome", 26 | "headless", 27 | "pupeteer" 28 | ], 29 | "devDependencies": { 30 | "ava": "2.4.0", 31 | "npm-run-all": "^4.1.5", 32 | "puppeteer": "^2.0.0" 33 | }, 34 | "dependencies": { 35 | "debug": "^4.1.1", 36 | "fs-extra": "^10.0.0", 37 | "puppeteer-extra-plugin": "^3.2.3", 38 | "rimraf": "^3.0.2" 39 | }, 40 | "peerDependencies": { 41 | "playwright-extra": "*", 42 | "puppeteer-extra": "*" 43 | }, 44 | "peerDependenciesMeta": { 45 | "puppeteer-extra": { 46 | "optional": true 47 | }, 48 | "playwright-extra": { 49 | "optional": true 50 | } 51 | }, 52 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 53 | } 54 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-user-data-dir/readme.md: -------------------------------------------------------------------------------- 1 | # puppeteer-extra-plugin-user-data-dir 2 | 3 | > A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra). 4 | 5 | ### Install 6 | 7 | ```bash 8 | yarn add puppeteer-extra-plugin-user-data-dir 9 | ``` 10 | 11 | ## API 12 | 13 | 14 | 15 | #### Table of Contents 16 | 17 | - [Plugin](#plugin) 18 | 19 | ### [Plugin](https://github.com/berstend/puppeteer-extra/blob/db57ea66cf10d407cf63af387892492e495a84f2/packages/puppeteer-extra-plugin-user-data-dir/index.js#L19-L113) 20 | 21 | **Extends: PuppeteerExtraPlugin** 22 | 23 | Further reading: 24 | 25 | 26 | Type: `function (opts)` 27 | 28 | - `opts` (optional, default `{}`) 29 | 30 | * * * 31 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-user-preferences/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const merge = require('deepmerge') 4 | 5 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 6 | 7 | /** 8 | * Launch puppeteer with arbitrary user preferences. 9 | * 10 | * The user defined preferences will be merged with preferences set by other plugins. 11 | * Plugins can add user preferences by exposing a data entry with the name `userPreferences`. 12 | * 13 | * Overview: 14 | * https://chromium.googlesource.com/chromium/src/+/master/chrome/common/pref_names.cc 15 | * 16 | * @param {Object} opts - Options 17 | * @param {Object} [opts.userPrefs={}] - An object containing the preferences. 18 | * 19 | * @example 20 | * const puppeteer = require('puppeteer-extra') 21 | * puppeteer.use(require('puppeteer-extra-plugin-user-preferences')({userPrefs: { 22 | * webkit: { 23 | * webprefs: { 24 | * default_font_size: 22 25 | * } 26 | * } 27 | * }})) 28 | * const browser = await puppeteer.launch() 29 | */ 30 | class Plugin extends PuppeteerExtraPlugin { 31 | constructor(opts = {}) { 32 | super(opts) 33 | this._userPrefsFromPlugins = {} 34 | 35 | const defaults = { 36 | userPrefs: {} 37 | } 38 | 39 | this._opts = Object.assign(defaults, opts) 40 | } 41 | 42 | get name() { 43 | return 'user-preferences' 44 | } 45 | 46 | get requirements() { 47 | return new Set(['runLast', 'dataFromPlugins']) 48 | } 49 | 50 | get dependencies() { 51 | return new Set(['user-data-dir']) 52 | } 53 | 54 | get data() { 55 | return [ 56 | { 57 | name: 'userDataDirFile', 58 | value: { 59 | target: 'Profile', 60 | file: 'Preferences', 61 | contents: JSON.stringify(this.combinedPrefs, null, 2) 62 | } 63 | } 64 | ] 65 | } 66 | 67 | get combinedPrefs() { 68 | return merge(this._opts.userPrefs, this._userPrefsFromPlugins) 69 | } 70 | 71 | async beforeLaunch(options) { 72 | this._userPrefsFromPlugins = merge.all( 73 | this.getDataFromPlugins('userPreferences').map(d => d.value) 74 | ) 75 | this.debug('_userPrefsFromPlugins', this._userPrefsFromPlugins) 76 | } 77 | } 78 | 79 | module.exports = function(pluginConfig) { 80 | return new Plugin(pluginConfig) 81 | } 82 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-user-preferences/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin-user-preferences", 3 | "version": "2.4.1", 4 | "description": "Launch puppeteer with arbitrary user preferences.", 5 | "main": "index.js", 6 | "repository": "berstend/puppeteer-extra", 7 | "author": "berstend", 8 | "license": "MIT", 9 | "scripts": { 10 | "docs": "node -e 0", 11 | "lint": "eslint --ext .js .", 12 | "test": "run-p lint", 13 | "test-ci": "run-s test" 14 | }, 15 | "engines": { 16 | "node": ">=8" 17 | }, 18 | "keywords": [ 19 | "puppeteer", 20 | "puppeteer-extra", 21 | "puppeteer-extra-plugin", 22 | "user-prefs", 23 | "user-preferences", 24 | "chrome", 25 | "headless", 26 | "pupeteer" 27 | ], 28 | "devDependencies": { 29 | "ava": "2.4.0", 30 | "npm-run-all": "^4.1.5", 31 | "puppeteer": "^2.0.0" 32 | }, 33 | "dependencies": { 34 | "debug": "^4.1.1", 35 | "deepmerge": "^4.2.2", 36 | "puppeteer-extra-plugin": "^3.2.3", 37 | "puppeteer-extra-plugin-user-data-dir": "^2.4.1" 38 | }, 39 | "peerDependencies": { 40 | "playwright-extra": "*", 41 | "puppeteer-extra": "*" 42 | }, 43 | "peerDependenciesMeta": { 44 | "puppeteer-extra": { 45 | "optional": true 46 | }, 47 | "playwright-extra": { 48 | "optional": true 49 | } 50 | }, 51 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 52 | } 53 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin-user-preferences/readme.md: -------------------------------------------------------------------------------- 1 | # puppeteer-extra-plugin-user-preferences 2 | 3 | > A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra). 4 | 5 | ### Install 6 | 7 | ```bash 8 | yarn add puppeteer-extra-plugin-user-preferences 9 | ``` 10 | 11 | ## API 12 | 13 | 14 | 15 | #### Table of Contents 16 | 17 | - [Plugin](#plugin) 18 | 19 | ### [Plugin](https://github.com/berstend/puppeteer-extra/blob/db57ea66cf10d407cf63af387892492e495a84f2/packages/puppeteer-extra-plugin-user-preferences/index.js#L30-L73) 20 | 21 | **Extends: PuppeteerExtraPlugin** 22 | 23 | Launch puppeteer with arbitrary user preferences. 24 | 25 | The user defined preferences will be merged with preferences set by other plugins. 26 | Plugins can add user preferences by exposing a data entry with the name `userPreferences`. 27 | 28 | Overview: 29 | 30 | 31 | Type: `function (opts)` 32 | 33 | - `opts` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** Options (optional, default `{}`) 34 | - `opts.userPrefs` **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** An object containing the preferences. (optional, default `{}`) 35 | 36 | Example: 37 | 38 | ```javascript 39 | const puppeteer = require('puppeteer-extra') 40 | puppeteer.use(require('puppeteer-extra-plugin-user-preferences')({userPrefs: { 41 | webkit: { 42 | webprefs: { 43 | default_font_size: 22 44 | } 45 | } 46 | }})) 47 | const browser = await puppeteer.launch() 48 | ``` 49 | 50 | * * * 51 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/ava.config-ts.js: -------------------------------------------------------------------------------- 1 | export default { 2 | compileEnhancements: false, 3 | environmentVariables: { 4 | TS_NODE_COMPILER_OPTIONS: '{"module":"commonjs"}' 5 | }, 6 | files: ['src/**/*.test.ts'], 7 | extensions: ['ts'], 8 | require: ['ts-node/register'] 9 | } 10 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/ava.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | files: ['dist/*.test.js'] 3 | } 4 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-extra-plugin", 3 | "version": "3.2.3", 4 | "description": "Base class for puppeteer-extra plugins.", 5 | "main": "dist/index.cjs.js", 6 | "module": "dist/index.esm.js", 7 | "typings": "dist/index.d.ts", 8 | "files": [ 9 | "dist" 10 | ], 11 | "repository": "berstend/puppeteer-extra", 12 | "homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin", 13 | "author": "berstend", 14 | "license": "MIT", 15 | "scripts": { 16 | "clean": "rimraf dist/*", 17 | "prebuild": "run-s clean", 18 | "build": "run-s build:tsc build:rollup", 19 | "build:tsc": "tsc --module commonjs", 20 | "build:rollup": "rollup -c rollup.config.ts", 21 | "docs": "documentation readme --quiet --shallow --github --markdown-theme transitivebs --readme-file readme.md --section API ./src/index.ts", 22 | "postdocs": "npx prettier --write readme.md", 23 | "test": "ava -v --config ava.config-ts.js", 24 | "pretest-ci": "run-s build", 25 | "test-ci": "ava --fail-fast -v" 26 | }, 27 | "engines": { 28 | "node": ">=9.11.2" 29 | }, 30 | "prettier": { 31 | "printWidth": 80, 32 | "semi": false, 33 | "singleQuote": true 34 | }, 35 | "keywords": [ 36 | "puppeteer", 37 | "puppeteer-extra", 38 | "puppeteer-extra-plugin", 39 | "ua", 40 | "user-agent", 41 | "chrome", 42 | "headless", 43 | "pupeteer" 44 | ], 45 | "devDependencies": { 46 | "@types/node": "14.14.34", 47 | "@types/puppeteer": "*", 48 | "ava": "2.4.0", 49 | "documentation-markdown-themes": "^12.1.5", 50 | "npm-run-all": "^4.1.5", 51 | "puppeteer": "9", 52 | "rimraf": "^3.0.0", 53 | "rollup": "^1.27.5", 54 | "rollup-plugin-commonjs": "^10.1.0", 55 | "rollup-plugin-node-resolve": "^5.2.0", 56 | "rollup-plugin-sourcemaps": "^0.4.2", 57 | "rollup-plugin-typescript2": "^0.25.2", 58 | "ts-node": "^8.5.4", 59 | "tslint": "^5.12.1", 60 | "tslint-config-prettier": "^1.18.0", 61 | "tslint-config-standard": "^9.0.0", 62 | "typescript": "4.4.3" 63 | }, 64 | "dependencies": { 65 | "@types/debug": "^4.1.0", 66 | "debug": "^4.1.1", 67 | "merge-deep": "^3.0.1" 68 | }, 69 | "peerDependencies": { 70 | "playwright-extra": "*", 71 | "puppeteer-extra": "*" 72 | }, 73 | "peerDependenciesMeta": { 74 | "puppeteer-extra": { 75 | "optional": true 76 | }, 77 | "playwright-extra": { 78 | "optional": true 79 | } 80 | }, 81 | "gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3" 82 | } 83 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import commonjs from 'rollup-plugin-commonjs' 2 | import resolve from 'rollup-plugin-node-resolve' 3 | import sourceMaps from 'rollup-plugin-sourcemaps' 4 | import typescript from 'rollup-plugin-typescript2' 5 | 6 | const pkg = require('./package.json') 7 | 8 | const entryFile = 'index' 9 | const banner = ` 10 | /*! 11 | * ${pkg.name} v${pkg.version} by ${pkg.author} 12 | * ${pkg.homepage || `https://github.com/${pkg.repository}`} 13 | * @license ${pkg.license} 14 | */ 15 | `.trim() 16 | 17 | export default { 18 | input: `src/${entryFile}.ts`, 19 | output: [ 20 | { 21 | file: pkg.main, 22 | format: 'cjs', 23 | sourcemap: true, 24 | banner 25 | }, 26 | { 27 | file: pkg.module, 28 | format: 'es', 29 | sourcemap: true, 30 | banner 31 | } 32 | ], 33 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 34 | external: [ 35 | ...Object.keys(pkg.dependencies || {}), 36 | ...Object.keys(pkg.peerDependencies || {}) 37 | ], 38 | watch: { 39 | include: 'src/**' 40 | }, 41 | plugins: [ 42 | // Compile TypeScript files 43 | typescript({ useTsconfigDeclarationDir: true }), 44 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 45 | commonjs(), 46 | // Allow node_modules resolution, so you can use 'external' to control 47 | // which external modules to include in the bundle 48 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 49 | resolve(), 50 | // Resolve source maps to the original source 51 | sourceMaps() 52 | ] 53 | } 54 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/src/ambient.d.ts: -------------------------------------------------------------------------------- 1 | export {} 2 | 3 | // https://github.com/sindresorhus/type-fest/issues/19 4 | declare global { 5 | interface SymbolConstructor { 6 | readonly observable: symbol 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/src/puppeteer.ts: -------------------------------------------------------------------------------- 1 | // A wildcard import would result in a `require("puppeteer")` statement 2 | // at the top of the transpiled js file, not what we want. :-/ 3 | // "import type" is a solution here but requires TS >= v3.8 which we don't want to require yet as a minimum. 4 | 5 | export { Browser } from 'puppeteer' 6 | export { Page } from 'puppeteer' 7 | export { Target } from 'puppeteer' 8 | export { ConnectOptions } from 'puppeteer' 9 | export { LaunchOptions } from 'puppeteer' 10 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./dist", 4 | "target": "es2017", 5 | "module": "es2015", 6 | "moduleResolution": "node", 7 | "lib": ["es2015", "es2016", "es2017", "dom"], 8 | "sourceMap": true, 9 | "declaration": true, 10 | "allowSyntheticDefaultImports": true, 11 | "esModuleInterop": true, 12 | "emitDecoratorMetadata": true, 13 | "experimentalDecorators": true, 14 | "strict": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "noImplicitReturns": false, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": false, 19 | "pretty": true, 20 | "stripInternal": true, 21 | "types": ["node"] 22 | }, 23 | "include": [ 24 | "./src/**/*.tsx", 25 | "./src/**/*.ts", 26 | "./src/**/*.test.ts", 27 | "./test/**/*.ts" 28 | ], 29 | "exclude": ["node_modules", "dist", "./test/**/*.spec.ts"] 30 | } 31 | -------------------------------------------------------------------------------- /packages/puppeteer-extra-plugin/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint-config-standard", "tslint-config-prettier"], 3 | "rules": {} 4 | } 5 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/ava.config-ts.js: -------------------------------------------------------------------------------- 1 | export default { 2 | compileEnhancements: false, 3 | environmentVariables: { 4 | TS_NODE_COMPILER_OPTIONS: '{"module":"commonjs"}' 5 | }, 6 | files: ['test/*.ts'], 7 | extensions: ['ts'], 8 | require: ['ts-node/register'] 9 | } 10 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/ava.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | files: ['test/*.js'] 3 | } 4 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import commonjs from 'rollup-plugin-commonjs' 2 | import resolve from 'rollup-plugin-node-resolve' 3 | import sourceMaps from 'rollup-plugin-sourcemaps' 4 | import typescript from 'rollup-plugin-typescript2' 5 | 6 | const pkg = require('./package.json') 7 | 8 | const entryFile = 'index' 9 | const banner = ` 10 | /*! 11 | * ${pkg.name} v${pkg.version} by ${pkg.author} 12 | * ${pkg.homepage || `https://github.com/${pkg.repository}`} 13 | * @license ${pkg.license} 14 | */ 15 | `.trim() 16 | 17 | const defaultExportOutro = ` 18 | module.exports = exports.default || {} 19 | Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value }) 20 | ` 21 | 22 | export default { 23 | input: `src/${entryFile}.ts`, 24 | output: [ 25 | { 26 | file: pkg.main, 27 | format: 'cjs', 28 | sourcemap: true, 29 | exports: 'named', 30 | outro: defaultExportOutro, 31 | banner 32 | }, 33 | { 34 | file: pkg.module, 35 | format: 'es', 36 | sourcemap: true, 37 | exports: 'named', 38 | banner 39 | } 40 | ], 41 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 42 | external: [ 43 | ...Object.keys(pkg.dependencies || {}), 44 | ...Object.keys(pkg.peerDependencies || {}) 45 | ], 46 | watch: { 47 | include: 'src/**' 48 | }, 49 | plugins: [ 50 | // Compile TypeScript files 51 | typescript({ useTsconfigDeclarationDir: true }), 52 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 53 | commonjs(), 54 | // Allow node_modules resolution, so you can use 'external' to control 55 | // which external modules to include in the bundle 56 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 57 | resolve(), 58 | // Resolve source maps to the original source 59 | sourceMaps() 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/src/ambient.d.ts: -------------------------------------------------------------------------------- 1 | export {} 2 | 3 | // https://github.com/sindresorhus/type-fest/issues/19 4 | declare global { 5 | interface SymbolConstructor { 6 | readonly observable: symbol 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/src/puppeteer-legacy.d.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | // NOTE: The above comment is crucial for all this to work 3 | // The puppeteer project caused a type breaking shift in v6 while switching from @types/puppeteer to built-in types 4 | // This type definition file is only relevant when puppeteer < v6 is being used, 5 | // if we don't instruct TS to skip checking this file it would cause errors when pptr >= v6 is used (e.g. ChromeArgOptions is missing) 6 | import {} from 'puppeteer' 7 | import { Browser, ConnectOptions, ChromeArgOptions, LaunchOptions, FetcherOptions, BrowserFetcher} from "puppeteer" 8 | 9 | // Make puppeteer-extra typings backwards compatible with puppeteer < v6 10 | // In pptr >= v6 they switched to built-in types and the `@types/puppeteer` package is not needed anymore. 11 | // This is essentially a shim for `PuppeteerNode`, which is found in pptr >= v6 and missing in `@types/puppeteer`. 12 | // Requires the `@types/puppeteer` package to be installed when using pptr < v6, `@types/puppeteer` will be ignored by TS when built-in types are available. 13 | interface VanillaPuppeteer { 14 | /** Attaches Puppeteer to an existing Chromium instance */ 15 | connect(options?: ConnectOptions): Promise 16 | /** The default flags that Chromium will be launched with */ 17 | defaultArgs(options?: ChromeArgOptions): string[] 18 | /** Path where Puppeteer expects to find bundled Chromium */ 19 | executablePath(): string 20 | /** The method launches a browser instance with given arguments. The browser will be closed when the parent node.js process is closed. */ 21 | launch(options?: LaunchOptions): Promise 22 | /** This methods attaches Puppeteer to an existing Chromium instance. */ 23 | createBrowserFetcher( 24 | options?: FetcherOptions 25 | ): BrowserFetcher 26 | } 27 | 28 | declare module 'puppeteer' { 29 | interface PuppeteerNode extends VanillaPuppeteer {} 30 | } 31 | declare module 'puppeteer-core' { 32 | interface PuppeteerNode extends VanillaPuppeteer {} 33 | } 34 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/test/addExtra.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import { addExtra } from '../src/index' 4 | 5 | test('is a function', async t => { 6 | t.is(typeof addExtra, 'function') 7 | }) 8 | 9 | test('is an instance of Function', async t => { 10 | t.is(addExtra.constructor.name, 'Function') 11 | }) 12 | 13 | test('returns an object', async t => { 14 | t.is(typeof addExtra(null as any), 'object') 15 | }) 16 | 17 | test('returns an instance of PuppeteerExtra', async t => { 18 | t.is(addExtra(null as any).constructor.name, 'PuppeteerExtra') 19 | }) 20 | 21 | test('will throw without puppeteer', async t => { 22 | const pptr = addExtra(null as any) 23 | t.throws(() => pptr.pptr, null, 'No puppeteer instance provided.') 24 | }) 25 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/test/basic.ts: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import puppeteer from '../src/index' 4 | 5 | test('is an object', async t => { 6 | t.is(typeof puppeteer, 'object') 7 | }) 8 | 9 | test('is an instance of PuppeteerExtra', async t => { 10 | t.is(puppeteer.constructor.name, 'PuppeteerExtra') 11 | }) 12 | 13 | test('should have the public class members', async t => { 14 | t.true(puppeteer.use instanceof Function) 15 | t.true(puppeteer.plugins instanceof Array) 16 | t.true(puppeteer.pluginNames instanceof Array) 17 | t.true(puppeteer.getPluginData instanceof Function) 18 | }) 19 | 20 | test('should have the internal class members', async t => { 21 | t.true('getPluginsByProp' in puppeteer) 22 | t.true('resolvePluginDependencies' in puppeteer) 23 | t.true('orderPlugins' in puppeteer) 24 | t.true('checkPluginRequirements' in puppeteer) 25 | t.true('callPlugins' in puppeteer) 26 | t.true('callPluginsWithValue' in puppeteer) 27 | }) 28 | 29 | test('should have the orginal puppeteer public class members', async t => { 30 | t.true(puppeteer.launch instanceof Function) 31 | t.true(puppeteer.connect instanceof Function) 32 | t.true(puppeteer.executablePath instanceof Function) 33 | t.true(puppeteer.defaultArgs instanceof Function) 34 | t.true(puppeteer.createBrowserFetcher instanceof Function) 35 | }) 36 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/test/connect.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'] 6 | 7 | test.beforeEach(t => { 8 | // Make sure we work with pristine modules 9 | try { 10 | delete require.cache[require.resolve('puppeteer-extra')] 11 | // delete require.cache[require.resolve('puppeteer-extra-plugin')] 12 | } catch (error) { 13 | console.log(error) 14 | } 15 | }) 16 | 17 | test('will remove headless from remote browser', async t => { 18 | // Mitigate CI quirks 19 | try { 20 | // Launch vanilla puppeteer browser with no plugins 21 | const puppeteerVanilla = require('puppeteer') 22 | const browserVanilla = await puppeteerVanilla.launch({ 23 | args: PUPPETEER_ARGS 24 | }) 25 | const browserWSEndpoint = browserVanilla.wsEndpoint() 26 | 27 | // Use puppeteer-extra with plugin to conntect to existing browser 28 | const puppeteer = require('puppeteer-extra') 29 | puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')()) 30 | const browser = await puppeteer.connect({ browserWSEndpoint }) 31 | 32 | // Let's ensure we've anonymized the user-agent, despite not using .launch 33 | const page = await browser.newPage() 34 | const ua = await page.evaluate(() => window.navigator.userAgent) 35 | t.true(!ua.includes('HeadlessChrome')) 36 | 37 | await browser.close() 38 | t.true(true) 39 | } catch (err) { 40 | console.log(`Caught error:`, err) 41 | if ( 42 | err.message && 43 | err.message.includes( 44 | 'Session closed. Most likely the page has been closed' 45 | ) 46 | ) { 47 | t.true(true) // ignore this error 48 | } else { 49 | throw err 50 | } 51 | } 52 | }) 53 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/test/plugin-support.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('ava') 4 | 5 | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'] 6 | const PAGE_TIMEOUT = 60 * 1000 // 60s 7 | 8 | test.beforeEach(t => { 9 | // Make sure we work with pristine modules 10 | try { 11 | delete require.cache[require.resolve('puppeteer-extra')] 12 | delete require.cache[require.resolve('puppeteer-extra-plugin')] 13 | } catch (error) { 14 | console.log(error) 15 | } 16 | }) 17 | 18 | test('will launch the browser normally', async t => { 19 | const puppeteer = require('puppeteer-extra') 20 | const browser = await puppeteer.launch({ args: PUPPETEER_ARGS }) 21 | const page = await browser.newPage() 22 | await page.goto('http://example.com', { 23 | waitUntil: 'domcontentloaded', 24 | timeout: PAGE_TIMEOUT 25 | }) 26 | await browser.close() 27 | t.true(true) 28 | }) 29 | 30 | test('will launch puppeteer with plugin support', async t => { 31 | const puppeteer = require('puppeteer-extra') 32 | const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') 33 | const pluginName = 'hello-world' 34 | const pluginData = [{ name: 'foo', value: 'bar' }] 35 | class Plugin extends PuppeteerExtraPlugin { 36 | constructor(opts = {}) { 37 | super(opts) 38 | } 39 | get name() { 40 | return pluginName 41 | } 42 | get data() { 43 | return pluginData 44 | } 45 | } 46 | const instance = new Plugin() 47 | puppeteer.use(instance) 48 | const browser = await puppeteer.launch({ args: PUPPETEER_ARGS }) 49 | const page = await browser.newPage() 50 | 51 | t.is(puppeteer.plugins.length, 1) 52 | t.is(puppeteer.plugins[0].name, pluginName) 53 | t.is(puppeteer.pluginNames.length, 1) 54 | t.is(puppeteer.pluginNames[0], pluginName) 55 | t.is(puppeteer.getPluginData().length, 1) 56 | t.deepEqual(puppeteer.getPluginData()[0], pluginData[0]) 57 | t.deepEqual(puppeteer.getPluginData('foo')[0], pluginData[0]) 58 | t.is(puppeteer.getPluginData('not-existing').length, 0) 59 | 60 | await page.goto('http://example.com', { 61 | waitUntil: 'domcontentloaded', 62 | timeout: PAGE_TIMEOUT 63 | }) 64 | await browser.close() 65 | t.true(true) 66 | }) 67 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./dist", 4 | "target": "es2017", 5 | "module": "es2015", 6 | "moduleResolution": "node", 7 | "lib": ["es2015", "es2016", "es2017", "dom"], 8 | "sourceMap": true, 9 | "declaration": true, 10 | "allowSyntheticDefaultImports": true, 11 | "esModuleInterop": true, 12 | "emitDecoratorMetadata": true, 13 | "experimentalDecorators": true, 14 | "strict": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "noImplicitReturns": false, 17 | "noUnusedLocals": true, 18 | "noUnusedParameters": false, 19 | "pretty": true, 20 | "stripInternal": true, 21 | "types": ["node"] 22 | }, 23 | "include": ["./src/**/*.tsx", "./src/**/*.ts"], 24 | "exclude": ["node_modules", "dist", "./test/**/*.spec.ts"] 25 | } 26 | -------------------------------------------------------------------------------- /packages/puppeteer-extra/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint-config-standard", "tslint-config-prettier"], 3 | "rules": { 4 | "ordered-imports": true 5 | } 6 | } 7 | --------------------------------------------------------------------------------