├── .editorconfig ├── .github └── workflows │ ├── docs.yaml │ └── tag.yaml ├── .gitignore ├── LICENSE ├── README.md ├── VERSION ├── composer.json ├── composer.lock ├── examples ├── common.php ├── download.php ├── following.php ├── foryou.php ├── hashtag.php ├── music.php ├── user.php └── video.php ├── js ├── fetch.js └── stealth │ ├── README.md │ ├── chrome.app.js │ ├── chrome.csi.js │ ├── chrome.loadtimes.js │ ├── chrome.runtime.js │ ├── iframe.contentWindow.js │ ├── media.codecs.js │ ├── navigator.hardwareConcurrency.js │ ├── navigator.languages.js │ ├── navigator.permissions.js │ ├── navigator.plugins.js │ ├── navigator.vendor.js │ ├── navigator.webdriver.js │ ├── utils.js │ ├── webgl.vendor.js │ └── window.outerdimensions.js ├── phpdoc.dist.xml ├── phpunit.xml ├── src ├── Api.php ├── Cache.php ├── Constants │ ├── CachingMode.php │ ├── Codes.php │ ├── DownloadMethods.php │ ├── Responses.php │ └── UserAgents.php ├── Download.php ├── Downloaders │ ├── BaseDownloader.php │ └── DefaultDownloader.php ├── Helpers │ ├── Algorithm.php │ ├── Converter.php │ ├── Misc.php │ ├── Request.php │ └── Tokens.php ├── Interfaces │ ├── ICache.php │ └── IDownloader.php ├── Items │ ├── Base.php │ ├── Following.php │ ├── ForYou.php │ ├── Hashtag.php │ ├── Music.php │ ├── User.php │ └── Video.php ├── Models │ ├── Base.php │ ├── Feed.php │ ├── Full.php │ ├── Info.php │ ├── Meta.php │ └── Response.php ├── Sender.php ├── Stream.php └── Wrappers │ ├── Guzzle.php │ └── Selenium.php └── tests ├── HashtagTest.php ├── MusicTest.php ├── Pest.php ├── UserTest.php └── VideoTest.php /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs 2 | # editorconfig.org 3 | 4 | root = true 5 | 6 | [*] 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [**.php] 13 | indent_style = space 14 | indent_size = 4 15 | -------------------------------------------------------------------------------- /.github/workflows/docs.yaml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | paths: ['src/**'] 7 | 8 | # Allow GITHUB_TOKEN to deploy to GitHub Pages 9 | permissions: 10 | contents: read 11 | pages: write 12 | id-token: write 13 | 14 | # Allow one concurrent deployment 15 | concurrency: 16 | group: pages 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | build: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v3 25 | - name: Setup graphviz 26 | uses: awalsh128/cache-apt-pkgs-action@latest 27 | with: 28 | packages: graphviz 29 | version: 1.0 30 | - name: Fetch PHPDocumentator 31 | run: wget https://phpdoc.org/phpDocumentor.phar -O /usr/local/bin/phpDocumentator.phar && chmod +x /usr/local/bin/phpDocumentator.phar 32 | - name: Setup PHP 33 | uses: shivammathur/setup-php@v2 34 | with: 35 | php-version: '8.3' 36 | extensions: mbstring 37 | - name: Configure GitHub Pages 38 | uses: actions/configure-pages@v1 39 | - name: Build docs 40 | run: phpDocumentator.phar run -vv -d src -t docs --cache-folder .phpdoc/cache 41 | - name: Upload artifact to GitHub Pages 42 | uses: actions/upload-pages-artifact@v1 43 | with: 44 | path: docs 45 | 46 | deploy: 47 | needs: build 48 | environment: 49 | name: github-pages 50 | url: ${{ steps.deployment.outputs.page_url }} 51 | runs-on: ubuntu-latest 52 | steps: 53 | - name: Deploy to GitHub Pages 54 | id: deployment 55 | uses: actions/deploy-pages@v1 56 | -------------------------------------------------------------------------------- /.github/workflows/tag.yaml: -------------------------------------------------------------------------------- 1 | name: Create Tag 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2 14 | - name: Read version 15 | id: version 16 | uses: juliangruber/read-file-action@v1 17 | with: 18 | path: ./VERSION 19 | trim: true 20 | - name: Create Release 21 | uses: ncipollo/release-action@v1 22 | with: 23 | tag: v${{steps.version.outputs.content}} 24 | token: ${{ secrets.GITHUB_TOKEN }} 25 | generateReleaseNotes: true 26 | skipIfReleaseExists: true 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.vscode 2 | /vendor 3 | .phpdoc 4 | .phpunit.result.cache 5 | /docs 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Pablo Ferreiro Romero 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TikWrapperPHP 2 | A Wrapper for the TikTok API made with PHP >= 8.1 3 | 4 | ## How to Use 5 | ```php 6 | $api = new \TikScraper\Api([ 7 | 'debug' => false, // Debug mode 8 | 'browser' => [ 9 | 'url' => 'http://localhost:4444', // Url to your chromedriver instance 10 | 'close_when_done' => false, // Close chrome instance when request finishes 11 | ], 12 | 'verify_fp' => 'verify_...', // Cookie used for skipping captcha requests 13 | 'device_id' => '596845...' // Custom device id 14 | 'user_agent' => 'YOUR_CUSTOM_USER_AGENT_HERE', 15 | 'proxy' => 'http://user:password@hostname:port' 16 | ], $cacheEngine); 17 | 18 | $tag = $api->hashtag('funny'); 19 | $tag->feed(); 20 | 21 | if ($hastag->ok()) { 22 | echo $hashtag->getFull()->toJson(true); 23 | } else { 24 | print_r($hashtag->error()); 25 | } 26 | ``` 27 | 28 | ## Documentation 29 | An initial version of the documentation is available [here](https://pablouser1.github.io/TikScraperPHP/) 30 | 31 | ## Caching 32 | TikScrapperPHP supports caching requests, to use it you need to implement [ICache.php](https://github.com/pablouser1/TikScraperPHP/blob/master/src/Interfaces/ICache.php) 33 | 34 | ## TODO 35 | * Search 36 | * Comments 37 | ### Left to implement from legacy 38 | * For the love of god, actually document everything properly this time 39 | 40 | ## Credits 41 | * @Sharqo78: Working TikTok downloader without watermark 42 | 43 | HUGE thanks to the following projects, this wouldn't be possible without their help 44 | 45 | * [puppeteer-extra-plugin-stealth](https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth), ported library to PHP 46 | * [TikTok-API-PHP](https://github.com/ssovit/TikTok-API-PHP) 47 | * [TikTok-Api](https://github.com/davidteather/TikTok-Api) 48 | * [tiktok-signature](https://github.com/carcabot/tiktok-signature) 49 | * [tiktok-scraper](https://github.com/drawrowfly/tiktok-scraper) 50 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 2.6.2.0 2 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pablouser1/tikscraper", 3 | "description": "Get data from TikTok API", 4 | "type": "library", 5 | "license": "MIT", 6 | "config": { 7 | "optimize-autoloader": true, 8 | "allow-plugins": { 9 | "pestphp/pest-plugin": true 10 | } 11 | }, 12 | "autoload": { 13 | "psr-4": { 14 | "TikScraper\\": "src/" 15 | } 16 | }, 17 | "authors": [ 18 | { 19 | "name": "Pablo Ferreiro", 20 | "homepage": "https://pabloferreiro.es" 21 | } 22 | ], 23 | "scripts": { 24 | "test": "pest" 25 | }, 26 | "require": { 27 | "php": "^8.1", 28 | "php-webdriver/webdriver": "^1.12", 29 | "guzzlehttp/guzzle": "^7.9" 30 | }, 31 | "require-dev": { 32 | "pestphp/pest": "^1.22" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /examples/common.php: -------------------------------------------------------------------------------- 1 | TIKTOK_DEBUG, 11 | "verify_fp" => TIKTOK_VERIFYFP 12 | ]); 13 | } 14 | 15 | function printError(Meta $meta) { 16 | echo "Error processing request!\n"; 17 | echo "HTTP " . $meta->httpCode . "\n"; 18 | echo "Code " . $meta->proxitokCode . " (" . $meta->proxitokMsg . ")"; 19 | } 20 | -------------------------------------------------------------------------------- /examples/download.php: -------------------------------------------------------------------------------- 1 | user('willsmith'); 11 | $user->feed(); 12 | 13 | if ($user->ok()) { 14 | $downloader = new Download(DownloadMethods::DEFAULT); 15 | $downloader->url($user->getFeed()->items[0]->video->playAddr, 'tiktok-video', false); 16 | } 17 | -------------------------------------------------------------------------------- /examples/following.php: -------------------------------------------------------------------------------- 1 | following(); 7 | $item->feed(); 8 | 9 | if ($item->ok()) { 10 | echo $item->getFull()->toJson(true); 11 | } else { 12 | printError($item->error()); 13 | } 14 | -------------------------------------------------------------------------------- /examples/foryou.php: -------------------------------------------------------------------------------- 1 | foryou(); 7 | $item->feed(); 8 | 9 | if ($item->ok()) { 10 | echo $item->getFull()->toJson(true); 11 | } else { 12 | printError($item->error()); 13 | } 14 | -------------------------------------------------------------------------------- /examples/hashtag.php: -------------------------------------------------------------------------------- 1 | hashtag('funny'); 7 | $item->feed(); 8 | 9 | if ($item->ok()) { 10 | echo $item->getFull()->toJson(true); 11 | } else { 12 | printError($item->error()); 13 | } 14 | -------------------------------------------------------------------------------- /examples/music.php: -------------------------------------------------------------------------------- 1 | music('6715002916702259202'); 7 | $item->feed(); 8 | 9 | if ($item->ok()) { 10 | echo $item->getFull()->toJson(true); 11 | } else { 12 | printError($item->error()); 13 | } 14 | -------------------------------------------------------------------------------- /examples/user.php: -------------------------------------------------------------------------------- 1 | user('ibaillanos'); 7 | $item->feed(); 8 | 9 | if ($item->ok()) { 10 | echo $item->getFull()->toJson(true); 11 | } else { 12 | printError($item->error()); 13 | } 14 | -------------------------------------------------------------------------------- /examples/video.php: -------------------------------------------------------------------------------- 1 | video("7078030558684564779"); 7 | $item->feed(); 8 | 9 | if ($item->ok()) { 10 | echo $item->getFull()->toJson(true); 11 | } else { 12 | printError($item->error()); 13 | } 14 | -------------------------------------------------------------------------------- /js/fetch.js: -------------------------------------------------------------------------------- 1 | class ApiError extends Error { 2 | code = -1; 3 | headers = {}; 4 | 5 | constructor(code, headers) { 6 | super(); 7 | this.code = code; 8 | this.headers = headers; 9 | } 10 | } 11 | 12 | async function fetchApi(url, referrer) { 13 | try { 14 | const res = await fetch(url, { 15 | referrer 16 | }); 17 | 18 | let headers = {}; 19 | for (const h of res.headers) { 20 | headers[h[0]] = h[1]; 21 | } 22 | 23 | if (res.ok) { 24 | const text = await res.text(); 25 | if (text !== "") { 26 | const json = JSON.parse(text); 27 | return { 28 | "type": "json", 29 | "code": res.status, 30 | "data": json, 31 | "headers": headers 32 | }; 33 | } 34 | 35 | // Empty response 36 | throw new ApiError(res.status, headers); 37 | } 38 | 39 | // HTTP Error 40 | throw new ApiError(res.status, headers); 41 | } catch (e) { 42 | return { 43 | "type": "json", 44 | "code": e.code ?? 503, 45 | "data": null, 46 | "headers": e.headers ?? {} 47 | }; 48 | } 49 | } 50 | 51 | window.fetchApi = fetchApi; 52 | -------------------------------------------------------------------------------- /js/stealth/README.md: -------------------------------------------------------------------------------- 1 | # NOTE 2 | This is a port of [puppeteer-extra-plugin-stealth](https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth) to PHP 3 | 4 | `navigator.webdriver` and `user-agent-override` are implemented using PHP code! 5 | -------------------------------------------------------------------------------- /js/stealth/chrome.app.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.js 2 | (function () { 3 | utils.init() 4 | if (!window.chrome) { 5 | // Use the exact property descriptor found in headful Chrome 6 | // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` 7 | Object.defineProperty(window, 'chrome', { 8 | writable: true, 9 | enumerable: true, 10 | configurable: false, // note! 11 | value: {} // We'll extend that later 12 | }) 13 | } 14 | 15 | // That means we're running headful and don't need to mock anything 16 | if ('app' in window.chrome) { 17 | return // Nothing to do here 18 | } 19 | 20 | const makeError = { 21 | ErrorInInvocation: fn => { 22 | const err = new TypeError(`Error in invocation of app.${fn}()`) 23 | return utils.stripErrorWithAnchor( 24 | err, 25 | `at ${fn} (eval at ` 26 | ) 27 | } 28 | } 29 | 30 | // There's a some static data in that property which doesn't seem to change, 31 | // we should periodically check for updates: `JSON.stringify(window.app, null, 2)` 32 | const STATIC_DATA = JSON.parse( 33 | ` 34 | { 35 | "isInstalled": false, 36 | "InstallState": { 37 | "DISABLED": "disabled", 38 | "INSTALLED": "installed", 39 | "NOT_INSTALLED": "not_installed" 40 | }, 41 | "RunningState": { 42 | "CANNOT_RUN": "cannot_run", 43 | "READY_TO_RUN": "ready_to_run", 44 | "RUNNING": "running" 45 | } 46 | } 47 | `.trim() 48 | ) 49 | 50 | window.chrome.app = { 51 | ...STATIC_DATA, 52 | 53 | get isInstalled() { 54 | return false 55 | }, 56 | 57 | getDetails: function getDetails() { 58 | if (arguments.length) { 59 | throw makeError.ErrorInInvocation(`getDetails`) 60 | } 61 | return null 62 | }, 63 | getIsInstalled: function getDetails() { 64 | if (arguments.length) { 65 | throw makeError.ErrorInInvocation(`getIsInstalled`) 66 | } 67 | return false 68 | }, 69 | runningState: function getDetails() { 70 | if (arguments.length) { 71 | throw makeError.ErrorInInvocation(`runningState`) 72 | } 73 | return 'cannot_run' 74 | } 75 | } 76 | utils.patchToStringNested(window.chrome.app) 77 | })() 78 | -------------------------------------------------------------------------------- /js/stealth/chrome.csi.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/index.js 2 | 3 | (function () { 4 | utils.init() 5 | if (!window.chrome) { 6 | // Use the exact property descriptor found in headful Chrome 7 | // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` 8 | Object.defineProperty(window, 'chrome', { 9 | writable: true, 10 | enumerable: true, 11 | configurable: false, // note! 12 | value: {} // We'll extend that later 13 | }) 14 | } 15 | 16 | // That means we're running headful and don't need to mock anything 17 | if ('csi' in window.chrome) { 18 | return // Nothing to do here 19 | } 20 | 21 | // Check that the Navigation Timing API v1 is available, we need that 22 | if (!window.performance || !window.performance.timing) { 23 | return 24 | } 25 | 26 | const { timing } = window.performance 27 | 28 | window.chrome.csi = function () { 29 | return { 30 | onloadT: timing.domContentLoadedEventEnd, 31 | startE: timing.navigationStart, 32 | pageT: Date.now() - timing.navigationStart, 33 | tran: 15 // Transition type or something 34 | } 35 | } 36 | utils.patchToString(window.chrome.csi) 37 | })() 38 | -------------------------------------------------------------------------------- /js/stealth/chrome.loadtimes.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/index.js 2 | 3 | (function () { 4 | utils.init() 5 | if (!window.chrome) { 6 | // Use the exact property descriptor found in headful Chrome 7 | // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` 8 | Object.defineProperty(window, 'chrome', { 9 | writable: true, 10 | enumerable: true, 11 | configurable: false, // note! 12 | value: {} // We'll extend that later 13 | }) 14 | } 15 | 16 | // That means we're running headful and don't need to mock anything 17 | if ('loadTimes' in window.chrome) { 18 | return // Nothing to do here 19 | } 20 | 21 | // Check that the Navigation Timing API v1 + v2 is available, we need that 22 | if ( 23 | !window.performance || 24 | !window.performance.timing || 25 | !window.PerformancePaintTiming 26 | ) { 27 | return 28 | } 29 | 30 | const { performance } = window 31 | 32 | // Some stuff is not available on about:blank as it requires a navigation to occur, 33 | // let's harden the code to not fail then: 34 | const ntEntryFallback = { 35 | nextHopProtocol: 'h2', 36 | type: 'other' 37 | } 38 | 39 | // The API exposes some funky info regarding the connection 40 | const protocolInfo = { 41 | get connectionInfo() { 42 | const ntEntry = 43 | performance.getEntriesByType('navigation')[0] || ntEntryFallback 44 | return ntEntry.nextHopProtocol 45 | }, 46 | get npnNegotiatedProtocol() { 47 | // NPN is deprecated in favor of ALPN, but this implementation returns the 48 | // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. 49 | const ntEntry = 50 | performance.getEntriesByType('navigation')[0] || ntEntryFallback 51 | return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) 52 | ? ntEntry.nextHopProtocol 53 | : 'unknown' 54 | }, 55 | get navigationType() { 56 | const ntEntry = 57 | performance.getEntriesByType('navigation')[0] || ntEntryFallback 58 | return ntEntry.type 59 | }, 60 | get wasAlternateProtocolAvailable() { 61 | // The Alternate-Protocol header is deprecated in favor of Alt-Svc 62 | // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this 63 | // should always return false. 64 | return false 65 | }, 66 | get wasFetchedViaSpdy() { 67 | // SPDY is deprecated in favor of HTTP/2, but this implementation returns 68 | // true for HTTP/2 or HTTP2+QUIC/39 as well. 69 | const ntEntry = 70 | performance.getEntriesByType('navigation')[0] || ntEntryFallback 71 | return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) 72 | }, 73 | get wasNpnNegotiated() { 74 | // NPN is deprecated in favor of ALPN, but this implementation returns true 75 | // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. 76 | const ntEntry = 77 | performance.getEntriesByType('navigation')[0] || ntEntryFallback 78 | return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) 79 | } 80 | } 81 | 82 | const { timing } = window.performance 83 | 84 | // Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3 85 | function toFixed(num, fixed) { 86 | var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?') 87 | return num.toString().match(re)[0] 88 | } 89 | 90 | const timingInfo = { 91 | get firstPaintAfterLoadTime() { 92 | // This was never actually implemented and always returns 0. 93 | return 0 94 | }, 95 | get requestTime() { 96 | return timing.navigationStart / 1000 97 | }, 98 | get startLoadTime() { 99 | return timing.navigationStart / 1000 100 | }, 101 | get commitLoadTime() { 102 | return timing.responseStart / 1000 103 | }, 104 | get finishDocumentLoadTime() { 105 | return timing.domContentLoadedEventEnd / 1000 106 | }, 107 | get finishLoadTime() { 108 | return timing.loadEventEnd / 1000 109 | }, 110 | get firstPaintTime() { 111 | const fpEntry = performance.getEntriesByType('paint')[0] || { 112 | startTime: timing.loadEventEnd / 1000 // Fallback if no navigation occured (`about:blank`) 113 | } 114 | return toFixed( 115 | (fpEntry.startTime + performance.timeOrigin) / 1000, 116 | 3 117 | ) 118 | } 119 | } 120 | 121 | window.chrome.loadTimes = function () { 122 | return { 123 | ...protocolInfo, 124 | ...timingInfo 125 | } 126 | } 127 | utils.patchToString(window.chrome.loadTimes) 128 | })() 129 | -------------------------------------------------------------------------------- /js/stealth/chrome.runtime.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js 2 | 3 | (function () { 4 | utils.init(); 5 | const STATIC_DATA = { 6 | "OnInstalledReason": { 7 | "CHROME_UPDATE": "chrome_update", 8 | "INSTALL": "install", 9 | "SHARED_MODULE_UPDATE": "shared_module_update", 10 | "UPDATE": "update" 11 | }, 12 | "OnRestartRequiredReason": { 13 | "APP_UPDATE": "app_update", 14 | "OS_UPDATE": "os_update", 15 | "PERIODIC": "periodic" 16 | }, 17 | "PlatformArch": { 18 | "ARM": "arm", 19 | "ARM64": "arm64", 20 | "MIPS": "mips", 21 | "MIPS64": "mips64", 22 | "X86_32": "x86-32", 23 | "X86_64": "x86-64" 24 | }, 25 | "PlatformNaclArch": { 26 | "ARM": "arm", 27 | "MIPS": "mips", 28 | "MIPS64": "mips64", 29 | "X86_32": "x86-32", 30 | "X86_64": "x86-64" 31 | }, 32 | "PlatformOs": { 33 | "ANDROID": "android", 34 | "CROS": "cros", 35 | "LINUX": "linux", 36 | "MAC": "mac", 37 | "OPENBSD": "openbsd", 38 | "WIN": "win" 39 | }, 40 | "RequestUpdateCheckStatus": { 41 | "NO_UPDATE": "no_update", 42 | "THROTTLED": "throttled", 43 | "UPDATE_AVAILABLE": "update_available" 44 | } 45 | } 46 | 47 | if (!window.chrome) { 48 | // Use the exact property descriptor found in headful Chrome 49 | // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` 50 | Object.defineProperty(window, 'chrome', { 51 | writable: true, 52 | enumerable: true, 53 | configurable: false, // note! 54 | value: {} // We'll extend that later 55 | }) 56 | } 57 | 58 | // That means we're running headful and don't need to mock anything 59 | const existsAlready = 'runtime' in window.chrome 60 | // `chrome.runtime` is only exposed on secure origins 61 | const isNotSecure = !window.location.protocol.startsWith('https') 62 | if (existsAlready || isNotSecure) { 63 | return // Nothing to do here 64 | } 65 | 66 | window.chrome.runtime = { 67 | // There's a bunch of static data in that property which doesn't seem to change, 68 | // we should periodically check for updates: `JSON.stringify(window.chrome.runtime, null, 2)` 69 | ...STATIC_DATA, 70 | // `chrome.runtime.id` is extension related and returns undefined in Chrome 71 | get id() { 72 | return undefined 73 | }, 74 | // These two require more sophisticated mocks 75 | connect: null, 76 | sendMessage: null 77 | } 78 | 79 | const makeCustomRuntimeErrors = (preamble, method, extensionId) => ({ 80 | NoMatchingSignature: new TypeError( 81 | preamble + `No matching signature.` 82 | ), 83 | MustSpecifyExtensionID: new TypeError( 84 | preamble + 85 | `${method} called from a webpage must specify an Extension ID (string) for its first argument.` 86 | ), 87 | InvalidExtensionID: new TypeError( 88 | preamble + `Invalid extension id: '${extensionId}'` 89 | ) 90 | }) 91 | 92 | // Valid Extension IDs are 32 characters in length and use the letter `a` to `p`: 93 | // https://source.chromium.org/chromium/chromium/src/+/master:components/crx_file/id_util.cc;drc=14a055ccb17e8c8d5d437fe080faba4c6f07beac;l=90 94 | const isValidExtensionID = str => 95 | str.length === 32 && str.toLowerCase().match(/^[a-p]+$/) 96 | 97 | /** Mock `chrome.runtime.sendMessage` */ 98 | const sendMessageHandler = { 99 | apply: function (target, ctx, args) { 100 | const [extensionId, options, responseCallback] = args || [] 101 | 102 | // Define custom errors 103 | const errorPreamble = `Error in invocation of runtime.sendMessage(optional string extensionId, any message, optional object options, optional function responseCallback): ` 104 | const Errors = makeCustomRuntimeErrors( 105 | errorPreamble, 106 | `chrome.runtime.sendMessage()`, 107 | extensionId 108 | ) 109 | 110 | // Check if the call signature looks ok 111 | const noArguments = args.length === 0 112 | const tooManyArguments = args.length > 4 113 | const incorrectOptions = options && typeof options !== 'object' 114 | const incorrectResponseCallback = 115 | responseCallback && typeof responseCallback !== 'function' 116 | if ( 117 | noArguments || 118 | tooManyArguments || 119 | incorrectOptions || 120 | incorrectResponseCallback 121 | ) { 122 | throw Errors.NoMatchingSignature 123 | } 124 | 125 | // At least 2 arguments are required before we even validate the extension ID 126 | if (args.length < 2) { 127 | throw Errors.MustSpecifyExtensionID 128 | } 129 | 130 | // Now let's make sure we got a string as extension ID 131 | if (typeof extensionId !== 'string') { 132 | throw Errors.NoMatchingSignature 133 | } 134 | 135 | if (!isValidExtensionID(extensionId)) { 136 | throw Errors.InvalidExtensionID 137 | } 138 | 139 | return undefined // Normal behavior 140 | } 141 | } 142 | utils.mockWithProxy( 143 | window.chrome.runtime, 144 | 'sendMessage', 145 | function sendMessage() { }, 146 | sendMessageHandler 147 | ) 148 | 149 | /** 150 | * Mock `chrome.runtime.connect` 151 | * 152 | * @see https://developer.chrome.com/apps/runtime#method-connect 153 | */ 154 | const connectHandler = { 155 | apply: function (target, ctx, args) { 156 | const [extensionId, connectInfo] = args || [] 157 | 158 | // Define custom errors 159 | const errorPreamble = `Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo): ` 160 | const Errors = makeCustomRuntimeErrors( 161 | errorPreamble, 162 | `chrome.runtime.connect()`, 163 | extensionId 164 | ) 165 | 166 | // Behavior differs a bit from sendMessage: 167 | const noArguments = args.length === 0 168 | const emptyStringArgument = args.length === 1 && extensionId === '' 169 | if (noArguments || emptyStringArgument) { 170 | throw Errors.MustSpecifyExtensionID 171 | } 172 | 173 | const tooManyArguments = args.length > 2 174 | const incorrectConnectInfoType = 175 | connectInfo && typeof connectInfo !== 'object' 176 | 177 | if (tooManyArguments || incorrectConnectInfoType) { 178 | throw Errors.NoMatchingSignature 179 | } 180 | 181 | const extensionIdIsString = typeof extensionId === 'string' 182 | if (extensionIdIsString && extensionId === '') { 183 | throw Errors.MustSpecifyExtensionID 184 | } 185 | if (extensionIdIsString && !isValidExtensionID(extensionId)) { 186 | throw Errors.InvalidExtensionID 187 | } 188 | 189 | // There's another edge-case here: extensionId is optional so we might find a connectInfo object as first param, which we need to validate 190 | const validateConnectInfo = ci => { 191 | // More than a first param connectInfo as been provided 192 | if (args.length > 1) { 193 | throw Errors.NoMatchingSignature 194 | } 195 | // An empty connectInfo has been provided 196 | if (Object.keys(ci).length === 0) { 197 | throw Errors.MustSpecifyExtensionID 198 | } 199 | // Loop over all connectInfo props an check them 200 | Object.entries(ci).forEach(([k, v]) => { 201 | const isExpected = ['name', 'includeTlsChannelId'].includes(k) 202 | if (!isExpected) { 203 | throw new TypeError( 204 | errorPreamble + `Unexpected property: '${k}'.` 205 | ) 206 | } 207 | const MismatchError = (propName, expected, found) => 208 | TypeError( 209 | errorPreamble + 210 | `Error at property '${propName}': Invalid type: expected ${expected}, found ${found}.` 211 | ) 212 | if (k === 'name' && typeof v !== 'string') { 213 | throw MismatchError(k, 'string', typeof v) 214 | } 215 | if (k === 'includeTlsChannelId' && typeof v !== 'boolean') { 216 | throw MismatchError(k, 'boolean', typeof v) 217 | } 218 | }) 219 | } 220 | if (typeof extensionId === 'object') { 221 | validateConnectInfo(extensionId) 222 | throw Errors.MustSpecifyExtensionID 223 | } 224 | 225 | // Unfortunately even when the connect fails Chrome will return an object with methods we need to mock as well 226 | return utils.patchToStringNested(makeConnectResponse()) 227 | } 228 | } 229 | utils.mockWithProxy( 230 | window.chrome.runtime, 231 | 'connect', 232 | function connect() { }, 233 | connectHandler 234 | ) 235 | 236 | function makeConnectResponse() { 237 | const onSomething = () => ({ 238 | addListener: function addListener() { }, 239 | dispatch: function dispatch() { }, 240 | hasListener: function hasListener() { }, 241 | hasListeners: function hasListeners() { 242 | return false 243 | }, 244 | removeListener: function removeListener() { } 245 | }) 246 | 247 | const response = { 248 | name: '', 249 | sender: undefined, 250 | disconnect: function disconnect() { }, 251 | onDisconnect: onSomething(), 252 | onMessage: onSomething(), 253 | postMessage: function postMessage() { 254 | if (!arguments.length) { 255 | throw new TypeError(`Insufficient number of arguments.`) 256 | } 257 | throw new Error(`Attempting to use a disconnected port object`) 258 | } 259 | } 260 | return response 261 | } 262 | })() 263 | -------------------------------------------------------------------------------- /js/stealth/iframe.contentWindow.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/index.js 2 | 3 | (function () { 4 | utils.init(); 5 | try { 6 | // Adds a contentWindow proxy to the provided iframe element 7 | const addContentWindowProxy = iframe => { 8 | const contentWindowProxy = { 9 | get(target, key) { 10 | // Now to the interesting part: 11 | // We actually make this thing behave like a regular iframe window, 12 | // by intercepting calls to e.g. `.self` and redirect it to the correct thing. :) 13 | // That makes it possible for these assertions to be correct: 14 | // iframe.contentWindow.self === window.top // must be false 15 | if (key === 'self') { 16 | return this 17 | } 18 | // iframe.contentWindow.frameElement === iframe // must be true 19 | if (key === 'frameElement') { 20 | return iframe 21 | } 22 | // Intercept iframe.contentWindow[0] to hide the property 0 added by the proxy. 23 | if (key === '0') { 24 | return undefined 25 | } 26 | return Reflect.get(target, key) 27 | } 28 | } 29 | 30 | if (!iframe.contentWindow) { 31 | const proxy = new Proxy(window, contentWindowProxy) 32 | Object.defineProperty(iframe, 'contentWindow', { 33 | get() { 34 | return proxy 35 | }, 36 | set(newValue) { 37 | return newValue // contentWindow is immutable 38 | }, 39 | enumerable: true, 40 | configurable: false 41 | }) 42 | } 43 | } 44 | 45 | // Handles iframe element creation, augments `srcdoc` property so we can intercept further 46 | const handleIframeCreation = (target, thisArg, args) => { 47 | const iframe = target.apply(thisArg, args) 48 | 49 | // We need to keep the originals around 50 | const _iframe = iframe 51 | const _srcdoc = _iframe.srcdoc 52 | 53 | // Add hook for the srcdoc property 54 | // We need to be very surgical here to not break other iframes by accident 55 | Object.defineProperty(iframe, 'srcdoc', { 56 | configurable: true, // Important, so we can reset this later 57 | get: function () { 58 | return _srcdoc 59 | }, 60 | set: function (newValue) { 61 | addContentWindowProxy(this) 62 | // Reset property, the hook is only needed once 63 | Object.defineProperty(iframe, 'srcdoc', { 64 | configurable: false, 65 | writable: false, 66 | value: _srcdoc 67 | }) 68 | _iframe.srcdoc = newValue 69 | } 70 | }) 71 | return iframe 72 | } 73 | 74 | // Adds a hook to intercept iframe creation events 75 | const addIframeCreationSniffer = () => { 76 | /* global document */ 77 | const createElementHandler = { 78 | // Make toString() native 79 | get(target, key) { 80 | return Reflect.get(target, key) 81 | }, 82 | apply: function (target, thisArg, args) { 83 | const isIframe = 84 | args && args.length && `${args[0]}`.toLowerCase() === 'iframe' 85 | if (!isIframe) { 86 | // Everything as usual 87 | return target.apply(thisArg, args) 88 | } else { 89 | return handleIframeCreation(target, thisArg, args) 90 | } 91 | } 92 | } 93 | // All this just due to iframes with srcdoc bug 94 | utils.replaceWithProxy( 95 | document, 96 | 'createElement', 97 | createElementHandler 98 | ) 99 | } 100 | 101 | // Let's go 102 | addIframeCreationSniffer() 103 | } catch (err) { 104 | // console.warn(err) 105 | } 106 | })() 107 | -------------------------------------------------------------------------------- /js/stealth/media.codecs.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.js 2 | 3 | (function () { 4 | utils.init(); 5 | const parseInput = arg => { 6 | const [mime, codecStr] = arg.trim().split(';') 7 | let codecs = [] 8 | if (codecStr && codecStr.includes('codecs="')) { 9 | codecs = codecStr 10 | .trim() 11 | .replace(`codecs="`, '') 12 | .replace(`"`, '') 13 | .trim() 14 | .split(',') 15 | .filter(x => !!x) 16 | .map(x => x.trim()) 17 | } 18 | return { 19 | mime, 20 | codecStr, 21 | codecs 22 | } 23 | } 24 | 25 | const canPlayType = { 26 | // Intercept certain requests 27 | apply: function (target, ctx, args) { 28 | if (!args || !args.length) { 29 | return target.apply(ctx, args) 30 | } 31 | const { mime, codecs } = parseInput(args[0]) 32 | // This specific mp4 codec is missing in Chromium 33 | if (mime === 'video/mp4') { 34 | if (codecs.includes('avc1.42E01E')) { 35 | return 'probably' 36 | } 37 | } 38 | // This mimetype is only supported if no codecs are specified 39 | if (mime === 'audio/x-m4a' && !codecs.length) { 40 | return 'maybe' 41 | } 42 | 43 | // This mimetype is only supported if no codecs are specified 44 | if (mime === 'audio/aac' && !codecs.length) { 45 | return 'probably' 46 | } 47 | // Everything else as usual 48 | return target.apply(ctx, args) 49 | } 50 | } 51 | 52 | /* global HTMLMediaElement */ 53 | utils.replaceWithProxy( 54 | HTMLMediaElement.prototype, 55 | 'canPlayType', 56 | canPlayType 57 | ) 58 | })() 59 | -------------------------------------------------------------------------------- /js/stealth/navigator.hardwareConcurrency.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | utils.init(); 3 | utils.replaceGetterWithProxy( 4 | Object.getPrototypeOf(navigator), 5 | 'hardwareConcurrency', 6 | utils.makeHandler().getterValue({ 7 | hardwareConcurrency: 4 8 | }) 9 | ) 10 | })() 11 | -------------------------------------------------------------------------------- /js/stealth/navigator.languages.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | utils.init(); 3 | const languages = ['en-US', 'en'] 4 | utils.replaceGetterWithProxy( 5 | Object.getPrototypeOf(navigator), 6 | 'languages', 7 | utils.makeHandler().getterValue(Object.freeze([...languages])) 8 | ) 9 | })() 10 | -------------------------------------------------------------------------------- /js/stealth/navigator.permissions.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js 2 | 3 | (function () { 4 | utils.init(); 5 | const isSecure = document.location.protocol.startsWith('https') 6 | 7 | // In headful on secure origins the permission should be "default", not "denied" 8 | if (isSecure) { 9 | utils.replaceGetterWithProxy(Notification, 'permission', { 10 | apply() { 11 | return 'default' 12 | } 13 | }) 14 | } 15 | 16 | // Another weird behavior: 17 | // On insecure origins in headful the state is "denied", 18 | // whereas in headless it's "prompt" 19 | if (!isSecure) { 20 | const handler = { 21 | apply(target, ctx, args) { 22 | const param = (args || [])[0] 23 | 24 | const isNotifications = 25 | param && param.name && param.name === 'notifications' 26 | if (!isNotifications) { 27 | return utils.cache.Reflect.apply(...arguments) 28 | } 29 | 30 | return Promise.resolve( 31 | Object.setPrototypeOf( 32 | { 33 | state: 'denied', 34 | onchange: null 35 | }, 36 | PermissionStatus.prototype 37 | ) 38 | ) 39 | } 40 | } 41 | // Note: Don't use `Object.getPrototypeOf` here 42 | utils.replaceWithProxy(Permissions.prototype, 'query', handler) 43 | } 44 | })() 45 | -------------------------------------------------------------------------------- /js/stealth/navigator.plugins.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js 2 | 3 | (function () { 4 | utils.init(); 5 | let fns = { 6 | generateMimeTypeArray: (fns) => mimeTypesData => { 7 | return fns.generateMagicArray(fns)( 8 | mimeTypesData, 9 | MimeTypeArray.prototype, 10 | MimeType.prototype, 11 | 'type' 12 | ) 13 | }, 14 | generatePluginArray: (fns) => pluginsData => { 15 | return fns.generateMagicArray(fns)( 16 | pluginsData, 17 | PluginArray.prototype, 18 | Plugin.prototype, 19 | 'name' 20 | ) 21 | }, 22 | generateMagicArray: (fns) => 23 | function ( 24 | dataArray = [], 25 | proto = MimeTypeArray.prototype, 26 | itemProto = MimeType.prototype, 27 | itemMainProp = 'type' 28 | ) { 29 | // Quick helper to set props with the same descriptors vanilla is using 30 | const defineProp = (obj, prop, value) => 31 | Object.defineProperty(obj, prop, { 32 | value, 33 | writable: false, 34 | enumerable: false, // Important for mimeTypes & plugins: `JSON.stringify(navigator.mimeTypes)` 35 | configurable: true 36 | }) 37 | 38 | // Loop over our fake data and construct items 39 | const makeItem = data => { 40 | const item = {} 41 | for (const prop of Object.keys(data)) { 42 | if (prop.startsWith('__')) { 43 | continue 44 | } 45 | defineProp(item, prop, data[prop]) 46 | } 47 | return patchItem(item, data) 48 | } 49 | 50 | const patchItem = (item, data) => { 51 | let descriptor = Object.getOwnPropertyDescriptors(item) 52 | 53 | // Special case: Plugins have a magic length property which is not enumerable 54 | // e.g. `navigator.plugins[i].length` should always be the length of the assigned mimeTypes 55 | if (itemProto === Plugin.prototype) { 56 | descriptor = { 57 | ...descriptor, 58 | length: { 59 | value: data.__mimeTypes.length, 60 | writable: false, 61 | enumerable: false, 62 | configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length` 63 | } 64 | } 65 | } 66 | 67 | // We need to spoof a specific `MimeType` or `Plugin` object 68 | const obj = Object.create(itemProto, descriptor) 69 | 70 | // Virtually all property keys are not enumerable in vanilla 71 | const blacklist = [...Object.keys(data), 'length', 'enabledPlugin'] 72 | return new Proxy(obj, { 73 | ownKeys(target) { 74 | return Reflect.ownKeys(target).filter(k => !blacklist.includes(k)) 75 | }, 76 | getOwnPropertyDescriptor(target, prop) { 77 | if (blacklist.includes(prop)) { 78 | return undefined 79 | } 80 | return Reflect.getOwnPropertyDescriptor(target, prop) 81 | } 82 | }) 83 | } 84 | 85 | const magicArray = [] 86 | 87 | // Loop through our fake data and use that to create convincing entities 88 | dataArray.forEach(data => { 89 | magicArray.push(makeItem(data)) 90 | }) 91 | 92 | // Add direct property access based on types (e.g. `obj['application/pdf']`) afterwards 93 | magicArray.forEach(entry => { 94 | defineProp(magicArray, entry[itemMainProp], entry) 95 | }) 96 | 97 | // This is the best way to fake the type to make sure this is false: `Array.isArray(navigator.mimeTypes)` 98 | const magicArrayObj = Object.create(proto, { 99 | ...Object.getOwnPropertyDescriptors(magicArray), 100 | 101 | // There's one ugly quirk we unfortunately need to take care of: 102 | // The `MimeTypeArray` prototype has an enumerable `length` property, 103 | // but headful Chrome will still skip it when running `Object.getOwnPropertyNames(navigator.mimeTypes)`. 104 | // To strip it we need to make it first `configurable` and can then overlay a Proxy with an `ownKeys` trap. 105 | length: { 106 | value: magicArray.length, 107 | writable: false, 108 | enumerable: false, 109 | configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length` 110 | } 111 | }) 112 | 113 | // Generate our functional function mocks :-) 114 | const functionMocks = fns.generateFunctionMocks()( 115 | proto, 116 | itemMainProp, 117 | magicArray 118 | ) 119 | 120 | // We need to overlay our custom object with a JS Proxy 121 | const magicArrayObjProxy = new Proxy(magicArrayObj, { 122 | get(target, key = '') { 123 | // Redirect function calls to our custom proxied versions mocking the vanilla behavior 124 | if (key === 'item') { 125 | return functionMocks.item 126 | } 127 | if (key === 'namedItem') { 128 | return functionMocks.namedItem 129 | } 130 | if (proto === PluginArray.prototype && key === 'refresh') { 131 | return functionMocks.refresh 132 | } 133 | // Everything else can pass through as normal 134 | return utils.cache.Reflect.get(...arguments) 135 | }, 136 | ownKeys(target) { 137 | // There are a couple of quirks where the original property demonstrates "magical" behavior that makes no sense 138 | // This can be witnessed when calling `Object.getOwnPropertyNames(navigator.mimeTypes)` and the absense of `length` 139 | // My guess is that it has to do with the recent change of not allowing data enumeration and this being implemented weirdly 140 | // For that reason we just completely fake the available property names based on our data to match what regular Chrome is doing 141 | // Specific issues when not patching this: `length` property is available, direct `types` props (e.g. `obj['application/pdf']`) are missing 142 | const keys = [] 143 | const typeProps = magicArray.map(mt => mt[itemMainProp]) 144 | typeProps.forEach((_, i) => keys.push(`${i}`)) 145 | typeProps.forEach(propName => keys.push(propName)) 146 | return keys 147 | }, 148 | getOwnPropertyDescriptor(target, prop) { 149 | if (prop === 'length') { 150 | return undefined 151 | } 152 | return Reflect.getOwnPropertyDescriptor(target, prop) 153 | } 154 | }) 155 | 156 | return magicArrayObjProxy 157 | }, 158 | generateFunctionMocks: () => ( 159 | proto, 160 | itemMainProp, 161 | dataArray 162 | ) => ({ 163 | /** Returns the MimeType object with the specified index. */ 164 | item: utils.createProxy(proto.item, { 165 | apply(target, ctx, args) { 166 | if (!args.length) { 167 | throw new TypeError( 168 | `Failed to execute 'item' on '${proto[Symbol.toStringTag] 169 | }': 1 argument required, but only 0 present.` 170 | ) 171 | } 172 | // Special behavior alert: 173 | // - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup 174 | // - If anything else than an integer (including as string) is provided it will return the first entry 175 | const isInteger = args[0] && Number.isInteger(Number(args[0])) // Cast potential string to number first, then check for integer 176 | // Note: Vanilla never returns `undefined` 177 | return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null 178 | } 179 | }), 180 | /** Returns the MimeType object with the specified name. */ 181 | namedItem: utils.createProxy(proto.namedItem, { 182 | apply(target, ctx, args) { 183 | if (!args.length) { 184 | throw new TypeError( 185 | `Failed to execute 'namedItem' on '${proto[Symbol.toStringTag] 186 | }': 1 argument required, but only 0 present.` 187 | ) 188 | } 189 | return dataArray.find(mt => mt[itemMainProp] === args[0]) || null // Not `undefined`! 190 | } 191 | }), 192 | /** Does nothing and shall return nothing */ 193 | refresh: proto.refresh 194 | ? utils.createProxy(proto.refresh, { 195 | apply(target, ctx, args) { 196 | return undefined 197 | } 198 | }) 199 | : undefined 200 | }) 201 | } 202 | 203 | const data = { 204 | "mimeTypes": [ 205 | { 206 | "type": "application/pdf", 207 | "suffixes": "pdf", 208 | "description": "", 209 | "__pluginName": "Chrome PDF Viewer" 210 | }, 211 | { 212 | "type": "application/x-google-chrome-pdf", 213 | "suffixes": "pdf", 214 | "description": "Portable Document Format", 215 | "__pluginName": "Chrome PDF Plugin" 216 | }, 217 | { 218 | "type": "application/x-nacl", 219 | "suffixes": "", 220 | "description": "Native Client Executable", 221 | "__pluginName": "Native Client" 222 | }, 223 | { 224 | "type": "application/x-pnacl", 225 | "suffixes": "", 226 | "description": "Portable Native Client Executable", 227 | "__pluginName": "Native Client" 228 | } 229 | ], 230 | "plugins": [ 231 | { 232 | "name": "Chrome PDF Plugin", 233 | "filename": "internal-pdf-viewer", 234 | "description": "Portable Document Format", 235 | "__mimeTypes": ["application/x-google-chrome-pdf"] 236 | }, 237 | { 238 | "name": "Chrome PDF Viewer", 239 | "filename": "mhjfbmdgcfjbbpaeojofohoefgiehjai", 240 | "description": "", 241 | "__mimeTypes": ["application/pdf"] 242 | }, 243 | { 244 | "name": "Native Client", 245 | "filename": "internal-nacl-plugin", 246 | "description": "", 247 | "__mimeTypes": ["application/x-nacl", "application/x-pnacl"] 248 | } 249 | ] 250 | } 251 | 252 | // That means we're running headful 253 | const hasPlugins = 'plugins' in navigator && navigator.plugins.length 254 | if (hasPlugins) { 255 | return // nothing to do here 256 | } 257 | 258 | const mimeTypes = fns.generateMimeTypeArray(fns)(data.mimeTypes) 259 | const plugins = fns.generatePluginArray(fns)(data.plugins) 260 | 261 | // Plugin and MimeType cross-reference each other, let's do that now 262 | // Note: We're looping through `data.plugins` here, not the generated `plugins` 263 | for (const pluginData of data.plugins) { 264 | pluginData.__mimeTypes.forEach((type, index) => { 265 | plugins[pluginData.name][index] = mimeTypes[type] 266 | 267 | Object.defineProperty(plugins[pluginData.name], type, { 268 | value: mimeTypes[type], 269 | writable: false, 270 | enumerable: false, // Not enumerable 271 | configurable: true 272 | }) 273 | Object.defineProperty(mimeTypes[type], 'enabledPlugin', { 274 | value: 275 | type === 'application/x-pnacl' 276 | ? mimeTypes['application/x-nacl'].enabledPlugin // these reference the same plugin, so we need to re-use the Proxy in order to avoid leaks 277 | : new Proxy(plugins[pluginData.name], {}), // Prevent circular references 278 | writable: false, 279 | enumerable: false, // Important: `JSON.stringify(navigator.plugins)` 280 | configurable: true 281 | }) 282 | }) 283 | } 284 | 285 | const patchNavigator = (name, value) => 286 | utils.replaceProperty(Object.getPrototypeOf(navigator), name, { 287 | get() { 288 | return value 289 | } 290 | }) 291 | 292 | patchNavigator('mimeTypes', mimeTypes) 293 | patchNavigator('plugins', plugins) 294 | })() 295 | -------------------------------------------------------------------------------- /js/stealth/navigator.vendor.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/index.js 2 | 3 | (function () { 4 | utils.init(); 5 | const vendor = "Google Inc." 6 | utils.replaceGetterWithProxy( 7 | Object.getPrototypeOf(navigator), 8 | 'vendor', 9 | utils.makeHandler().getterValue(vendor)) 10 | })() 11 | -------------------------------------------------------------------------------- /js/stealth/navigator.webdriver.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | if (navigator.webdriver) { 3 | delete Object.getPrototypeOf(navigator).webdriver 4 | } 5 | })(); 6 | -------------------------------------------------------------------------------- /js/stealth/utils.js: -------------------------------------------------------------------------------- 1 | // https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js 2 | 3 | /** 4 | * A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving traces. 5 | * 6 | * Meant to be passed down in puppeteer and used in the context of the page (everything in here runs in NodeJS as well as a browser). 7 | * 8 | * Note: If for whatever reason you need to use this outside of `puppeteer-extra`: 9 | * Just remove the `module.exports` statement at the very bottom, the rest can be copy pasted into any browser context. 10 | * 11 | * Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities. 12 | * 13 | */ 14 | var utils = {} 15 | 16 | utils.init = () => { 17 | utils.preloadCache() 18 | } 19 | 20 | /** 21 | * Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw. 22 | * 23 | * The presence of a JS Proxy can be revealed as it shows up in error stack traces. 24 | * 25 | * @param {object} handler - The JS Proxy handler to wrap 26 | */ 27 | utils.stripProxyFromErrors = (handler = {}) => { 28 | const newHandler = { 29 | setPrototypeOf: function (target, proto) { 30 | if (proto === null) 31 | throw new TypeError('Cannot convert object to primitive value') 32 | if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) { 33 | throw new TypeError('Cyclic __proto__ value') 34 | } 35 | return Reflect.setPrototypeOf(target, proto) 36 | } 37 | } 38 | // We wrap each trap in the handler in a try/catch and modify the error stack if they throw 39 | const traps = Object.getOwnPropertyNames(handler) 40 | traps.forEach(trap => { 41 | newHandler[trap] = function () { 42 | try { 43 | // Forward the call to the defined proxy handler 44 | return handler[trap].apply(this, arguments || []) 45 | } catch (err) { 46 | // Stack traces differ per browser, we only support chromium based ones currently 47 | if (!err || !err.stack || !err.stack.includes(`at `)) { 48 | throw err 49 | } 50 | 51 | // When something throws within one of our traps the Proxy will show up in error stacks 52 | // An earlier implementation of this code would simply strip lines with a blacklist, 53 | // but it makes sense to be more surgical here and only remove lines related to our Proxy. 54 | // We try to use a known "anchor" line for that and strip it with everything above it. 55 | // If the anchor line cannot be found for some reason we fall back to our blacklist approach. 56 | 57 | const stripWithBlacklist = (stack, stripFirstLine = true) => { 58 | const blacklist = [ 59 | `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply 60 | `at Object.${trap} `, // e.g. Object.get or Object.apply 61 | `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-) 62 | ] 63 | return ( 64 | err.stack 65 | .split('\n') 66 | // Always remove the first (file) line in the stack (guaranteed to be our proxy) 67 | .filter((line, index) => !(index === 1 && stripFirstLine)) 68 | // Check if the line starts with one of our blacklisted strings 69 | .filter(line => !blacklist.some(bl => line.trim().startsWith(bl))) 70 | .join('\n') 71 | ) 72 | } 73 | 74 | const stripWithAnchor = (stack, anchor) => { 75 | const stackArr = stack.split('\n') 76 | anchor = anchor || `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium 77 | const anchorIndex = stackArr.findIndex(line => 78 | line.trim().startsWith(anchor) 79 | ) 80 | if (anchorIndex === -1) { 81 | return false // 404, anchor not found 82 | } 83 | // Strip everything from the top until we reach the anchor line 84 | // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) 85 | stackArr.splice(1, anchorIndex) 86 | return stackArr.join('\n') 87 | } 88 | 89 | // Special cases due to our nested toString proxies 90 | err.stack = err.stack.replace( 91 | 'at Object.toString (', 92 | 'at Function.toString (' 93 | ) 94 | if ((err.stack || '').includes('at Function.toString (')) { 95 | err.stack = stripWithBlacklist(err.stack, false) 96 | throw err 97 | } 98 | 99 | // Try using the anchor method, fallback to blacklist if necessary 100 | err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack) 101 | 102 | throw err // Re-throw our now sanitized error 103 | } 104 | } 105 | }) 106 | return newHandler 107 | } 108 | 109 | /** 110 | * Strip error lines from stack traces until (and including) a known line the stack. 111 | * 112 | * @param {object} err - The error to sanitize 113 | * @param {string} anchor - The string the anchor line starts with 114 | */ 115 | utils.stripErrorWithAnchor = (err, anchor) => { 116 | const stackArr = err.stack.split('\n') 117 | const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor)) 118 | if (anchorIndex === -1) { 119 | return err // 404, anchor not found 120 | } 121 | // Strip everything from the top until we reach the anchor line (remove anchor line as well) 122 | // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) 123 | stackArr.splice(1, anchorIndex) 124 | err.stack = stackArr.join('\n') 125 | return err 126 | } 127 | 128 | /** 129 | * Replace the property of an object in a stealthy way. 130 | * 131 | * Note: You also want to work on the prototype of an object most often, 132 | * as you'd otherwise leave traces (e.g. showing up in Object.getOwnPropertyNames(obj)). 133 | * 134 | * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty 135 | * 136 | * @example 137 | * replaceProperty(WebGLRenderingContext.prototype, 'getParameter', { value: "alice" }) 138 | * // or 139 | * replaceProperty(Object.getPrototypeOf(navigator), 'languages', { get: () => ['en-US', 'en'] }) 140 | * 141 | * @param {object} obj - The object which has the property to replace 142 | * @param {string} propName - The property name to replace 143 | * @param {object} descriptorOverrides - e.g. { value: "alice" } 144 | */ 145 | utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => { 146 | return Object.defineProperty(obj, propName, { 147 | // Copy over the existing descriptors (writable, enumerable, configurable, etc) 148 | ...(Object.getOwnPropertyDescriptor(obj, propName) || {}), 149 | // Add our overrides (e.g. value, get()) 150 | ...descriptorOverrides 151 | }) 152 | } 153 | 154 | /** 155 | * Preload a cache of function copies and data. 156 | * 157 | * For a determined enough observer it would be possible to overwrite and sniff usage of functions 158 | * we use in our internal Proxies, to combat that we use a cached copy of those functions. 159 | * 160 | * Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before, 161 | * by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups). 162 | * 163 | * This is evaluated once per execution context (e.g. window) 164 | */ 165 | utils.preloadCache = () => { 166 | if (utils.cache) { 167 | return 168 | } 169 | utils.cache = { 170 | // Used in our proxies 171 | Reflect: { 172 | get: Reflect.get.bind(Reflect), 173 | apply: Reflect.apply.bind(Reflect) 174 | }, 175 | // Used in `makeNativeString` 176 | nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }` 177 | } 178 | } 179 | 180 | /** 181 | * Utility function to generate a cross-browser `toString` result representing native code. 182 | * 183 | * There's small differences: Chromium uses a single line, whereas FF & Webkit uses multiline strings. 184 | * To future-proof this we use an existing native toString result as the basis. 185 | * 186 | * The only advantage we have over the other team is that our JS runs first, hence we cache the result 187 | * of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it. 188 | * 189 | * @example 190 | * makeNativeString('foobar') // => `function foobar() { [native code] }` 191 | * 192 | * @param {string} [name] - Optional function name 193 | */ 194 | utils.makeNativeString = (name = '') => { 195 | return utils.cache.nativeToStringStr.replace('toString', name || '') 196 | } 197 | 198 | /** 199 | * Helper function to modify the `toString()` result of the provided object. 200 | * 201 | * Note: Use `utils.redirectToString` instead when possible. 202 | * 203 | * There's a quirk in JS Proxies that will cause the `toString()` result to differ from the vanilla Object. 204 | * If no string is provided we will generate a `[native code]` thing based on the name of the property object. 205 | * 206 | * @example 207 | * patchToString(WebGLRenderingContext.prototype.getParameter, 'function getParameter() { [native code] }') 208 | * 209 | * @param {object} obj - The object for which to modify the `toString()` representation 210 | * @param {string} str - Optional string used as a return value 211 | */ 212 | utils.patchToString = (obj, str = '') => { 213 | const handler = { 214 | apply: function (target, ctx) { 215 | // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` 216 | if (ctx === Function.prototype.toString) { 217 | return utils.makeNativeString('toString') 218 | } 219 | // `toString` targeted at our proxied Object detected 220 | if (ctx === obj) { 221 | // We either return the optional string verbatim or derive the most desired result automatically 222 | return str || utils.makeNativeString(obj.name) 223 | } 224 | // Check if the toString protype of the context is the same as the global prototype, 225 | // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case 226 | const hasSameProto = Object.getPrototypeOf( 227 | Function.prototype.toString 228 | ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins 229 | if (!hasSameProto) { 230 | // Pass the call on to the local Function.prototype.toString instead 231 | return ctx.toString() 232 | } 233 | return target.call(ctx) 234 | } 235 | } 236 | 237 | const toStringProxy = new Proxy( 238 | Function.prototype.toString, 239 | utils.stripProxyFromErrors(handler) 240 | ) 241 | utils.replaceProperty(Function.prototype, 'toString', { 242 | value: toStringProxy 243 | }) 244 | } 245 | 246 | /** 247 | * Make all nested functions of an object native. 248 | * 249 | * @param {object} obj 250 | */ 251 | utils.patchToStringNested = (obj = {}) => { 252 | return utils.execRecursively(obj, ['function'], utils.patchToString) 253 | } 254 | 255 | /** 256 | * Redirect toString requests from one object to another. 257 | * 258 | * @param {object} proxyObj - The object that toString will be called on 259 | * @param {object} originalObj - The object which toString result we wan to return 260 | */ 261 | utils.redirectToString = (proxyObj, originalObj) => { 262 | const handler = { 263 | apply: function (target, ctx) { 264 | // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` 265 | if (ctx === Function.prototype.toString) { 266 | return utils.makeNativeString('toString') 267 | } 268 | 269 | // `toString` targeted at our proxied Object detected 270 | if (ctx === proxyObj) { 271 | const fallback = () => 272 | originalObj && originalObj.name 273 | ? utils.makeNativeString(originalObj.name) 274 | : utils.makeNativeString(proxyObj.name) 275 | 276 | // Return the toString representation of our original object if possible 277 | return originalObj + '' || fallback() 278 | } 279 | 280 | if (typeof ctx === 'undefined' || ctx === null) { 281 | return target.call(ctx) 282 | } 283 | 284 | // Check if the toString protype of the context is the same as the global prototype, 285 | // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case 286 | const hasSameProto = Object.getPrototypeOf( 287 | Function.prototype.toString 288 | ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins 289 | if (!hasSameProto) { 290 | // Pass the call on to the local Function.prototype.toString instead 291 | return ctx.toString() 292 | } 293 | 294 | return target.call(ctx) 295 | } 296 | } 297 | 298 | const toStringProxy = new Proxy( 299 | Function.prototype.toString, 300 | utils.stripProxyFromErrors(handler) 301 | ) 302 | utils.replaceProperty(Function.prototype, 'toString', { 303 | value: toStringProxy 304 | }) 305 | } 306 | 307 | /** 308 | * All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps. 309 | * 310 | * Will stealthify these aspects (strip error stack traces, redirect toString, etc). 311 | * Note: This is meant to modify native Browser APIs and works best with prototype objects. 312 | * 313 | * @example 314 | * replaceWithProxy(WebGLRenderingContext.prototype, 'getParameter', proxyHandler) 315 | * 316 | * @param {object} obj - The object which has the property to replace 317 | * @param {string} propName - The name of the property to replace 318 | * @param {object} handler - The JS Proxy handler to use 319 | */ 320 | utils.replaceWithProxy = (obj, propName, handler) => { 321 | const originalObj = obj[propName] 322 | const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler)) 323 | 324 | utils.replaceProperty(obj, propName, { value: proxyObj }) 325 | utils.redirectToString(proxyObj, originalObj) 326 | 327 | return true 328 | } 329 | /** 330 | * All-in-one method to replace a getter with a JS Proxy using the provided Proxy handler with traps. 331 | * 332 | * @example 333 | * replaceGetterWithProxy(Object.getPrototypeOf(navigator), 'vendor', proxyHandler) 334 | * 335 | * @param {object} obj - The object which has the property to replace 336 | * @param {string} propName - The name of the property to replace 337 | * @param {object} handler - The JS Proxy handler to use 338 | */ 339 | utils.replaceGetterWithProxy = (obj, propName, handler) => { 340 | const fn = Object.getOwnPropertyDescriptor(obj, propName).get 341 | const fnStr = fn.toString() // special getter function string 342 | const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler)) 343 | 344 | utils.replaceProperty(obj, propName, { get: proxyObj }) 345 | utils.patchToString(proxyObj, fnStr) 346 | 347 | return true 348 | } 349 | 350 | /** 351 | * All-in-one method to replace a getter and/or setter. Functions get and set 352 | * of handler have one more argument that contains the native function. 353 | * 354 | * @example 355 | * replaceGetterSetter(HTMLIFrameElement.prototype, 'contentWindow', handler) 356 | * 357 | * @param {object} obj - The object which has the property to replace 358 | * @param {string} propName - The name of the property to replace 359 | * @param {object} handlerGetterSetter - The handler with get and/or set 360 | * functions 361 | * @see https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty#description 362 | */ 363 | utils.replaceGetterSetter = (obj, propName, handlerGetterSetter) => { 364 | const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName) 365 | const handler = { ...ownPropertyDescriptor } 366 | 367 | if (handlerGetterSetter.get !== undefined) { 368 | const nativeFn = ownPropertyDescriptor.get 369 | handler.get = function() { 370 | return handlerGetterSetter.get.call(this, nativeFn.bind(this)) 371 | } 372 | utils.redirectToString(handler.get, nativeFn) 373 | } 374 | 375 | if (handlerGetterSetter.set !== undefined) { 376 | const nativeFn = ownPropertyDescriptor.set 377 | handler.set = function(newValue) { 378 | handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this)) 379 | } 380 | utils.redirectToString(handler.set, nativeFn) 381 | } 382 | 383 | Object.defineProperty(obj, propName, handler) 384 | } 385 | 386 | /** 387 | * All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps. 388 | * 389 | * Will stealthify these aspects (strip error stack traces, redirect toString, etc). 390 | * 391 | * @example 392 | * mockWithProxy(chrome.runtime, 'sendMessage', function sendMessage() {}, proxyHandler) 393 | * 394 | * @param {object} obj - The object which has the property to replace 395 | * @param {string} propName - The name of the property to replace or create 396 | * @param {object} pseudoTarget - The JS Proxy target to use as a basis 397 | * @param {object} handler - The JS Proxy handler to use 398 | */ 399 | utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => { 400 | const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)) 401 | 402 | utils.replaceProperty(obj, propName, { value: proxyObj }) 403 | utils.patchToString(proxyObj) 404 | 405 | return true 406 | } 407 | 408 | /** 409 | * All-in-one method to create a new JS Proxy with stealth tweaks. 410 | * 411 | * This is meant to be used whenever we need a JS Proxy but don't want to replace or mock an existing known property. 412 | * 413 | * Will stealthify certain aspects of the Proxy (strip error stack traces, redirect toString, etc). 414 | * 415 | * @example 416 | * createProxy(navigator.mimeTypes.__proto__.namedItem, proxyHandler) // => Proxy 417 | * 418 | * @param {object} pseudoTarget - The JS Proxy target to use as a basis 419 | * @param {object} handler - The JS Proxy handler to use 420 | */ 421 | utils.createProxy = (pseudoTarget, handler) => { 422 | const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)) 423 | utils.patchToString(proxyObj) 424 | 425 | return proxyObj 426 | } 427 | 428 | /** 429 | * Helper function to split a full path to an Object into the first part and property. 430 | * 431 | * @example 432 | * splitObjPath(`HTMLMediaElement.prototype.canPlayType`) 433 | * // => {objName: "HTMLMediaElement.prototype", propName: "canPlayType"} 434 | * 435 | * @param {string} objPath - The full path to an object as dot notation string 436 | */ 437 | utils.splitObjPath = objPath => ({ 438 | // Remove last dot entry (property) ==> `HTMLMediaElement.prototype` 439 | objName: objPath.split('.').slice(0, -1).join('.'), 440 | // Extract last dot entry ==> `canPlayType` 441 | propName: objPath.split('.').slice(-1)[0] 442 | }) 443 | 444 | /** 445 | * Convenience method to replace a property with a JS Proxy using the provided objPath. 446 | * 447 | * Supports a full path (dot notation) to the object as string here, in case that makes it easier. 448 | * 449 | * @example 450 | * replaceObjPathWithProxy('WebGLRenderingContext.prototype.getParameter', proxyHandler) 451 | * 452 | * @param {string} objPath - The full path to an object (dot notation string) to replace 453 | * @param {object} handler - The JS Proxy handler to use 454 | */ 455 | utils.replaceObjPathWithProxy = (objPath, handler) => { 456 | const { objName, propName } = utils.splitObjPath(objPath) 457 | const obj = eval(objName) // eslint-disable-line no-eval 458 | return utils.replaceWithProxy(obj, propName, handler) 459 | } 460 | 461 | /** 462 | * Traverse nested properties of an object recursively and apply the given function on a whitelist of value types. 463 | * 464 | * @param {object} obj 465 | * @param {array} typeFilter - e.g. `['function']` 466 | * @param {Function} fn - e.g. `utils.patchToString` 467 | */ 468 | utils.execRecursively = (obj = {}, typeFilter = [], fn) => { 469 | function recurse(obj) { 470 | for (const key in obj) { 471 | if (obj[key] === undefined) { 472 | continue 473 | } 474 | if (obj[key] && typeof obj[key] === 'object') { 475 | recurse(obj[key]) 476 | } else { 477 | if (obj[key] && typeFilter.includes(typeof obj[key])) { 478 | fn.call(this, obj[key]) 479 | } 480 | } 481 | } 482 | } 483 | recurse(obj) 484 | return obj 485 | } 486 | 487 | /** 488 | * Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one. 489 | * That means we cannot just use reference variables and functions from outside code, we need to pass everything as a parameter. 490 | * 491 | * Unfortunately the data we can pass is only allowed to be of primitive types, regular functions don't survive the built-in serialization process. 492 | * This utility function will take an object with functions and stringify them, so we can pass them down unharmed as strings. 493 | * 494 | * We use this to pass down our utility functions as well as any other functions (to be able to split up code better). 495 | * 496 | * @see utils.materializeFns 497 | * 498 | * @param {object} fnObj - An object containing functions as properties 499 | */ 500 | utils.stringifyFns = (fnObj = { hello: () => 'world' }) => { 501 | // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine 502 | // https://github.com/feross/fromentries 503 | function fromEntries(iterable) { 504 | return [...iterable].reduce((obj, [key, val]) => { 505 | obj[key] = val 506 | return obj 507 | }, {}) 508 | } 509 | return (Object.fromEntries || fromEntries)( 510 | Object.entries(fnObj) 511 | .filter(([key, value]) => typeof value === 'function') 512 | .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval 513 | ) 514 | } 515 | 516 | /** 517 | * Utility function to reverse the process of `utils.stringifyFns`. 518 | * Will materialize an object with stringified functions (supports classic and fat arrow functions). 519 | * 520 | * @param {object} fnStrObj - An object containing stringified functions as properties 521 | */ 522 | utils.materializeFns = (fnStrObj = { hello: "() => 'world'" }) => { 523 | return Object.fromEntries( 524 | Object.entries(fnStrObj).map(([key, value]) => { 525 | if (value.startsWith('function')) { 526 | // some trickery is needed to make oldschool functions work :-) 527 | return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval 528 | } else { 529 | // arrow functions just work 530 | return [key, eval(value)] // eslint-disable-line no-eval 531 | } 532 | }) 533 | ) 534 | } 535 | 536 | // Proxy handler templates for re-usability 537 | utils.makeHandler = () => ({ 538 | // Used by simple `navigator` getter evasions 539 | getterValue: value => ({ 540 | apply(target, ctx, args) { 541 | // Let's fetch the value first, to trigger and escalate potential errors 542 | // Illegal invocations like `navigator.__proto__.vendor` will throw here 543 | utils.cache.Reflect.apply(...arguments) 544 | return value 545 | } 546 | }) 547 | }) 548 | 549 | /** 550 | * Compare two arrays. 551 | * 552 | * @param {array} array1 - First array 553 | * @param {array} array2 - Second array 554 | */ 555 | utils.arrayEquals = (array1, array2) => { 556 | if (array1.length !== array2.length) { 557 | return false 558 | } 559 | for (let i = 0; i < array1.length; ++i) { 560 | if (array1[i] !== array2[i]) { 561 | return false 562 | } 563 | } 564 | return true 565 | } 566 | 567 | /** 568 | * Cache the method return according to its arguments. 569 | * 570 | * @param {Function} fn - A function that will be cached 571 | */ 572 | utils.memoize = fn => { 573 | const cache = [] 574 | return function(...args) { 575 | if (!cache.some(c => utils.arrayEquals(c.key, args))) { 576 | cache.push({ key: args, value: fn.apply(this, args) }) 577 | } 578 | return cache.find(c => utils.arrayEquals(c.key, args)).value 579 | } 580 | } 581 | -------------------------------------------------------------------------------- /js/stealth/webgl.vendor.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | utils.init(); 3 | const getParameterProxyHandler = { 4 | apply: function (target, ctx, args) { 5 | const param = (args || [])[0] 6 | const result = utils.cache.Reflect.apply(target, ctx, args) 7 | // UNMASKED_VENDOR_WEBGL 8 | if (param === 37445) { 9 | return 'Intel Inc.' // default in headless: Google Inc. 10 | } 11 | // UNMASKED_RENDERER_WEBGL 12 | if (param === 37446) { 13 | return 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader 14 | } 15 | return result 16 | } 17 | } 18 | 19 | // There's more than one WebGL rendering context 20 | // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility 21 | // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter) 22 | const addProxy = (obj, propName) => { 23 | utils.replaceWithProxy(obj, propName, getParameterProxyHandler) 24 | } 25 | // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing: 26 | addProxy(WebGLRenderingContext.prototype, 'getParameter') 27 | addProxy(WebGL2RenderingContext.prototype, 'getParameter') 28 | })() 29 | -------------------------------------------------------------------------------- /js/stealth/window.outerdimensions.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | try { 3 | if (window.outerWidth && window.outerHeight) { 4 | return // nothing to do here 5 | } 6 | const windowFrame = 85 // probably OS and WM dependent 7 | window.outerWidth = window.innerWidth 8 | window.outerHeight = window.innerHeight + windowFrame 9 | } catch (err) { } 10 | })() 11 | -------------------------------------------------------------------------------- /phpdoc.dist.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | docs 10 | .phpdoc/cache 11 | 12 | 13 | 14 | 15 | src 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | ./tests 10 | 11 | 12 | 13 | 14 | ./src 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/Api.php: -------------------------------------------------------------------------------- 1 | sender = new Sender($config); 21 | $this->cache = new Cache($cache_engine); 22 | } 23 | 24 | // -- Main methods -- // 25 | /** 26 | * Gets user from username (@...) 27 | * @param string $term Username 28 | * @return \TikScraper\Items\User 29 | */ 30 | public function user(string $term): User { 31 | return new User($term, $this->sender, $this->cache); 32 | } 33 | 34 | /** 35 | * Gets hashtag from name. 36 | * Also known as tag or challenge 37 | * @param string $term Hashtag name 38 | * @return \TikScraper\Items\Hashtag 39 | */ 40 | public function hashtag(string $term): Hashtag { 41 | return new Hashtag($term, $this->sender, $this->cache); 42 | } 43 | 44 | /** 45 | * Gets videos that use a specific song 46 | * @param string $term Song ID 47 | * @return \TikScraper\Items\Music 48 | */ 49 | public function music(string $term): Music { 50 | return new Music($term, $this->sender, $this->cache); 51 | } 52 | 53 | /** 54 | * Gets video from ID, supports both Webapp and phone 55 | * @param string $term ID 56 | * @return \TikScraper\Items\Video 57 | */ 58 | public function video(string $term): Video { 59 | return new Video($term, $this->sender, $this->cache); 60 | } 61 | 62 | /** 63 | * Gets for you feed. 64 | * @return \TikScraper\Items\ForYou 65 | */ 66 | public function foryou(): ForYou { 67 | return new ForYou($this->sender, $this->cache); 68 | } 69 | 70 | /** 71 | * Gets recommended users 72 | * @return \TikScraper\Items\Following 73 | */ 74 | public function following(): Following { 75 | return new Following($this->sender, $this->cache); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Cache.php: -------------------------------------------------------------------------------- 1 | enabled = true; 19 | $this->engine = $cache_engine; 20 | } 21 | } 22 | 23 | /** 24 | * Get cached item or null if it doesn't exist 25 | * @param string $key Cache key 26 | * @return object|null Cache value 27 | */ 28 | public function get(string $key): ?object { 29 | return $this->enabled ? $this->engine->get($key) : null; 30 | } 31 | 32 | /** 33 | * Checks if `key` exists 34 | * @param string $key Cache key 35 | * @return bool 36 | */ 37 | public function exists(string $key): bool { 38 | return $this->enabled && $this->engine->exists($key); 39 | } 40 | 41 | /** 42 | * Writes data to cache 43 | * @param string $key Cache key 44 | * @param string $data Unserialized data 45 | * @return void 46 | */ 47 | public function set(string $key, string $data): void { 48 | if ($this->enabled) $this->engine->set($key, $data, self::TIMEOUT); 49 | } 50 | 51 | /** 52 | * Gets feed from cache key 53 | * @param string $key Cache key 54 | * @return \TikScraper\Models\Feed Feed data 55 | */ 56 | public function handleFeed(string $key): Feed { 57 | return Feed::fromObj($this->get($key)); 58 | } 59 | 60 | /** 61 | * Gets info from cache key 62 | * @param string $key Cache key 63 | * @return \TikScraper\Models\Info Info data 64 | */ 65 | public function handleInfo(string $key): Info { 66 | return Info::fromObj($this->get($key)); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/Constants/CachingMode.php: -------------------------------------------------------------------------------- 1 | $contentType, 20 | "code" => $code, 21 | "success" => $code >= 200 && $code < 400, 22 | "data" => ["statusCode" => $statusCode], 23 | "headers" => [] 24 | ]); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Constants/UserAgents.php: -------------------------------------------------------------------------------- 1 | downloader = $this->__getDownloader($method, $config); 16 | } 17 | 18 | public function url(string $item, $file_name = "tiktok-video", $watermark = true) { 19 | header('Content-Type: video/mp4'); 20 | header('Content-Disposition: attachment; filename="' . $file_name . '.mp4"'); 21 | header("Content-Transfer-Encoding: Binary"); 22 | 23 | if ($watermark) { 24 | $this->downloader->watermark($item); 25 | } else { 26 | $this->downloader->noWatermark($item); 27 | } 28 | exit; 29 | } 30 | 31 | /** 32 | * Picks downloader from env variable 33 | * @param string $method 34 | * @return \TikScraper\Interfaces\IDownloader 35 | */ 36 | private function __getDownloader(string $method, array $config): IDownloader { 37 | $class_str = ''; 38 | switch ($method) { 39 | case DownloadMethods::DEFAULT: 40 | $class_str = DefaultDownloader::class; 41 | break; 42 | default: 43 | $class_str = DefaultDownloader::class; 44 | } 45 | 46 | return new $class_str($config); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Downloaders/BaseDownloader.php: -------------------------------------------------------------------------------- 1 | tokens = new Tokens($config); 16 | $this->selenium = new Selenium($config, $this->tokens); 17 | $this->guzzle = new Guzzle($config, $this->selenium); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/Downloaders/DefaultDownloader.php: -------------------------------------------------------------------------------- 1 | guzzle->getClient(); 21 | 22 | $res = $client->get($url, [ 23 | "stream" => true, 24 | "http_errors" => false 25 | ]); 26 | 27 | $body = $res->getBody(); 28 | 29 | while (!$body->eof()) { 30 | echo $body->read(self::BUFFER_SIZE); 31 | } 32 | } 33 | 34 | /** 35 | * Downloads video without watermark using playAddr. 36 | * The method is identical from watermark, the url is the only thing that changes 37 | * @param string $url Video URL 38 | * @return void 39 | */ 40 | public function noWatermark(string $url): void { 41 | $this->watermark($url); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Helpers/Algorithm.php: -------------------------------------------------------------------------------- 1 | v->a); // var prefix 22 | 23 | $expect = bin2hex(base64_decode($c->v->c)); // var expect 24 | 25 | $i = 0; 26 | $result = ''; 27 | while ($i < 1000000 && $result === '') { 28 | $hashFinal = ''; 29 | $res = []; 30 | // Build sha256 with $prefix and current $i 31 | $hash = hash_init("sha256"); 32 | hash_update($hash, $prefix); 33 | hash_update($hash, strval($i)); 34 | $hashResult = hash_final($hash, true); 35 | $strArr = str_split($hashResult); 36 | 37 | // Do some byte shifting required by the challenge 38 | foreach ($strArr as $el) { 39 | $tmp = []; 40 | $chr = ord($el); // var v 41 | $tmpNum = $chr < 0 ? $chr + 256 : $chr; // var c 42 | array_push($tmp, dechex(self::uRShift($tmpNum, 4))); 43 | array_push($tmp, dechex($tmpNum & 0xf)); 44 | 45 | array_push($res, implode($tmp)); 46 | } 47 | 48 | $hashFinal = implode($res); 49 | 50 | // Check if the challenge is completed 51 | if ($expect === $hashFinal) { 52 | $c->d = base64_encode(strval($i)); 53 | // We use unescaped slashes to get the same result as the original JS version 54 | $result = base64_encode(json_encode($c, JSON_UNESCAPED_SLASHES)); 55 | } 56 | $i++; 57 | } 58 | 59 | return $result; 60 | } 61 | 62 | // -- Generic -- // 63 | /** 64 | * Generates a random number as a string 65 | * @param int $digits nº of digits 66 | * @return string Result 67 | */ 68 | static public function randomNumber(int $digits = 8): string { 69 | $characters = '0123456789'; 70 | $randomString = ''; 71 | for ($i = 0; $i < $digits; $i++) { 72 | $index = rand(0, strlen($characters) - 1); 73 | $randomString .= $characters[$index]; 74 | } 75 | return $randomString; 76 | } 77 | 78 | /** 79 | * Generates a random string (a-z, A-Z, 0-9) 80 | * @param int $length nº of characters 81 | * @return string Result 82 | */ 83 | static public function randomString(int $length = 8): string { 84 | return bin2hex(random_bytes($length / 2)); 85 | } 86 | 87 | /** 88 | * Does a bitwise unsigned right shift, used only for `Algorithm::challenge` 89 | * @deprecated See `Algorithm::challenge` 90 | * @param int $a 91 | * @param int $b 92 | * @return int Result 93 | */ 94 | static public function uRShift(int $a, int $b) { 95 | if($b == 0) return $a; 96 | return ($a >> $b) & ~(1<<(8*PHP_INT_SIZE-1)>>($b-1)); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/Helpers/Converter.php: -------------------------------------------------------------------------------- 1 | loadHTML($body); 15 | return $dom; 16 | } 17 | return null; 18 | } 19 | /** 20 | * Get `__UNIVERSAL_DATA_FOR_REHYDRATION__` from `DOMDocument` or string if `$dom` is null 21 | * @param string $body HTML body as a string 22 | * @param \DOMDocument|null HTML body as `DOMDocument` 23 | */ 24 | public static function extractHydra(string $body, ?\DOMDocument $dom = null): ?object { 25 | return self::__extractByTagName("__UNIVERSAL_DATA_FOR_REHYDRATION__", $body, $dom); 26 | } 27 | 28 | /** 29 | * Get JSON data from tag inside document 30 | * @param string $tagName HTML element id 31 | * @param string $body HTML body as a string 32 | * @param \DOMDocument|null $dom HTML body as `DOMDocument` 33 | * @return object|null Object with JSON data if successful, null if not 34 | */ 35 | private static function __extractByTagName(string $tagName, string $body, ?\DOMDocument $dom = null): ?object { 36 | // Disallow empty strings 37 | $dom = $dom ?? self::getDoc($body); 38 | if ($dom !== null) { 39 | $script = $dom->getElementById($tagName); 40 | if ($script !== null) { 41 | return json_decode($script->textContent); 42 | } 43 | } 44 | return null; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/Helpers/Request.php: -------------------------------------------------------------------------------- 1 | time(), 16 | "aid" => 1988, 17 | "app_language" => 'en-US', 18 | "app_name" => "tiktok_web", 19 | "browser_language" => $nav->browser_language, 20 | "browser_name" => $nav->browser_name, 21 | "browser_online" => "true", 22 | "browser_platform" => $nav->browser_platform, 23 | "browser_version" => $nav->browser_version, 24 | "channel" => "tiktok_web", 25 | "cookie_enabled" => "true", 26 | "data_collection_enabled" => "false", 27 | "device_id" => $device_id, 28 | "device_platform" => "web_pc", 29 | "focus_state" => "true", 30 | "history_len" => rand(1, 10), 31 | "is_fullscreen" => "true", 32 | "is_page_visible" => "true", 33 | "language" => "en", 34 | "os" => "windows", 35 | "priority_region" => "", 36 | "referer" => "", 37 | "region" => "US", 38 | "screen_width" => 1920, 39 | "screen_height" => 1080, 40 | "tz_name" => "America/Chicago", 41 | "user_is_login" => false, 42 | "webcast_language" => "en", 43 | "verifyFp" => $verifyFp 44 | ]); 45 | 46 | ksort($query_merged); 47 | 48 | return '?' . http_build_query($query_merged); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Helpers/Tokens.php: -------------------------------------------------------------------------------- 1 | verifyFp = $config["verify_fp"] ?? ""; 15 | $this->device_id = $config["device_id"] ?? ""; 16 | } 17 | 18 | public function getVerifyFp(): string { 19 | return $this->verifyFp; 20 | } 21 | 22 | public function setVerifyFp(string $verifyFp): void { 23 | $this->verifyFp = $verifyFp; 24 | } 25 | 26 | public function getDeviceId(): string { 27 | return $this->device_id; 28 | } 29 | 30 | public function setDeviceId(string $device_id): void { 31 | $this->device_id = $device_id; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Interfaces/ICache.php: -------------------------------------------------------------------------------- 1 | term = urlencode($term); 32 | $this->type = $type; 33 | $this->sender = $sender; 34 | $this->cache = $cache; 35 | 36 | // Sets info from cache if it exists 37 | if ($this->caching_mode === CachingMode::FULL) { 38 | $key = $this->getCacheKey(); 39 | if ($this->cache->exists($key)) { 40 | $this->info = $this->cache->handleInfo($key); 41 | } 42 | } 43 | } 44 | 45 | /** 46 | * Destruct function, handles cache 47 | */ 48 | function __destruct() { 49 | if ($this->caching_mode === CachingMode::NONE) { 50 | return; 51 | } 52 | 53 | $key_info = $this->getCacheKey(); 54 | $key_feed = $this->getCacheKey(true); 55 | 56 | // Info 57 | if ($this->caching_mode === CachingMode::FULL) { 58 | if ($this->infoOk() && !$this->cache->exists($key_info)) { 59 | $this->cache->set($key_info, $this->info->toJson()); 60 | } 61 | } 62 | 63 | // Feed 64 | if ($this->feedOk() && !$this->cache->exists($key_feed)) { 65 | $this->cache->set($key_feed, $this->feed->toJson()); 66 | } 67 | } 68 | 69 | public function getInfo(): Info { 70 | return $this->info; 71 | } 72 | 73 | /** 74 | * Returns feed, returns null if $this->feed has not been called 75 | */ 76 | public function getFeed(): ?Feed { 77 | return $this->feed ?? null; 78 | } 79 | 80 | public function getFull(): Full { 81 | return new Full($this->info, $this->feed); 82 | } 83 | 84 | /** 85 | * Checks if info request went OK 86 | */ 87 | public function infoOk(): bool { 88 | return isset($this->info, $this->info->detail) && $this->info->meta->success; 89 | } 90 | 91 | /** 92 | * Checks if feed request went ok 93 | */ 94 | public function feedOk(): bool { 95 | return isset($this->feed) && $this->feed->meta->success; 96 | } 97 | 98 | /** 99 | * Checks if both info and feed requests went ok 100 | */ 101 | public function ok(): bool { 102 | return $this->infoOk() && $this->feedOk(); 103 | } 104 | 105 | /** 106 | * Get Meta from feed if $this->feed has been called, info if not 107 | */ 108 | public function error(): Meta { 109 | return isset($this->feed) ? $this->feed->meta : $this->info->meta; 110 | } 111 | 112 | /** 113 | * Builds cache key from type (video, tag...) and key (id of user, hashtag name...) 114 | * @param bool $addCursor Add current cursor to key 115 | */ 116 | private function getCacheKey(bool $addCursor = false): string { 117 | $key = $this->type . '-' . $this->term; 118 | if ($addCursor) $key .= '-' . $this->cursor; 119 | return $key; 120 | } 121 | 122 | /** 123 | * Checks if cache exists and sets the value of `$this->feed` 124 | * @return bool Exists? 125 | */ 126 | protected function handleFeedCache(): bool { 127 | $key = $this->getCacheKey(true); 128 | $exists = $this->cache->exists($key); 129 | if ($exists) { 130 | $this->feed = $this->cache->handleFeed($key); 131 | } 132 | return $exists; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/Items/Following.php: -------------------------------------------------------------------------------- 1 | info)) { 18 | $this->info(); 19 | } 20 | } 21 | 22 | public function info(): self { 23 | // There is no info in For You, fill with some predefined data 24 | $info = Info::fromObj((object) [ 25 | "detail" => (object) [] 26 | ]); 27 | 28 | $this->info = $info; 29 | 30 | return $this; 31 | } 32 | 33 | /** 34 | * Feed for Following Page 35 | * @param int $cursor Not used (uses ttwid cookie) 36 | * @return \TikScraper\Items\ForYou 37 | */ 38 | public function feed(int $cursor = 0): self { 39 | $this->cursor = $cursor; 40 | 41 | $res = $this->sender->sendApi('/discover/user/', [ 42 | 'count' => 20, 43 | 'data_collection_enabled' => false, 44 | 'discoverType' => 0, 45 | 'from_page' => 'following', 46 | 'needItemList' => true, 47 | 'keyWord' => '', 48 | 'offset' => $cursor, 49 | 'useRecommend' => false 50 | ]); 51 | 52 | $this->feed = Feed::fromReq($res); 53 | 54 | return $this; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Items/ForYou.php: -------------------------------------------------------------------------------- 1 | info)) { 18 | $this->info(); 19 | } 20 | } 21 | 22 | public function info(): self { 23 | // There is no info in For You, fill with some predefined data 24 | $info = Info::fromObj((object) [ 25 | "detail" => (object) [] 26 | ]); 27 | 28 | $this->info = $info; 29 | 30 | return $this; 31 | } 32 | 33 | /** 34 | * Feed for ForYou Page. 35 | * Data collection and feed personalization is DISABLED 36 | * @param int $cursor Not used (uses ttwid cookie) 37 | * @return \TikScraper\Items\ForYou 38 | */ 39 | public function feed(int $cursor = 0): self { 40 | $this->cursor = $cursor; 41 | $res = $this->sender->sendApi('/recommend/item_list/', [ 42 | 'count' => 20, 43 | 'from_page' => 'fyp' 44 | ]); 45 | 46 | $this->feed = Feed::fromReq($res); 47 | 48 | return $this; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Items/Hashtag.php: -------------------------------------------------------------------------------- 1 | info)) { 13 | $this->info(); 14 | } 15 | } 16 | 17 | public function info(): self { 18 | $req = $this->sender->sendApi("/challenge/detail/", [ 19 | "challengeName" => $this->term 20 | ], "/tag/" . $this->term); 21 | 22 | $info = Info::fromReq($req); 23 | if ($info->meta->success && isset($req->jsonBody->challengeInfo)) { 24 | if (isset($req->jsonBody->challengeInfo->challenge)) { 25 | $info->setDetail($req->jsonBody->challengeInfo->challenge); 26 | } 27 | 28 | if (isset($req->jsonBody->challengeInfo->stats)) { 29 | $info->setStats($req->jsonBody->challengeInfo->stats); 30 | } 31 | } 32 | 33 | $this->info = $info; 34 | 35 | return $this; 36 | } 37 | 38 | public function feed(int $cursor = 0): self { 39 | $this->cursor = $cursor; 40 | 41 | if ($this->infoOk()) { 42 | $preloaded = $this->handleFeedCache(); 43 | if (!$preloaded) { 44 | $query = [ 45 | "count" => 30, 46 | "challengeID" => $this->info->detail->id, 47 | "coverFormat" => 2, 48 | "cursor" => $cursor, 49 | "from_page" => "hashtag" 50 | ]; 51 | $req = $this->sender->sendApi('/challenge/item_list/', $query, "/tag/" . $this->term); 52 | $this->feed = Feed::fromReq($req); 53 | } 54 | } 55 | return $this; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Items/Music.php: -------------------------------------------------------------------------------- 1 | info)) { 13 | $this->info(); 14 | } 15 | } 16 | 17 | public function info(): self { 18 | $req = $this->sender->sendApi("/music/detail/", [ 19 | 'from_page' => 'music', 20 | 'musicId' => $this->term 21 | ]); 22 | 23 | $info = Info::fromReq($req); 24 | if ($info->meta->success && isset($req->jsonBody->musicInfo)) { 25 | if (isset($req->jsonBody->musicInfo->music)) { 26 | $info->setDetail($req->jsonBody->musicInfo->music); 27 | } 28 | 29 | if (isset($req->jsonBody->musicInfo->stats)) { 30 | $info->setStats($req->jsonBody->musicInfo->stats); 31 | } 32 | } 33 | 34 | $this->info = $info; 35 | 36 | return $this; 37 | } 38 | 39 | public function feed(int $cursor = 0): self { 40 | $this->cursor = $cursor; 41 | 42 | if ($this->infoOk()) { 43 | $preloaded = $this->handleFeedCache(); 44 | if (!$preloaded) { 45 | $query = [ 46 | "secUid" => "", 47 | "musicID" => $this->info->detail->id, 48 | "cursor" => $cursor, 49 | "shareUid" => "", 50 | "count" => 30 51 | ]; 52 | $req = $this->sender->sendApi('/music/item_list/', $query); 53 | $this->feed = Feed::fromReq($req); 54 | } 55 | } 56 | 57 | return $this; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/Items/User.php: -------------------------------------------------------------------------------- 1 | info)) { 13 | $this->info(); 14 | } 15 | } 16 | 17 | public function info(): self { 18 | $req = $this->sender->sendApi("/user/detail/", [ 19 | "abTestVersion" => "[object Object]", 20 | "appType" => "m", 21 | "secUid" => "", 22 | "uniqueId" => $this->term 23 | ], "/@" . $this->term); 24 | 25 | $info = Info::fromReq($req); 26 | if ($info->meta->success) { 27 | if (isset($req->jsonBody->userInfo, $req->jsonBody->userInfo->user)) { 28 | // userInfo is available 29 | if (isset($req->jsonBody->userInfo->user)) { 30 | // Set details 31 | $info->setDetail($req->jsonBody->userInfo->user); 32 | } 33 | 34 | if (isset($req->jsonBody->userInfo->stats)) { 35 | // Set stats 36 | $info->setStats($req->jsonBody->userInfo->stats); 37 | } 38 | } 39 | } 40 | $this->info = $info; 41 | 42 | return $this; 43 | } 44 | 45 | public function feed(int $cursor = 0): self { 46 | $this->cursor = $cursor; 47 | 48 | if ($this->infoOk()) { 49 | $preloaded = $this->handleFeedCache(); 50 | 51 | if (!$preloaded) { 52 | $query = [ 53 | "count" => 35, 54 | "coverFormat" => 2, 55 | "cursor" => $cursor, 56 | "from_page" => "user", 57 | "needPinnedItemIds" => "true", 58 | "post_item_list_request_type" => 0, 59 | "secUid" => $this->info->detail->secUid, 60 | "userId" => $this->info->detail->id 61 | ]; 62 | 63 | $req = $this->sender->sendApi('/post/item_list/', $query, "/@" . $this->term); 64 | $this->feed = Feed::fromReq($req); 65 | } 66 | } 67 | return $this; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/Items/Video.php: -------------------------------------------------------------------------------- 1 | info)) { 15 | $this->info(); 16 | } 17 | } 18 | 19 | public function info(): self { 20 | $subdomain = ''; 21 | $endpoint = ''; 22 | if (is_numeric($this->term)) { 23 | $subdomain = 'm'; 24 | $endpoint = '/v/' . $this->term; 25 | } else { 26 | $subdomain = 'www'; 27 | $endpoint = '/t/' . $this->term; 28 | } 29 | 30 | $req = $this->sender->sendHTML($endpoint, $subdomain); 31 | 32 | $info = Info::fromReq($req); 33 | if ($info->meta->success) { 34 | if ($req->hasRehidrate() && isset($req->rehidrateState->__DEFAULT_SCOPE__->{'webapp.video-detail'})) { 35 | $root = $req->rehidrateState->__DEFAULT_SCOPE__->{'webapp.video-detail'}; 36 | $this->item = $root->itemInfo->itemStruct; 37 | $info->setDetail($this->item->author); 38 | $info->setStats($this->item->stats); 39 | } 40 | } 41 | $this->info = $info; 42 | 43 | return $this; 44 | } 45 | 46 | public function feed(): self { 47 | $this->cursor = 0; 48 | if ($this->infoOk()) { 49 | $preloaded = $this->handleFeedCache(); 50 | if (!$preloaded && $this->item !== null) { 51 | $this->feed = Feed::fromObj((object) [ 52 | "items" => [$this->item], 53 | "hasMore" => false, 54 | "cursor" => 0 55 | ]); 56 | } 57 | } 58 | return $this; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Models/Base.php: -------------------------------------------------------------------------------- 1 | setMeta($req); 19 | if ($feed->meta->success) { 20 | $data = $req->jsonBody; 21 | 22 | // Videos 23 | if (isset($data->itemList)) { 24 | $feed->setItems($data->itemList); 25 | // Comments 26 | } else if (isset($data->comments)) { 27 | $feed->setItems($data->comments); 28 | // user info list 29 | } else if (isset($data->userInfoList)) { 30 | $items = self::_buildUserInfoList($data); 31 | $feed->setItems($items); 32 | } 33 | 34 | // Nav 35 | $hasMore = false; 36 | if (isset($data->hasMore)) { 37 | $hasMore = $data->hasMore; 38 | } 39 | 40 | // Cursor (named offset in following) 41 | $cursor = 0; 42 | if (isset($data->cursor)) { 43 | $cursor = $data->cursor; 44 | } else if (isset($data->offset)) { 45 | // Check if reached end 46 | $cursor = intval($data->offset); 47 | 48 | $hasMore = $cursor !== 0; 49 | } 50 | 51 | $feed->setNav($hasMore, $cursor); 52 | } 53 | 54 | return $feed; 55 | } 56 | 57 | public static function fromObj(object $cache): self { 58 | $feed = new Feed; 59 | $feed->setMeta(Responses::ok()); 60 | $feed->setItems($cache->items); 61 | $feed->setNav($cache->hasMore, $cache->cursor); 62 | return $feed; 63 | } 64 | 65 | private static function _buildUserInfoList(object $data): array { 66 | $items = []; 67 | foreach ($data->userInfoList as $userInfo) { 68 | $tmpInfo = (object) [ 69 | "detail" => $userInfo->user, 70 | "stats" => $userInfo->stats 71 | ]; 72 | 73 | $tmpFeed = (object) [ 74 | "items" => $userInfo->itemList, 75 | "cursor" => 0, 76 | "hasMore" => false 77 | ]; 78 | 79 | $items[] = (object) [ 80 | "info" => $tmpInfo, 81 | "feed" => $tmpFeed 82 | ]; 83 | } 84 | 85 | return $items; 86 | } 87 | 88 | private function setMeta(Response $res) { 89 | $this->meta = new Meta($res); 90 | } 91 | 92 | private function setNav(bool $hasMore, int $cursor) { 93 | $this->hasMore = $hasMore; 94 | $this->cursor = $cursor; 95 | } 96 | 97 | private function setItems(array $items) { 98 | $this->items = $items; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/Models/Full.php: -------------------------------------------------------------------------------- 1 | info = $info; 10 | $this->feed = $feed; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/Models/Info.php: -------------------------------------------------------------------------------- 1 | setMeta($req); 13 | return $info; 14 | } 15 | 16 | public static function fromObj(object $cache): self { 17 | $info = new Info; 18 | $info->setMeta(Responses::ok()); 19 | if (isset($cache->meta->og)) { 20 | $info->meta->og = $cache->meta->og; 21 | } 22 | 23 | $info->setDetail($cache->detail); 24 | 25 | if (isset($cache->stats)) { 26 | $info->setStats($cache->stats); 27 | } 28 | 29 | return $info; 30 | } 31 | 32 | private function setMeta(Response $req): void { 33 | $this->meta = new Meta($req); 34 | } 35 | 36 | public function setDetail(object $detail): void { 37 | $this->detail = $detail; 38 | } 39 | 40 | public function setStats(object $stats): void { 41 | $this->stats = $stats; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Models/Meta.php: -------------------------------------------------------------------------------- 1 | = 200 and < 300 and $tiktok_code is 0 9 | * @param int $httpCode HTTP Code response 10 | * @param int $proxitokCode TikTok/ProxiTok's own error messages 11 | * @param string $proxitokMsg Detailed error message for $proxitokCode 12 | * @param Response $response Original response for debugging purposes 13 | */ 14 | class Meta { 15 | public bool $success = false; 16 | public int $httpCode = 503; 17 | public int $proxitokCode = -1; 18 | public string $proxitokMsg = ''; 19 | public Response $response; 20 | public object $og; 21 | 22 | function __construct(Response $res) { 23 | $this->httpCode = $res->code; 24 | $this->response = $res; 25 | 26 | if (isset($res->origRes["headers"]["bdturing-verify"])) { 27 | // Captcha detected 28 | $this->setState($res->http_success, Codes::VERIFY->value, ""); 29 | return; 30 | } 31 | 32 | if (empty($res->origRes["data"])) { 33 | // No data 34 | $this->setState($res->http_success, Codes::EMPTY_RESPONSE->value, ""); 35 | return; 36 | } 37 | if ($res->isJson) { 38 | if ($res->jsonBody === null) { 39 | // Couldn't decode JSON 40 | $this->setState($res->http_success, 11, ""); 41 | return; 42 | } 43 | // JSON Data 44 | if (isset($res->jsonBody->shareMeta)) { 45 | $this->setOgIfExists($res->jsonBody); 46 | } 47 | 48 | // Seemingly ok 49 | $this->setState($res->http_success, $this->getCode($res->jsonBody), $this->getMsg($res->jsonBody)); 50 | } elseif ($res->isHtml) { 51 | if (!$res->hasRehidrate()) { 52 | // Response doesn't have valid data 53 | $this->setState($res->http_success, Codes::STATE_DECODE_ERROR->value, ""); 54 | return; 55 | } 56 | 57 | $scope = $res->rehidrateState->__DEFAULT_SCOPE__; 58 | $root = null; 59 | 60 | if (!isset($res->rehidrateState->__DEFAULT_SCOPE__->{"webapp.video-detail"})) { 61 | // Response doesn't have valid data 62 | $this->setState($res->http_success, Codes::STATE_DECODE_ERROR->value, ""); 63 | return; 64 | } 65 | 66 | $root = $res->rehidrateState->__DEFAULT_SCOPE__->{"webapp.video-detail"}; 67 | $this->setState($res->http_success, $root->statusCode, $root->statusMsg); 68 | 69 | $this->setOgIfExists($root); 70 | } 71 | } 72 | 73 | private function setState(bool $http_success, int $proxitokCode, string $proxitokMsg) { 74 | $this->success = $http_success && $proxitokCode === 0; 75 | $this->proxitokCode = $proxitokCode; 76 | if ($proxitokMsg === '') { 77 | // Get message from enum 78 | $code = Codes::tryFrom($proxitokCode); 79 | 80 | $this->proxitokMsg = $code === null ? Codes::UNKNOWN->name : $code->name; 81 | } 82 | } 83 | 84 | private function getCode(object $data): int { 85 | $code = -1; 86 | if (isset($data->statusCode)) { 87 | $code = intval($data->statusCode); 88 | } elseif (isset($data->status_code)) { 89 | $code = intval($data->status_code); 90 | } elseif (isset($data->type) && $data->type === "verify") { 91 | // Check verify 92 | $code = Codes::VERIFY->value; 93 | } 94 | return $code; 95 | } 96 | 97 | private function getMsg(object $data): string { 98 | $msg = ''; 99 | if (isset($data->statusMsg)) { 100 | $msg = $data->statusMsg; 101 | } elseif (isset($data->status_msg)) { 102 | $msg = $data->status_msg; 103 | } 104 | return $msg; 105 | } 106 | 107 | private function setOgIfExists(?object $root): void { 108 | if (isset($root->shareMeta)) { 109 | $this->og = new \stdClass; 110 | $this->og->title = $root->shareMeta->title; 111 | $this->og->description = $root->shareMeta->desc; 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/Models/Response.php: -------------------------------------------------------------------------------- 1 | code = $res["code"]; 20 | $this->http_success = $this->code >= 200 && $this->code <= 399; 21 | 22 | $this->isHtml = $res["type"] === "html"; 23 | $this->isJson = $res["type"] === "json"; 24 | 25 | if ($this->isJson) { 26 | // Converts body into an object 27 | // TODO: Maybe a better way to do this? 28 | $this->jsonBody = json_decode(json_encode($res["data"])); 29 | } elseif ($this->isHtml) { 30 | $dom = Misc::getDoc($res["data"]); 31 | if ($dom->getElementById("__UNIVERSAL_DATA_FOR_REHYDRATION__") !== null) { 32 | $this->rehidrateState = Misc::extractHydra($res["data"], $dom); 33 | } 34 | } 35 | 36 | $this->origRes = $res; 37 | } 38 | 39 | public function hasRehidrate(): bool { 40 | return $this->rehidrateState !== null; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Sender.php: -------------------------------------------------------------------------------- 1 | tokens = new Tokens($config); 24 | $this->selenium = new Selenium($config, $this->tokens); 25 | $this->guzzle = new Guzzle($config, $this->selenium); 26 | } 27 | 28 | /** 29 | * Send request to TikTok's internal API 30 | * @param string $endpoint Api endpoint 31 | * @param array $query Custom query to be sent, later to me merged with some default values 32 | * @param string $referrer Custom `Referrer` to be sent 33 | */ 34 | public function sendApi( 35 | string $endpoint, 36 | array $query = [], 37 | string $referrer = "/" 38 | ): Response { 39 | $driver = $this->selenium->getDriver(); 40 | $nav = $this->selenium->getNavigator(); 41 | $full_referrer = self::WEB_URL . $referrer; 42 | $url = self::API_URL . $endpoint . Request::buildQuery($query, $nav, $this->tokens->getVerifyFp(), $this->tokens->getDeviceId()); 43 | 44 | $res = $driver->executeAsyncScript( 45 | "var callback = arguments[2]; window.fetchApi(arguments[0], arguments[1]).then(d => callback(d))", 46 | [$url, $full_referrer] 47 | ); 48 | 49 | return new Response($res); 50 | } 51 | 52 | /** 53 | * Send regular HTML request using Guzzle 54 | * @param string $endpoint HTML endpoint 55 | * @param string $subdomain Subdomain to be used 56 | */ 57 | public function sendHTML(string $endpoint, string $subdomain): Response { 58 | $client = $this->guzzle->getClient(); 59 | $url = "https://" . $subdomain . ".tiktok.com" . $endpoint; 60 | 61 | $data = [ 62 | "type" => "html", 63 | "code" => -1, 64 | "success" => false, 65 | "data" => null 66 | ]; 67 | 68 | try { 69 | $res = $client->get($url); 70 | $code = $res->getStatusCode(); 71 | $data["code"] = $code; 72 | $data["success"] = $code >= 200 && $code < 400; 73 | $data["data"] = (string) $res->getBody(); 74 | } catch (ClientException | ServerException $e) { 75 | $code = $e->getCode(); 76 | $data["code"] = $code; 77 | $data["success"] = $code >= 200 && $code < 400; 78 | } catch (ConnectException $e) { 79 | $data["code"] = 503; 80 | } 81 | 82 | return new Response($data); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/Stream.php: -------------------------------------------------------------------------------- 1 | null, 18 | 'Content-Length' => null, 19 | 'Content-Range' => null, 20 | // Always send this one to explicitly say we accept ranged requests 21 | 'Accept-Ranges' => 'bytes' 22 | ]; 23 | 24 | private Tokens $tokens; 25 | private Selenium $selenium; 26 | private Guzzle $guzzle; 27 | 28 | public function __construct(array $config = []) { 29 | $this->tokens = new Tokens($config); 30 | $this->selenium = new Selenium($config, $this->tokens); 31 | $this->guzzle = new Guzzle($config, $this->selenium); 32 | } 33 | 34 | /** 35 | * Streams selected url 36 | * @param string $url 37 | * @return void 38 | */ 39 | public function url(string $url): void { 40 | $client = $this->guzzle->getClient(); 41 | 42 | $headers_to_send = [ 43 | "Accept" => "video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5", 44 | "Accept-Language" => "en-US", 45 | "Referer" => "https://www.tiktok.com/", 46 | "DNT" => "1", 47 | "Sec-Fetch-Dest" => "video", 48 | "Sec-Fetch-Mode" => "cors", 49 | "Sec-Fetch-Site" => "same-site", 50 | "Accept-Encoding" => "identity" 51 | ]; 52 | if (isset($_SERVER['HTTP_RANGE'])) { 53 | $headers_to_send['Range'] = $_SERVER['HTTP_RANGE']; 54 | http_response_code(206); 55 | } 56 | 57 | try { 58 | $res = $client->get($url, [ 59 | "headers" => $headers_to_send, 60 | "http_errors" => false, 61 | "on_headers" => function (ResponseInterface $response) { 62 | $headers = $response->getHeaders(); 63 | foreach ($headers as $key => $value) { 64 | if (array_key_exists($key, $this->headers_to_forward)) { 65 | $this->headers_to_forward[$key] = $value; 66 | } 67 | } 68 | }, 69 | "stream" => true 70 | ]); 71 | 72 | $code = $res->getStatusCode(); 73 | 74 | foreach ($this->headers_to_forward as $key => $value) { 75 | if ($value !== null) { 76 | if (is_array($value)) { 77 | foreach ($value as $currentVal) { 78 | header($key . ': ' . $currentVal, false); 79 | } 80 | } else { 81 | header($key . ': ' . $value, false); 82 | } 83 | } 84 | } 85 | 86 | if ($code >= 400 && $code < 500) { 87 | http_response_code($code); 88 | } 89 | 90 | $body = $res->getBody(); 91 | while (!$body->eof()) { 92 | echo $body->read(self::BUFFER_SIZE); 93 | } 94 | } catch (ConnectException $e) { 95 | die("Couldn't connect to TikTok!"); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/Wrappers/Guzzle.php: -------------------------------------------------------------------------------- 1 | "*/*", 15 | "Accept-Language" => "en-US,en;q=0.5", 16 | "Accept-Encoding" => "gzip, deflate, br", 17 | "Referer" => "https://www.tiktok.com/explore" 18 | ]; 19 | 20 | private Client $client; 21 | 22 | function __construct(array $config, Selenium $selenium) { 23 | $driver = $selenium->getDriver(); 24 | 25 | // Share cookies with Selenium 26 | $jar = new CookieJar(); 27 | $cookies = $driver->manage()->getCookies(); 28 | foreach ($cookies as $c) { 29 | $set = new SetCookie(); 30 | $set->setName($c->getName()); 31 | $set->setValue($c->getValue()); 32 | $set->setDomain($c->getDomain()); 33 | $jar->setCookie($set); 34 | } 35 | 36 | // Use selenium's user agent or user-defined 37 | $this->userAgent = $config['user_agent'] ?? $selenium->getUserAgent(); 38 | $httpConfig = [ 39 | 'timeout' => 5.0, 40 | 'cookies' => $jar, 41 | 'allow_redirects' => true, 42 | 'headers' => [ 43 | 'User-Agent' => $this->userAgent, 44 | ...self::DEFAULT_HEADERS 45 | ] 46 | ]; 47 | 48 | // PROXY CONFIG 49 | if (isset($config['proxy'])) { 50 | $httpConfig['proxy'] = $config['proxy']; 51 | } 52 | 53 | $this->client = new Client($httpConfig); 54 | } 55 | 56 | 57 | public function getClient(): Client { 58 | return $this->client; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Wrappers/Selenium.php: -------------------------------------------------------------------------------- 1 | self::DEFAULT_DRIVER_URL, 35 | "close_when_done" => false 36 | ]; 37 | 38 | $args = ["--disable-blink-features=AutomationControlled"]; 39 | 40 | // Chrome flags 41 | $opts = new ChromeOptions(); 42 | $opts->setExperimentalOption("excludeSwitches", [ 43 | "enable-automation", 44 | "disable-extensions", 45 | "disable-default-apps", 46 | "disable-component-extensions-with-background-pages" 47 | ]); 48 | 49 | if (!$debug) { 50 | // Enable headless if not debugging 51 | $args[] = "--headless=new"; 52 | } 53 | 54 | // User defined user agent 55 | if (isset($config["user_agent"]) && !empty($config["user_agent"])) { 56 | $agent = $config["user_agent"]; 57 | $args[] = "--user-agent=$agent"; 58 | } 59 | 60 | // Proxy 61 | if (isset($config['proxy']) && !empty($config["proxy"])) { 62 | $proxy = $config['proxy']; 63 | $args[] = "--proxy-server=$proxy"; 64 | } 65 | 66 | if (count($args) > 0) { 67 | $opts->addArguments($args); 68 | } 69 | 70 | $cap = DesiredCapabilities::chrome(); 71 | $cap->setCapability(ChromeOptions::CAPABILITY, $opts); 72 | 73 | // Get sessionç 74 | $sessions = $this->_getSessions($browser["url"]); 75 | if (count($sessions) > 0) { 76 | // Reuse session 77 | $this->driver = RemoteWebDriver::createBySessionID($sessions[0]["id"], $browser["url"], null, null, true, $cap); 78 | } else { 79 | // Build new session 80 | $this->_buildSession($browser["url"], $cap, $tokens); 81 | } 82 | 83 | if ($tokens->getDeviceId() === "") { 84 | // Get Device Id from localStorage 85 | $sess_id = $this->driver->executeScript('return sessionStorage.getItem("webapp_session_id")'); 86 | if ($sess_id !== null) { 87 | $tokens->setDeviceId(substr($sess_id, 0, 19)); 88 | } 89 | } 90 | } 91 | 92 | public function getDriver(): RemoteWebDriver { 93 | return $this->driver; 94 | } 95 | 96 | public function getNavigator(): object { 97 | return (object) $this->driver->executeScript("return { 98 | user_agent: window.navigator.userAgent, 99 | browser_language: window.navigator.language, 100 | browser_platform: window.navigator.platform, 101 | browser_name: window.navigator.appCodeName, 102 | browser_version: window.navigator.appVersion 103 | }"); 104 | } 105 | 106 | public function getUserAgent(): string { 107 | return $this->getNavigator()->user_agent; 108 | } 109 | 110 | /** 111 | * Build selenium session, executes only on first run. 112 | * Waits until `window.byted_acrawler` is available or timeout 113 | * @param string $url Chromedriver url 114 | * @param \Facebook\WebDriver\Remote\DesiredCapabilities $cap Chrome's capabilities 115 | * @param \TikScraper\Helpers\Tokens $tokens 116 | * @return void 117 | */ 118 | private function _buildSession(string $url, DesiredCapabilities $cap, Tokens $tokens): void { 119 | // Create session 120 | $this->driver = RemoteWebDriver::create($url, $cap); 121 | 122 | $devTools = new ChromeDevToolsDriver($this->driver); 123 | $this->_spoof($devTools); 124 | 125 | $fetch = file_get_contents(__DIR__ . "/../../js/fetch.js"); 126 | // Inject custom JS code for fetching TikTok's API 127 | $devTools->execute("Page.addScriptToEvaluateOnNewDocument", [ 128 | "source" => $fetch 129 | ]); 130 | 131 | $this->driver->get(self::DEFAULT_TIKTOK_URL); 132 | 133 | // Add captcha cookie to Selenium's jar 134 | if ($tokens->getVerifyFp() !== '') { 135 | $cookie = new Cookie("s_v_web_id", $tokens->getVerifyFp()); 136 | $cookie->setDomain(".tiktok.com"); 137 | $cookie->setSecure(true); 138 | $this->driver->manage()->addCookie($cookie); 139 | } 140 | 141 | // Wait until window.byted_acrawler is ready or timeout 142 | (new WebDriverWait($this->driver, 10))->until(function () { 143 | return $this->driver->executeScript("return window.byted_acrawler !== undefined && this.byted_acrawler.frontierSign !== undefined"); 144 | }); 145 | } 146 | 147 | private function _getSessions(string $url): array { 148 | $executor = new HttpCommandExecutor($url, null, null); 149 | $executor->setConnectionTimeout(30000); 150 | $command = new WebDriverCommand( 151 | null, 152 | DriverCommand::GET_ALL_SESSIONS, 153 | [] 154 | ); 155 | 156 | return $executor->execute($command)->getValue(); 157 | } 158 | 159 | private function _spoof(ChromeDevToolsDriver $devTools): void { 160 | foreach (self::SPOOF_JS as $js) { 161 | $js_str = file_get_contents(__DIR__ . '/../../js/stealth/' . $js); 162 | if ($js_str !== false) { 163 | $devTools->execute("Page.addScriptToEvaluateOnNewDocument", [ 164 | "source" => $js_str 165 | ]); 166 | } 167 | } 168 | $this->_spoofUa($devTools); 169 | } 170 | 171 | private function _spoofUa(ChromeDevToolsDriver $devTools): void { 172 | $ua = $this->getUserAgent(); 173 | $ua = str_replace("HeadlessChrome", "Chrome", $ua); 174 | 175 | // Spoof Linux 176 | if (str_contains($ua, "Linux") && !str_contains($ua, "Android")) { 177 | $ua = preg_replace("/\(([^)]+)\)/", '(Windows NT 10.0; Win64; x64)', $ua); 178 | } 179 | 180 | // Get version 181 | $uaVersion = ""; 182 | if (str_contains($ua, "Chrome")) { 183 | $matches = []; 184 | preg_match("/Chrome\/([\d|.]+)/", $ua, $matches); 185 | $uaVersion = $matches[1]; 186 | } else { 187 | $matches = []; 188 | preg_match("/\/([\d|.]+)/", $this->driver->getCapabilities()->getVersion(), $matches); 189 | } 190 | 191 | // Get platform 192 | $platform = ''; 193 | if (str_contains('Mac OS X', $ua)) { 194 | $platform = 'Mac OS X'; 195 | } else if (str_contains('Android', $ua)) { 196 | $platform = 'Android'; 197 | } else if (str_contains('Linux', $ua)) { 198 | $platform = 'Linux'; 199 | } else { 200 | $platform = 'Windows'; 201 | } 202 | 203 | // Get brands 204 | $seed = explode('.', $uaVersion)[0]; // Major chrome version 205 | $order = [ 206 | [0, 1, 2], 207 | [0, 2, 1], 208 | [1, 0, 2], 209 | [1, 2, 0], 210 | [2, 0, 1], 211 | [2, 1, 0] 212 | ][$seed % 6]; 213 | 214 | $escapedChars = [' ', ' ', ';']; 215 | 216 | $char1 = $escapedChars[$order[0]]; 217 | $char2 = $escapedChars[$order[1]]; 218 | $char3 = $escapedChars[$order[2]]; 219 | 220 | $greaseyBrand = "{$char1}Not{$char2}A{$char3}Brand"; 221 | $greasedBrandVersionList = []; 222 | 223 | $greasedBrandVersionList[$order[0]] = [ 224 | "brand" => $greaseyBrand, 225 | "version" => "99" 226 | ]; 227 | 228 | $greasedBrandVersionList[$order[1]] = [ 229 | "brand" => "Chromium", 230 | "version" => $seed 231 | ]; 232 | 233 | $greasedBrandVersionList[$order[2]] = [ 234 | "brand" => "Google Chrome", 235 | "version" => $seed 236 | ]; 237 | 238 | $os_version = ''; 239 | if (str_contains('Mac OS X ', $ua)) { 240 | $matches = []; 241 | preg_match("/Mac OS X ([^)]+)/", $ua, $matches); 242 | 243 | $os_version = $matches[1]; 244 | } else if (str_contains('Android ', $ua)) { 245 | $matches = []; 246 | preg_match("/Android ([^;]+)/", $ua, $matches); 247 | 248 | $os_version = $matches[1]; 249 | } else if (str_contains('Windows ', $ua)) { 250 | $matches = []; 251 | preg_match("/Windows .*?([\d|.]+);?/", $ua, $matches); 252 | 253 | $os_version = $matches[1]; 254 | } 255 | 256 | $arch = ''; 257 | $model = ''; 258 | $mobile = str_contains('Android', $ua); 259 | if ($mobile) { 260 | $matches = []; 261 | 262 | preg_match("/Android.*?;\s([^)]+)/", $ua, $matches); 263 | $model = $matches[1]; 264 | } else { 265 | $arch = 'x86'; 266 | } 267 | 268 | $ua_rewrite = [ 269 | 'userAgent' => $ua, 270 | 'acceptLanguage' => 'en-US,en', 271 | 'platform' => $platform 272 | ]; 273 | 274 | $devTools->execute('Emulation.setUserAgentOverride', $ua_rewrite); 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /tests/HashtagTest.php: -------------------------------------------------------------------------------- 1 | hashtag(TAG_NAME); 8 | expect($tag->infoOk())->toBeTrue(); 9 | }); 10 | 11 | test('Hashtag Feed', function () use ($api) { 12 | $tag = $api->hashtag(TAG_NAME)->feed(); 13 | expect($tag->feedOk())->toBeTrue(); 14 | expect($tag->getFeed()->items)->toBeGreaterThan(0); 15 | }); 16 | -------------------------------------------------------------------------------- /tests/MusicTest.php: -------------------------------------------------------------------------------- 1 | music(MUSIC_NAME); 8 | expect($music->infoOk())->toBeTrue(); 9 | }); 10 | 11 | test('Music Feed', function () use ($api) { 12 | $music = $api->music(MUSIC_NAME)->feed(); 13 | expect($music->feedOk())->toBeTrue(); 14 | expect($music->getFeed()->items)->toBeGreaterThan(0); 15 | }); 16 | -------------------------------------------------------------------------------- /tests/Pest.php: -------------------------------------------------------------------------------- 1 | true, 7 | "verify_fp" => $verifyfp, 8 | "chromedriver" => $chromedriver 9 | ]); 10 | return $api; 11 | } 12 | 13 | function randStr(): string { 14 | return bin2hex(random_bytes(16)); 15 | } 16 | -------------------------------------------------------------------------------- /tests/UserTest.php: -------------------------------------------------------------------------------- 1 | user(USERNAME); 8 | expect($user->infoOk())->toBeTrue(); 9 | }); 10 | 11 | test('User Feed', function () use ($api) { 12 | $user = $api->user(USERNAME)->feed(); 13 | expect($user->feedOk())->toBeTrue(); 14 | expect($user->getFeed()->items)->toBeGreaterThan(0); 15 | }); 16 | 17 | // Checks if sending an invalid username actually does send an error 18 | test('Invalid User', function () use ($api) { 19 | $user = $api->user(randStr()); 20 | expect($user->infoOk())->toBeFalse(); 21 | $meta = $user->error(); 22 | expect($meta->proxitokCode)->not()->toBe(0); 23 | }); 24 | -------------------------------------------------------------------------------- /tests/VideoTest.php: -------------------------------------------------------------------------------- 1 | video(VIDEO_ID)->feed(); 9 | expect($vid->ok())->toBeTrue(); 10 | }); 11 | 12 | test('Invalid video', function () use ($api) { 13 | $vid = $api->video(VIDEO_INVALID)->feed(); 14 | expect($vid->ok())->toBeFalse(); 15 | $meta = $vid->error(); 16 | expect($meta->proxitokCode)->not()->toBe(0); 17 | }); 18 | --------------------------------------------------------------------------------