├── .eslintrc.json ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── package.json ├── page_samples ├── sample_challenge_page.html └── sample_nonchallenge.html ├── src ├── humanoidReqHandler.js ├── response.js ├── solver.js └── ua.text └── tests └── main.test.js /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es6": true, 4 | "jest": true 5 | }, 6 | "extends": "eslint:recommended", 7 | "parserOptions": { 8 | "ecmaVersion": 2017, 9 | "sourceType": "module" 10 | }, 11 | "plugins": [ 12 | "jest" 13 | ], 14 | "rules": { 15 | "indent": [ 16 | "error", 17 | "tab" 18 | ], 19 | "linebreak-style": [ 20 | "error", 21 | "unix" 22 | ], 23 | "quotes": [ 24 | "error", 25 | "double" 26 | ], 27 | "semi": [ 28 | "error", 29 | "always" 30 | ] 31 | } 32 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # webstorm 33 | .idea 34 | 35 | # Compiled binary addons (https://nodejs.org/api/addons.html) 36 | build/Release 37 | 38 | # Dependency directories 39 | node_modules/ 40 | jspm_packages/ 41 | 42 | # TypeScript v1 declaration files 43 | typings/ 44 | 45 | # Optional npm cache directory 46 | .npm 47 | 48 | # Optional eslint cache 49 | .eslintcache 50 | 51 | # Optional REPL history 52 | .node_repl_history 53 | 54 | # Output of 'npm pack' 55 | *.tgz 56 | 57 | # Yarn Integrity file 58 | .yarn-integrity 59 | 60 | # dotenv environment variables file 61 | .env 62 | 63 | # parcel-bundler cache (https://parceljs.org/) 64 | .cache 65 | 66 | # next.js build output 67 | .next 68 | 69 | # nuxt.js build output 70 | .nuxt 71 | 72 | # vuepress build output 73 | .vuepress/dist 74 | 75 | # Serverless directories 76 | .serverless 77 | package-lock.json -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "10" 4 | before_install: 5 | - npm i -g npm@6.4.1 6 | cache: 7 | directories: 8 | - "node_modules" 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Evyatar Meged 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Humanoid 2 | ![Build Status](https://travis-ci.org/evyatarmeged/Humanoid.svg?branch=master) 3 | ![license](https://img.shields.io/badge/license-MIT-green.svg) 4 | ![version](https://img.shields.io/badge/version-1.0.1-blue.svg) 5 | ![tested with jest](https://img.shields.io/badge/tested_with-jest-99424f.svg) 6 | 7 | A Node.js package to bypass WAF anti-bot JS challenges. 8 | 9 | ## About 10 | Humanoid is a Node.js package to solve and bypass CloudFlare (and hopefully in the future - other WAFs' as well) JavaScript anti-bot challenges.
11 | While anti-bot pages are solvable via headless browsers, they are pretty heavy and are usually considered over the top for scraping.
12 | Humanoid can solve these challenges using the Node.js runtime and present the protected HTML page.
13 | The session cookies can also be delegated to other bots to continue scraping causing them to avoid the JS challenges altogether. 14 | 15 | ## Features 16 | * Random browser User-Agent 17 | * Auto-retry on failed challenges 18 | * Highly configurable - hack custom cookies, headers, etc 19 | * Clearing cookies and rotating User-Agent is supported 20 | * Supports decompression of `Brotli` content-encoding. Not supported by Node.js' `request` by default! 21 | 22 | 23 | ## Installation 24 | via npm: 25 | ``` 26 | npm install --save humanoid-js 27 | ``` 28 | 29 | ## Usage 30 | Basic usage with promises: 31 | ```javascript 32 | const Humanoid = require("humanoid-js"); 33 | 34 | let humanoid = new Humanoid(); 35 | humanoid.get("https://www.cloudflare-protected.com") 36 | .then(res => { 37 | console.log(res.body) // ... 38 | }) 39 | .catch(err => { 40 | console.error(err) 41 | }) 42 | ``` 43 | Humanoid uses auto-bypass by default. You can override it on instance creation: 44 | ```javascript 45 | let humanoid = new Humanoid(autoBypass=false) 46 | 47 | humanoid.get("https://canyoupwn.me") 48 | .then(res => { 49 | console.log(res.statusCode) // 503 50 | console.log(res.isSessionChallenged) // true 51 | humanoid.bypassJSChallenge(res) 52 | .then(challengeResponse => { 53 | // Note that challengeResponse.isChallengeSolved won't be set to true when doing manual bypassing. 54 | console.log(challengeResponse.body) // ... 55 | }) 56 | } 57 | ) 58 | .catch(err => { 59 | console.error(err) 60 | }) 61 | ``` 62 | `async/await` is also supported, and is the preferred way to go: 63 | ```javascript 64 | (async function() { 65 | let humanoid = new Humanoid(); 66 | let response = await humanoid.sendRequest("www.cloudflare-protected.com") 67 | console.log(response.body) // ... 68 | }()) 69 | ``` 70 | ### Humanoid API Methods 71 | ```javascript 1.8 72 | rotateUA() // Replace the currently set user agent with a different one 73 | 74 | clearCookies() // "Set a new, empty cookie jar for the humanoid instance" 75 | 76 | get(url, queryString=undefined, headers=undefined) // Send a GET request to `url`. 77 | // if passed, queryString and headers should be objects 78 | 79 | post(url, postBody=undefined, headers=undefined, dataType=undefined) // Send a POST request to `url`. 80 | // `dataType` should be either "form" or "json" - based on the content type of the POST request. 81 | 82 | sendRequest(url, method=undefined, data=undefined, headers=undefined, dataType=undefined) 83 | // Send a request of method `method` to `url` 84 | 85 | bypassJSChallenge(response) // Bypass the anti-bot JS challenge found in response.body 86 | ``` 87 | 88 | ## TODOs 89 | - [ ] Add command line support 90 | * Support a flag to return the cookie jar after challenge solved - for better integration with other tools and scrapers 91 | * Have an option to simply bypass and return the protected HTML 92 | - [ ] Solve other WAFs similar anti-bot challenges 93 | - [ ] Add tests for request sending and challenge solving 94 | - [ ] Add Docker support :whale: 95 | 96 | ## Issues and Contributions 97 | All anti-bot challenges are likely to change in the future. If this is the case, please open an issue explaining 98 | the problem - try to include the target page if possible. I'll do my best to keep the code up to date with 99 | new challenges.
100 | Any and all contributions are welcome - and are highly appreciated. 101 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const request = require("request-promise-native"); 2 | const Solver = require("./src/solver"); 3 | const HumanoidReqHandler = require("./src/humanoidReqHandler"); 4 | 5 | 6 | class Humanoid extends HumanoidReqHandler { 7 | constructor(autoBypass=true, maxRetries=3) { 8 | super(); 9 | this._getRandomTimeout = () => Math.floor(Math.random() * (7000 - 5000 + 1)) + 5000; 10 | this.autoBypass = autoBypass; 11 | this.maxRetries = maxRetries; 12 | this.currMaxRetries = maxRetries; 13 | this._resetCurrMaxRetries = () => this.currMaxRetries = this.maxRetries; 14 | } 15 | 16 | async _asyncTimeout(ms) { 17 | return new Promise(resolve => { 18 | setTimeout(resolve, ms); 19 | }); 20 | } 21 | 22 | rotateUA() { 23 | super.UA = super._getRandomUA(); 24 | } 25 | 26 | clearCookies() { 27 | super.cookieJar = request.jar(); 28 | } 29 | 30 | _buildAnswerObject(values) { 31 | let [vc, pass, answer] = [...values]; 32 | return { 33 | jschl_vc: vc, 34 | pass: pass, 35 | jschl_answer: answer 36 | } 37 | } 38 | 39 | async get(url, queryString=undefined, headers=undefined) { 40 | return await this.sendRequest(url, "GET", queryString, headers) 41 | } 42 | 43 | async post(url, postBody=undefined, headers=undefined, dataType=undefined) { 44 | return await this.sendRequest(url, "POST", postBody, headers, dataType) 45 | } 46 | 47 | async sendRequest(url, method=undefined, data=undefined, headers=undefined, dataType=undefined) { 48 | let response = await super.sendRequest(url, method, data, headers, dataType); 49 | if (response.isSessionChallenged) { 50 | if (this.autoBypass) { 51 | if (--this.currMaxRetries <= 0) { 52 | this._resetCurrMaxRetries(); 53 | throw Error( 54 | `Max retries limit reached. Cannot Solve JavaScript challenge from response: 55 | ${JSON.stringify(response)}` 56 | ) 57 | } else { 58 | let challengeResponse = await this.bypassJSChallenge(response); 59 | // If we got a 200, mark challenge and solved and return 60 | challengeResponse.isChallengeSolved = challengeResponse.statusCode === 200; 61 | this._resetCurrMaxRetries(); 62 | return challengeResponse; 63 | } 64 | } 65 | } 66 | this._resetCurrMaxRetries() 67 | return response; 68 | } 69 | 70 | async bypassJSChallenge(response) { 71 | let {...solution} = Solver.solveChallenge(response); 72 | let timeout = Solver._extractTimeoutFromScript(response.body) || this._getRandomTimeout(); 73 | if (![solution.vc, solution.pass, solution.answer, solution.origin].every(elem => !!elem)) { 74 | throw Error(`Failed to Extract one or more necessary values. 75 | Values obtained: 76 | vc: ${solution.vc} 77 | pass: ${solution.pass} 78 | answer${solution.answer}`) 79 | } else { 80 | // Wait the desired time; 81 | await this._asyncTimeout(timeout); 82 | let answerUrl = `${solution.origin}/cdn-cgi/l/chk_jschl`; 83 | let answerObj = this._buildAnswerObject([solution.vc, solution.pass, solution.answer]); 84 | let headers = super._getRequestHeaders(answerUrl); 85 | headers["Referer"] = response.origin; 86 | 87 | let solvedChallengeRes = await this.get(answerUrl, answerObj, headers); 88 | // All requests that reached here were from a challenged session 89 | solvedChallengeRes.isSessionChallenged = true; 90 | 91 | return solvedChallengeRes; 92 | } 93 | } 94 | } 95 | 96 | module.exports = Humanoid; 97 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "humanoid-js", 3 | "version": "1.0.1", 4 | "description": "Node.js package to bypass WAF anti-bot JavaScript challenges", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/evyatarmeged/Humanoid" 8 | }, 9 | "main": "index.js", 10 | "scripts": { 11 | "lint": "eslint index.js src/*.js", 12 | "test": "jest tests", 13 | "coverage": "jest --collectCoverageFrom=tests/**.js --coverage src" 14 | }, 15 | "keywords": [ 16 | "bot", 17 | "WAF", 18 | "bypass", 19 | "cloudflare", 20 | "anti-bot", 21 | "scrape", 22 | "scraping", 23 | "protected-pages", 24 | "captcha" 25 | ], 26 | "author": "Evyatar Meged", 27 | "homepage": "https://github.com/evyatarmeged/Humanoid", 28 | "license": "MIT", 29 | "dependencies": { 30 | "cheerio": "^1.0.0-rc.2", 31 | "iltorb": "^2.4.0", 32 | "request": "^2.88.0", 33 | "request-promise-native": "^1.0.5", 34 | "safe-eval": "^0.4.1", 35 | "url-parse": "^1.4.3" 36 | }, 37 | "devDependencies": { 38 | "eslint": "^5.6.1", 39 | "eslint-config-standard": "^12.0.0", 40 | "eslint-plugin-jest": "^21.25.1", 41 | "jest": "^23.6.0" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /page_samples/sample_challenge_page.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Just a moment... 10 | 21 | 22 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 76 | 77 | 78 |
49 |
50 | 51 | 62 | 63 |
64 | 65 | 66 | 67 |
68 |
69 | 70 | 71 |
72 | DDoS protection by Cloudflare 73 |
74 |
75 |
79 | 80 | 81 | -------------------------------------------------------------------------------- /page_samples/sample_nonchallenge.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /src/humanoidReqHandler.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const rpn = require("request-promise-native"); 3 | const URL = require("url-parse"); 4 | const Response = require("./response"); 5 | const brotli = require('iltorb'); 6 | 7 | 8 | class HumanoidReqHandler { 9 | constructor() { 10 | this.cookieJar = rpn.jar() 11 | this._userAgentList = fs.readFileSync(__dirname + "/ua.text").toString().split("\n"); 12 | this.UA = this._getRandomUA(); // Set UserAgent 13 | this.config = { // Set default config values 14 | resolveWithFullResponse: true, 15 | jar: this.cookieJar, 16 | simple: false, 17 | gzip: true, 18 | encoding: null 19 | } 20 | } 21 | 22 | _getRandomUA() { 23 | return this._userAgentList[Math.floor(Math.random() * this._userAgentList.length)]; 24 | } 25 | 26 | _parseUrl(url) { 27 | return URL(url); 28 | } 29 | 30 | _getConfForMethod(method, config, data, dataType) { 31 | if (method === "GET") { 32 | config.qs = data 33 | } else { 34 | if (dataType === "form") { 35 | config.form = data; 36 | } else if (dataType === "json") { 37 | config.body = data; 38 | config.json = true; 39 | } else { 40 | throw Error(`Data types must be either "Form" or "JSON" as supported by the request npm package`) 41 | } 42 | } 43 | return config; 44 | } 45 | 46 | isCaptchaInResponse(html) { 47 | return html.indexOf("Attention Required! | Cloudflare") > -1 && html.indexOf("CAPTCHA") > -1 48 | } 49 | 50 | isChallengeInResponse(html) { 51 | return html.indexOf("jschl") > -1 && html.indexOf("DDoS protection by Cloudflare") > -1; 52 | } 53 | 54 | async _decompressBrotli(res) { 55 | res.body = await brotli.decompress(res.body); 56 | return res; 57 | } 58 | 59 | _getRequestHeaders(url) { 60 | let headers = {}; 61 | headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; 62 | headers["Accept-Encoding"] = "gzip, deflate, br"; 63 | headers["Connection"] = "keep-alive"; 64 | headers["Host"] = this._parseUrl(url).host; 65 | headers["User-Agent"] = this.UA; 66 | 67 | return headers; 68 | } 69 | 70 | async sendRequest(url, method=undefined, data=undefined, headers=undefined, dataType="form") { 71 | // Sanitize parameters 72 | let parsedURL = this._parseUrl(url); 73 | let isSessionChallenged = false; 74 | headers = headers || this._getRequestHeaders(url); 75 | headers["Host"] = !headers.Host ? parsedURL.host : headers["Host"]; 76 | method = method !== undefined ? method.toUpperCase() : "GET"; 77 | dataType = dataType.toLowerCase(); 78 | // Build configuration 79 | let currConfig = {...this.config}; 80 | currConfig.headers = headers; 81 | currConfig.method = method; 82 | currConfig = data !== undefined ? this._getConfForMethod(method, currConfig, data, dataType) : currConfig; 83 | 84 | // Send the request 85 | let res = await rpn(url, currConfig); 86 | // Decompress Brotli content-type if returned (Unsupported natively by `request`) 87 | res = res.headers["content-encoding"] === "br" ? await this._decompressBrotli(res) : res; 88 | 89 | if (dataType === "json") { 90 | res.body = JSON.stringify(res.body); 91 | } else { 92 | res.body = res.body.toString(); 93 | } 94 | 95 | if (this.isCaptchaInResponse(res.body)) { 96 | throw Error("CAPTCHA page encountered. Cannot perform bypass.") 97 | } 98 | 99 | if (res.statusCode === 503 && this.isChallengeInResponse(res.body)) { 100 | // Session is definitely challenged 101 | isSessionChallenged = true; 102 | } 103 | 104 | return new Response( 105 | res.statusCode, res.statusMessage, 106 | res.headers, res.body, 107 | parsedURL.host, parsedURL.origin, 108 | res.headers["set-cookie"], isSessionChallenged) 109 | } 110 | } 111 | 112 | module.exports = HumanoidReqHandler; 113 | -------------------------------------------------------------------------------- /src/response.js: -------------------------------------------------------------------------------- 1 | class Response { 2 | constructor(statusCode, statusMessage, headers, body, host, origin, cookies = null, 3 | isSessionChallenged = false, isChallengeSolved = false) { 4 | this.statusCode = statusCode; 5 | this.statusMessage = statusMessage; 6 | this.headers = headers; 7 | this.body = body; 8 | this.host = host; 9 | this.origin = origin; 10 | this.cookies = cookies; // cf session & clearance cookies 11 | this.isSessionChallenged = isSessionChallenged; 12 | this.isChallengeSolved = isChallengeSolved; 13 | } 14 | } 15 | 16 | module.exports = Response; 17 | -------------------------------------------------------------------------------- /src/solver.js: -------------------------------------------------------------------------------- 1 | const cheerio = require("cheerio"); 2 | const safeEval = require("safe-eval"); 3 | 4 | 5 | class Solver { 6 | constructor() {} 7 | 8 | static _extractTimeoutFromScript(html) { 9 | let $ = cheerio.load(html); 10 | let script = $("script").html(); 11 | let match = script.match(/,\s[0-9]0{3}\);/g); 12 | if (match) { 13 | match = match[0].replace(/,|\s|\)|;/g, ""); 14 | } 15 | return match; 16 | } 17 | 18 | static _extractInputValuesFromHTML(html) { 19 | let $ = cheerio.load(html); 20 | return [$("input[name=jschl_vc]").val(), $("input[name=pass]").val()]; 21 | } 22 | 23 | static _extractChallengeFromHTML(html) { 24 | let $ = cheerio.load(html); 25 | let script = $("script"); 26 | return script.html(); 27 | } 28 | 29 | static _operateOnResult(operator, expr, result) { 30 | switch(operator) { 31 | case "+=": 32 | return result += safeEval(expr); 33 | case "*=": 34 | return result *= safeEval(expr); 35 | case "-=": 36 | return result -= safeEval(expr); 37 | case "/=": 38 | return result /= safeEval(expr); 39 | default: 40 | throw Error("Could not match operator. Cannot solve JS challenge"); 41 | } 42 | } 43 | 44 | static _buildAnswer(answerMutations, currResult) { 45 | for (let ans of answerMutations) { 46 | let operator = ans.slice(0,2); 47 | let expr = ans.slice(3); 48 | currResult = this._operateOnResult(operator, expr, currResult); 49 | } 50 | return currResult; 51 | } 52 | 53 | static _parseChallenge(matches) { 54 | // Perform the necessary parsing on both challenge parts 55 | let [challengeInit, challengeMutations] = [...matches]; 56 | // Perform the necessary parsing 57 | challengeInit = challengeInit.replace(/[;}]/g, ""); 58 | challengeMutations = challengeMutations 59 | .split(";") 60 | .map(s => s.match(/(.=.)?(\(\(!\+).*/g)) 61 | .filter(s => s !== null) 62 | .map(s => s[0]); 63 | 64 | return [challengeInit, challengeMutations]; 65 | } 66 | 67 | static _matchChallengeFromScript(script) { 68 | let testMatches = script.match(/(.=\+)?(\(\(!\+).*/g); // Match the challenge part 69 | if (testMatches.length === 2) { 70 | return testMatches; 71 | } 72 | throw Error("Failed to match JS challenge with Regular Expressions"); 73 | } 74 | 75 | static solveChallenge(response) { 76 | let {html, host, origin} = {html: response.body, host: response.host, origin: response.origin}; 77 | let script = this._extractChallengeFromHTML(html); 78 | let [vc, pass] = [...this._extractInputValuesFromHTML(html)]; 79 | 80 | try { 81 | // Parse only the actual math challenge parts from the script tag and assign them 82 | let challengeMatches = this._matchChallengeFromScript(script); 83 | let [challengeInit, challengeMutations] = this._parseChallenge(challengeMatches); 84 | let answer = this._buildAnswer(challengeMutations, safeEval(challengeInit)); 85 | answer = parseFloat(answer.toFixed(10)) + host.length; 86 | 87 | return {vc: vc, pass: pass, answer: answer, origin: origin}; 88 | } catch (err) { 89 | throw Error(`Could not solve or parse JavaScript challenge. Caused due to error:\n${err}`); 90 | } 91 | } 92 | } 93 | 94 | module.exports = Solver; 95 | -------------------------------------------------------------------------------- /src/ua.text: -------------------------------------------------------------------------------- 1 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 2 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 3 | Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36 4 | Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 5 | Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 6 | Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 7 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 8 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36 9 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 10 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 11 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36 12 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 13 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36 14 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36 15 | Mozilla/5.0 (Linux; Android 6.0.1; SM-G532G Build/MMB29T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.83 Mobile Safari/537.36 16 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36 17 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 18 | Mozilla/5.0 (Linux; Android 6.0; vivo 1713 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36 19 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36 20 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36 21 | Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 22 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36 23 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 24 | Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36 25 | Mozilla/5.0 (Linux; Android 7.1; Mi A1 Build/N2G47H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36 26 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.65 Safari/537.36 27 | Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 28 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36 29 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36 30 | Mozilla/5.0 (Windows NT 6.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 31 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 32 | Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 33 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 34 | Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 35 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36 36 | Mozilla/5.0 (Linux; Android 6.0.1; vivo 1603 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36 37 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 38 | Mozilla/5.0 (Linux; Android 6.0.1; CPH1607 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/63.0.3239.111 Mobile Safari/537.36 39 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36 40 | Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36 41 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.104 Safari/537.36 42 | Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 43 | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36 44 | Mozilla/5.0 (Linux; Android 6.0.1; Redmi 4A Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.116 Mobile Safari/537.36 45 | Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 46 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 47 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36 48 | Mozilla/5.0 (Linux; Android 6.0; vivo 1606 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36 49 | Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36 -------------------------------------------------------------------------------- /tests/main.test.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const Humanoid = require("../index"); 3 | const Response = require("../src/response"); 4 | const Solver = require("../src/solver"); 5 | const humanoidReqHandler = require("../src/humanoidReqHandler"); 6 | 7 | 8 | const testHumanoid = new Humanoid() 9 | const testRequestHandler = new humanoidReqHandler(); 10 | const testResponse = new Response(); 11 | 12 | 13 | test("Response default values", () => { 14 | expect(testResponse.isChallengeSolved).toBeFalsy(); 15 | expect(testResponse.isSessionChallenged).toBeFalsy(); 16 | expect(testResponse.cookies).toBeNull(); 17 | }) 18 | 19 | test("Random User Agent", () => { 20 | expect(testRequestHandler._getRandomUA()).toEqual(expect.stringContaining("Mozilla/5.0")); 21 | }) 22 | 23 | test("Parsing URL to object with origin/host/etc", () => { 24 | let url = testRequestHandler._parseUrl("http://www.google.com"); 25 | expect(url).toHaveProperty("href"); 26 | expect(url).toHaveProperty("origin"); 27 | expect(url).toHaveProperty("host"); 28 | }) 29 | 30 | test("Getting configuration for HTTP GET with data", () => { 31 | let getConf = testRequestHandler._getConfForMethod("GET", {}, {a: 1}); 32 | expect(getConf.qs).toEqual({a: 1}); 33 | }) 34 | 35 | 36 | test("Getting configuration for HTTP POST with data", () => { 37 | let postConfJSON = testRequestHandler._getConfForMethod("POST", {}, {b: 2}, "json"); 38 | expect(postConfJSON.body).toEqual({b: 2}); 39 | expect(postConfJSON.json).toBeTruthy(); 40 | let postConfForm = testRequestHandler._getConfForMethod("POST", {}, {c: 3}, "form"); 41 | expect(postConfForm.form).toEqual({c: 3}); 42 | expect(postConfForm.json).toBeUndefined(); 43 | }) 44 | 45 | test("CloudFlare JS challenge in page", () => { 46 | let challengeHTML = fs.readFileSync(`${__dirname}/../page_samples/sample_challenge_page.html`); 47 | expect(testRequestHandler.isChallengeInResponse(challengeHTML)).toBeTruthy(); 48 | }) 49 | 50 | test("CloudFlare JS challenge not in page", () => { 51 | let noChallengeHTML = fs.readFileSync(`${__dirname}/../page_samples/sample_nonchallenge.html`); 52 | expect(testRequestHandler.isChallengeInResponse(noChallengeHTML)).toBeFalsy(); 53 | }) 54 | 55 | test("Get request headers", () => { 56 | let headers = testRequestHandler._getRequestHeaders("https://www.google.com"); 57 | expect(headers).toMatchObject({ 58 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 59 | "Accept-Encoding": "gzip, deflate, br", 60 | "Connection": "keep-alive", 61 | "Host": expect.anything(), 62 | "User-Agent": expect.anything() 63 | }) 64 | }) 65 | 66 | test("Humanoid default values", () => { 67 | expect(testHumanoid.maxRetries).toBe(3); 68 | expect(testHumanoid.autoBypass).toBeTruthy(); 69 | }) 70 | 71 | test("Reset current max retries", () => { 72 | testHumanoid.currMaxRetries--; 73 | expect(testHumanoid.currMaxRetries).toBe(2); 74 | 75 | testHumanoid._resetCurrMaxRetries(); 76 | expect(testHumanoid.currMaxRetries).toBe(3); 77 | }) 78 | 79 | test("Clear cookies from jar", () => { 80 | let emptyJar = testHumanoid.cookieJar._jar.store.idx 81 | testHumanoid.cookieJar._jar.store.idx = "google.com" 82 | expect(emptyJar).not.toEqual(testHumanoid.cookieJar._jar.store.idx) 83 | 84 | testHumanoid.clearCookies(); 85 | expect(emptyJar).toEqual(testHumanoid.cookieJar._jar.store.idx) 86 | }) 87 | 88 | test("Build answer object 1", () => { 89 | let answerObject = testHumanoid._buildAnswerObject([1.123123, 3.123123123, 2.19283918]); 90 | expect(answerObject).toMatchObject({ 91 | jschl_vc: 1.123123, 92 | pass: 3.123123123, 93 | jschl_answer: 2.19283918 94 | }) 95 | }) 96 | 97 | test("Build answer object 2", () => { 98 | let answerObject = testHumanoid._buildAnswerObject([1, 2, 3]); 99 | expect(answerObject).toMatchObject({ 100 | jschl_vc: 1, 101 | pass: 2, 102 | jschl_answer: 3 103 | }) 104 | }) 105 | 106 | test("Extract timeout from script", () => { 107 | let challengeHTML = fs.readFileSync(`${__dirname}/../page_samples/sample_challenge_page.html`); 108 | let timeout = Solver._extractTimeoutFromScript(challengeHTML); 109 | expect(timeout).toBe("4000"); 110 | expect(parseInt(timeout)).toBe(4000); 111 | }) 112 | 113 | test("Extract input form values", () => { 114 | let challengeHTML = fs.readFileSync(`${__dirname}/../page_samples/sample_challenge_page.html`); 115 | let [a, b] = Solver._extractInputValuesFromHTML(challengeHTML); 116 | expect(a).not.toBeNull(); 117 | expect(b).not.toBeNull(); 118 | }) 119 | 120 | test("Extract challenge", () => { 121 | let challengeHTML = fs.readFileSync(`${__dirname}/../page_samples/sample_challenge_page.html`); 122 | let chal = Solver._extractChallengeFromHTML(challengeHTML); 123 | expect([chal,chal,chal]).toEqual([ 124 | expect.stringContaining("f.action += location.hash"), 125 | expect.stringContaining("var s,t,o,p,b,r,e,a,k,i,n,g,f"), 126 | expect.stringContaining("setTimeout(function(){") 127 | ]); 128 | }) 129 | 130 | // TODO: Add tests for request sending/challenge solving. Mock functions as needed --------------------------------------------------------------------------------