├── .node-version ├── Dockerfile ├── example.toml ├── package.json ├── .gitignore ├── README.md └── index.js /.node-version: -------------------------------------------------------------------------------- 1 | 10.15.3 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:10-alpine 2 | WORKDIR /app 3 | COPY package*.json ./ 4 | RUN npm install 5 | COPY . . 6 | EXPOSE 8080 7 | CMD [ "npm", "start" ] -------------------------------------------------------------------------------- /example.toml: -------------------------------------------------------------------------------- 1 | COOKIE_NAME = 'searx' 2 | ENDPOINT_NAME = 'searx.css' 3 | HTML_TAG_REPLACE = '(

)|()' 4 | JAIL_PATH = '/tmp' 5 | MAX_RETRY = 5 6 | PORT = 3000 7 | TARGET = '127.0.0.1:8888' 8 | TIMEOUT_LOAD = 120 9 | WHITELIST = ['8.8.8.8', '1.1.1.1'] 10 | 11 | [WHITELIST_PAGES] 12 | GET = ['^/$', '^/autocompleter(.*)$', '^/favicon.ico$', '^/about$', '^/robots.txt$', '^/opensearch.xml$'] 13 | POST = ['^/preferences$'] -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "antibot-proxy", 3 | "version": "0.0.1", 4 | "description": "Proxy designed to combat against basic bots that aren't a full browser.", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js", 8 | "test": "echo \"Error: no test specified\" && exit 1", 9 | "debug": "DEBUG=express-http-proxy node index.js" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/unixfox/antibot-proxy.git" 14 | }, 15 | "author": "unixfox", 16 | "license": "ISC", 17 | "bugs": { 18 | "url": "https://github.com/unixfox/antibot-proxy/issues" 19 | }, 20 | "homepage": "https://github.com/unixfox/antibot-proxy#readme", 21 | "dependencies": { 22 | "@iarna/toml": "^2.2.3", 23 | "better-logging": "^3.4.0", 24 | "cookie-parser": "^1.4.4", 25 | "express": "^4.17.1", 26 | "express-http-proxy": "^1.5.1", 27 | "express-useragent": "^1.0.13" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Optional REPL history 57 | .node_repl_history 58 | 59 | # Output of 'npm pack' 60 | *.tgz 61 | 62 | # Yarn Integrity file 63 | .yarn-integrity 64 | 65 | # dotenv environment variables file 66 | .env 67 | .env.test 68 | 69 | # parcel-bundler cache (https://parceljs.org/) 70 | .cache 71 | 72 | # next.js build output 73 | .next 74 | 75 | # nuxt.js build output 76 | .nuxt 77 | 78 | # vuepress build output 79 | .vuepress/dist 80 | 81 | # Serverless directories 82 | .serverless/ 83 | 84 | # FuseBox cache 85 | .fusebox/ 86 | 87 | # DynamoDB Local files 88 | .dynamodb/ 89 | 90 | package-lock.json 91 | 92 | config.toml 93 | views/bot.pug 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![GitHub stars](https://img.shields.io/github/stars/unixfox/antibot-proxy.svg?style=social) [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/unixfox/antibot-proxy.svg)](https://hub.docker.com/r/unixfox/antibot-proxy) [![Docker Cloud Automated build](https://img.shields.io/docker/cloud/automated/unixfox/antibot-proxy.svg)](https://hub.docker.com/r/unixfox/antibot-proxy) ![GitHub package.json version](https://img.shields.io/github/package-json/v/unixfox/antibot-proxy.svg) 2 | 3 | # Description 4 | 5 | :warning: This program is still experimental so there are probably some bugs and vulnerabilities in the bot detection system. :warning: 6 | 7 | The algorithm is based on the fact that every browser will accept cookie and load the external CSS that you included in your HTML page so in case of a basic bot it won't simply process the cookie or/and load that external CSS file because it does not process the HTML code. 8 | 9 | # How to make it work? (minimalistic configuration) 10 | 11 | 1. Copy the `example.toml` to `config.toml`. 12 | 13 | 2. Modify the setting `TARGET` in the `config.toml` to the URL where your application is listening to. For example if your application reachable on `http://127.0.0.1:8080` just set `127.0.0.1:8080` in the setting. 14 | 15 | 3. Modify the setting `JAIL_PATH` in the `config.toml` to an empty directory that you created for the application. 16 | 17 | 3. On your main webserver (nginx, apache, caddy,...) when you will proxy the `antibot-proxy` application you will also need to pass the IP address of the client to the application (with the `X-Real-IP` header). Here is how to do it on: 18 | - Apache: 19 | 20 | ```apache 21 | RemoteIPHeader X-Real-IP 22 | ``` 23 | 24 | - NGINX: 25 | 26 | ```nginx 27 | proxy_set_header X-Real-IP $remote_addr; 28 | ``` 29 | 30 | - Caddy: 31 | 32 | ```caddy 33 | transparent 34 | ``` 35 | > Note: You may consult the documentation of your webserver for further details about proxying an application. 36 | 37 | # Reference of each setting in `config.toml` for advanced configuration 38 | 39 | - `COOKIE_NAME`: The name of the cookie that will be used for checking if the client can handle the cookie. 40 | - `ENDPOINT_NAME`: The name of the CSS file that will be used for checking if the client can process the HTML code. 41 | - `JAIL_PATH`: The path to the directory where the banned IP will be stored. Optionally to be used with the [ipfilter](https://caddyserver.com/docs/http.ipfilter) plugin of Caddy. 42 | - `MAX_RETRY`: The number of retries allowed for the client before getting banned. 43 | - `PORT`: The port of which the proxy application will listen to. 44 | - `TARGET`: The URL of the application to proxy/protect. 45 | - `TIMEOUT_LOAD`: The time before the program consider that the client failed to reach the CSS file. 46 | - `WHITELIST`: The IP that you want to be whitelisted. 47 | - `WHITELIST_PAGES`: Pages to whitelist from the blocked page for bots. 48 | 49 | 50 | # How to configure the ipfilter Caddy plugin to handle the banned IPs? 51 | 52 | You just need to add this block to your `Caddyfile`: 53 | ````JSON 54 | ipfilter / { 55 | rule block 56 | prefix_dir JAIL_PATH 57 | } 58 | ```` 59 | and change `JAIL_PATH` with the value of `JAIL_PATH` that you have previously defined in the `config.toml` file. 60 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const proxy = require('express-http-proxy'); 2 | const express = require('express'); 3 | const userAgentExpress = require('express-useragent'); 4 | const fs = require('fs'); 5 | const app = express(); 6 | const crypto = require('crypto'); 7 | const cookieParser = require('cookie-parser'); 8 | const TOML = require('@iarna/toml'); 9 | require('better-logging')(console, { 10 | format: ctx => `${ctx.date} ${ctx.time24} ${ctx.type} ${ctx.msg}` 11 | }); 12 | 13 | function checkFileExist(path, exit) { 14 | if (fs.existsSync(path)) 15 | return (true); 16 | else 17 | if (exit) { 18 | console.error("The file " + path + " doesn't exist, can't continue. Please check the documentation for further details."); 19 | process.exit(1); 20 | } 21 | else 22 | return (false); 23 | } 24 | 25 | checkFileExist("config.toml", true); 26 | const configFile = TOML.parse(fs.readFileSync('./config.toml')); 27 | 28 | let timeoutObject = new Object(); 29 | let failedCounter = new Object(); 30 | 31 | function addFailedCounter(IP) { 32 | if (!failedCounter[IP]) 33 | failedCounter[IP] = 1; 34 | else if (failedCounter[IP] >= (configFile.MAX_RETRY - 1)) { 35 | fs.closeSync(fs.openSync(configFile.JAIL_PATH + "/" + IP, 'w')); 36 | clearTimeout(timeoutObject[IP]); 37 | delete failedCounter[IP]; 38 | delete timeoutObject[IP]; 39 | console.warn(IP + " - Bot detected: adding to the jail list!"); 40 | } 41 | else if (failedCounter[IP] < (configFile.MAX_RETRY - 1)) 42 | failedCounter[IP] += 1; 43 | } 44 | 45 | function whitelistPageChecker(fullURL, reqMethod) { 46 | let matching = false; 47 | for (const regex of configFile["WHITELIST_PAGES"][reqMethod]) { 48 | if (new RegExp(regex).test(fullURL)) { 49 | matching = true; 50 | break; 51 | } 52 | } 53 | return (matching); 54 | } 55 | 56 | app.use(cookieParser()); 57 | 58 | app.get("/" + configFile.ENDPOINT_NAME, function (userReq, userRes) { 59 | const IP = (userReq.headers["x-real-ip"] || userReq.connection.remoteAddress); 60 | clearTimeout(timeoutObject[IP]); 61 | userRes.setHeader('Content-Type', 'text/css'); 62 | userRes.setHeader('Cache-Control', 'no-store'); 63 | userRes.end(); 64 | }); 65 | 66 | app.all("*", function (userReq, userRes, next) { 67 | const IP = (userReq.headers["x-real-ip"] || userReq.connection.remoteAddress); 68 | const secretCookie = crypto.createHash('md5').update(IP).digest('hex'); 69 | if ((userReq.cookies && userReq.cookies[configFile.COOKIE_NAME] === secretCookie) 70 | || configFile.WHITELIST.indexOf(IP) > -1 || whitelistPageChecker(userReq.url, userReq.method)) 71 | next(); 72 | else { 73 | userRes.cookie(configFile.COOKIE_NAME, secretCookie); 74 | userRes.redirect(307, userReq.url); 75 | userRes.end(); 76 | } 77 | }); 78 | 79 | app.all('*', proxy(configFile.TARGET, { 80 | timeout: 10000, 81 | preserveHostHdr: true, 82 | filter: function (userReq, userRes) { 83 | const IP = (userReq.headers["x-real-ip"] || userReq.connection.remoteAddress); 84 | const userAgent = userAgentExpress.parse(userReq.headers['user-agent']); 85 | if ((userAgent.isBot || userAgent.isCurl || (userAgent.isChrome && !userReq.rawHeaders.includes("Accept-Language"))) 86 | && !whitelistPageChecker(userReq.url, userReq.method) && configFile.WHITELIST.indexOf(IP) == -1) 87 | return false; 88 | else if (checkFileExist(configFile.JAIL_PATH + "/" + IP, false)) 89 | return false; 90 | else if (userReq.method == 'GET') 91 | return true; 92 | else if (userReq.method == 'POST') 93 | return true; 94 | else 95 | return false; 96 | }, 97 | userResHeaderDecorator(headers, userReq, userRes, proxyReq, proxyRes) { 98 | const IP = (userReq.headers["x-real-ip"] || userReq.connection.remoteAddress); 99 | if (headers['content-type'] && configFile.WHITELIST.indexOf(IP) == -1 && !whitelistPageChecker(userReq.url, userReq.method)) { 100 | if (headers['content-type'].includes("text/html") && (proxyRes.statusCode >= 200 && proxyRes.statusCode < 400)) { 101 | if (!timeoutObject[IP] || timeoutObject[IP]._called) { 102 | timeoutObject[IP] = setTimeout(function () { 103 | console.info(IP + " - Client did not reach the anti bot endpoint!"); 104 | addFailedCounter(IP); 105 | if (timeoutObject[IP]) { 106 | timeoutObject[IP]._called = true; 107 | } 108 | }, configFile.TIMEOUT_LOAD * 1000); 109 | timeoutObject[IP]._called = false; 110 | } 111 | else if (timeoutObject[IP]._called == false && 112 | timeoutObject[IP]._idleTimeout != -1) { 113 | console.info(IP + " - Bot behavior detected : loading of a new HTML page before reaching the anti bot endpoint!"); 114 | timeoutObject[IP].refresh(); 115 | addFailedCounter(IP); 116 | } 117 | } 118 | } 119 | return headers; 120 | }, 121 | userResDecorator: function (proxyRes, proxyResData, userReq, userRes) { 122 | if (proxyRes.headers['content-type'].includes("text/html") && (proxyRes.statusCode >= 200 && proxyRes.statusCode < 400)) { 123 | const data = proxyResData.toString('utf8').replace(new RegExp(configFile.HTML_TAG_REPLACE, "g"), function (match) { 124 | return (match + '') 125 | }); 126 | return (data); 127 | } 128 | else 129 | return (proxyResData); 130 | } 131 | })); 132 | 133 | app.all("*", function (userReq, userRes) { 134 | userRes.status(403); 135 | userRes.end(); 136 | }); 137 | 138 | app.listen(configFile.PORT, () => { 139 | console.log("Listening on port " + configFile.PORT); 140 | }); 141 | --------------------------------------------------------------------------------