├── LICENCE ├── README.md ├── crawler-user-agents.json ├── custom.json ├── generate └── generate.go ├── go.mod ├── go.sum ├── isbot.go ├── isbot_test.go └── user-agents-bots.txt /LICENCE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Stephen Afam-Osemene 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IsBot 2 | 3 | Detect bots/crawlers/spiders using the user agent string. 4 | 5 | Using a list downloaded from , the regexes in this package hit a 90% detection rate. 6 | If nothing matches, it also checks if the user agent is present in the list. 7 | 8 | ## Usage 9 | 10 | ```go 11 | import "github.com/stephenafamo/isbot" 12 | 13 | func main() { 14 | userAgent := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.0.0 Safari/537.36 Chrome-Lighthouse" 15 | isBot := isbot.Check(userAgent) 16 | if isBot { 17 | // do something 18 | } 19 | } 20 | ``` 21 | 22 | ## Sources 23 | 24 | The sources for detection are: 25 | 26 | * 27 | * 28 | * A manual list 29 | 30 | ## Contributing 31 | 32 | Run `go generate` to refresh the lists from the sources. 33 | 34 | If something is misisng, send in a pull request. 35 | -------------------------------------------------------------------------------- /crawler-user-agents.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pattern": "Googlebot\\/", 4 | "url": "http://www.google.com/bot.html", 5 | "instances": [ 6 | "Googlebot/2.1 (+http://www.google.com/bot.html)", 7 | "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 8 | "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 9 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 10 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 11 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 12 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 13 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36" 14 | ] 15 | } 16 | , 17 | { 18 | "pattern": "Googlebot-Mobile", 19 | "instances": [ 20 | "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", 21 | "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", 22 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", 23 | "Nokia6820/2.0 (4.83) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)", 24 | "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" 25 | ] 26 | } 27 | , 28 | { 29 | "pattern": "Googlebot-Image", 30 | "instances": [ 31 | "Googlebot-Image/1.0" 32 | ] 33 | } 34 | , 35 | { 36 | "pattern": "Googlebot-News", 37 | "instances": [ 38 | "Googlebot-News" 39 | ] 40 | } 41 | , 42 | { 43 | "pattern": "Googlebot-Video", 44 | "instances": [ 45 | "Googlebot-Video/1.0" 46 | ] 47 | } 48 | , 49 | { 50 | "pattern": "AdsBot-Google([^-]|$)", 51 | "url": "https://support.google.com/webmasters/answer/1061943?hl=en", 52 | "instances": [ 53 | "AdsBot-Google (+http://www.google.com/adsbot.html)" 54 | ] 55 | } 56 | , 57 | { 58 | "pattern": "AdsBot-Google-Mobile", 59 | "addition_date": "2017/08/21", 60 | "url": "https://support.google.com/adwords/answer/2404197", 61 | "instances": [ 62 | "AdsBot-Google-Mobile-Apps", 63 | "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", 64 | "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)" 65 | ] 66 | } 67 | , 68 | { 69 | "pattern": "Feedfetcher-Google", 70 | "addition_date": "2018/06/27", 71 | "url": "https://support.google.com/webmasters/answer/178852", 72 | "instances": [ 73 | "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=728742641706423)" 74 | ] 75 | } 76 | , 77 | { 78 | "pattern": "Mediapartners-Google", 79 | "url": "https://support.google.com/webmasters/answer/1061943?hl=en", 80 | "instances": [ 81 | "Mediapartners-Google", 82 | "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa/4.0 (Following Mediapartners-Google)", 83 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 10_0 like Mac OS X; en-us) AppleWebKit/602.1.38 (KHTML, like Gecko) Version/10.0 Mobile/14A5297c Safari/602.1 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)", 84 | "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)" 85 | ] 86 | } 87 | , 88 | { 89 | "pattern": "Mediapartners \\(Googlebot\\)", 90 | "addition_date": "2017/08/08", 91 | "url": "https://support.google.com/webmasters/answer/1061943?hl=en", 92 | "instances": [] 93 | } 94 | , 95 | { 96 | "pattern": "APIs-Google", 97 | "addition_date": "2017/08/08", 98 | "url": "https://support.google.com/webmasters/answer/1061943?hl=en", 99 | "instances": [ 100 | "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)" 101 | ] 102 | } 103 | , 104 | { 105 | "pattern": "Google-InspectionTool", 106 | "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", 107 | "instances": [ 108 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)", 109 | "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)" 110 | ] 111 | } 112 | , 113 | { 114 | "pattern": "Storebot-Google", 115 | "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", 116 | "instances": [ 117 | "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", 118 | "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36" 119 | ] 120 | } 121 | , 122 | { 123 | "pattern": "GoogleOther", 124 | "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", 125 | "instances": [ 126 | "GoogleOther" 127 | ] 128 | } 129 | , 130 | { 131 | "pattern": "bingbot", 132 | "url": "http://www.bing.com/bingbot.htm", 133 | "instances": [ 134 | "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", 135 | "Mozilla/5.0 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", 136 | "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", 137 | "Mozilla/5.0 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", 138 | "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm", 139 | "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 140 | "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) SitemapProbe", 141 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; http://www.bing.com/bingbot.htm)", 142 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", 143 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)", 144 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 145 | "Mozilla/5.0 (seoanalyzer; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 146 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Safari/537.36", 147 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36" 148 | ] 149 | } 150 | , 151 | { 152 | "pattern": "Slurp", 153 | "url": "http://help.yahoo.com/help/us/ysearch/slurp", 154 | "instances": [ 155 | "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", 156 | "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", 157 | "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)" 158 | ] 159 | } 160 | , 161 | { 162 | "pattern": "[wW]get", 163 | "instances": [ 164 | "WGETbot/1.0 (+http://wget.alanreed.org)", 165 | "Wget/1.14 (linux-gnu)", 166 | "Wget/1.20.3 (linux-gnu)" 167 | ] 168 | } 169 | , 170 | { 171 | "pattern": "LinkedInBot", 172 | "instances": [ 173 | "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", 174 | "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)", 175 | "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)" 176 | ] 177 | } 178 | , 179 | { 180 | "pattern": "Python-urllib", 181 | "instances": [ 182 | "Python-urllib/1.17", 183 | "Python-urllib/2.5", 184 | "Python-urllib/2.6", 185 | "Python-urllib/2.7", 186 | "Python-urllib/3.1", 187 | "Python-urllib/3.2", 188 | "Python-urllib/3.3", 189 | "Python-urllib/3.4", 190 | "Python-urllib/3.5", 191 | "Python-urllib/3.6", 192 | "Python-urllib/3.7" 193 | ] 194 | } 195 | , 196 | { 197 | "pattern": "python-requests", 198 | "addition_date": "2018/05/27", 199 | "instances": [ 200 | "python-requests/2.9.2", 201 | "python-requests/2.11.1", 202 | "python-requests/2.18.4", 203 | "python-requests/2.19.1", 204 | "python-requests/2.20.0", 205 | "python-requests/2.21.0", 206 | "python-requests/2.22.0" 207 | ] 208 | } 209 | , 210 | { 211 | "pattern": "aiohttp", 212 | "addition_date": "2019/12/23", 213 | "instances": [ 214 | "Python/3.9 aiohttp/3.7.3", 215 | "Python/3.8 aiohttp/3.7.2", 216 | "Python/3.7 aiohttp/3.6.2a2" 217 | ], 218 | "url": "https://docs.aiohttp.org/en/stable/" 219 | } 220 | , 221 | { 222 | "pattern": "httpx", 223 | "addition_date":" 2019/12/23", 224 | "instances": [ 225 | "python-httpx/0.16.1", 226 | "python-httpx/0.13.0.dev1" 227 | 228 | ], 229 | "url": "https://www.python-httpx.org" 230 | } 231 | , 232 | { 233 | "pattern": "libwww-perl", 234 | "instances": [ 235 | "2Bone_LinkChecker/1.0 libwww-perl/6.03", 236 | "2Bone_LinkChkr/1.0 libwww-perl/6.03", 237 | "amibot - http://www.amidalla.de - tech@amidalla.com libwww-perl/5.831" 238 | ] 239 | } 240 | , 241 | { 242 | "pattern": "httpunit", 243 | "instances": [ 244 | "httpunit/1.x" 245 | ] 246 | } 247 | , 248 | { 249 | "pattern": "nutch", 250 | "instances": [ 251 | "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", 252 | "istellabot-nutch/Nutch-1.10" 253 | ] 254 | } 255 | , 256 | { 257 | "pattern": "Go-http-client", 258 | "addition_date": "2016/03/26", 259 | "url": "https://golang.org/pkg/net/http/", 260 | "instances": [ 261 | "Go-http-client/1.1", 262 | "Go-http-client/2.0" 263 | ] 264 | } 265 | , 266 | { 267 | "pattern": "phpcrawl", 268 | "addition_date": "2012-09/17", 269 | "url": "http://phpcrawl.cuab.de/", 270 | "instances": [ 271 | "phpcrawl" 272 | ] 273 | } 274 | , 275 | { 276 | "pattern": "msnbot", 277 | "url": "http://search.msn.com/msnbot.htm", 278 | "instances": [ 279 | "adidxbot/1.1 (+http://search.msn.com/msnbot.htm)", 280 | "adidxbot/2.0 (+http://search.msn.com/msnbot.htm)", 281 | "librabot/1.0 (+http://search.msn.com/msnbot.htm)", 282 | "librabot/2.0 (+http://search.msn.com/msnbot.htm)", 283 | "msnbot-NewsBlogs/2.0b (+http://search.msn.com/msnbot.htm)", 284 | "msnbot-UDiscovery/2.0b (+http://search.msn.com/msnbot.htm)", 285 | "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)", 286 | "msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)", 287 | "msnbot-media/2.0b (+http://search.msn.com/msnbot.htm)", 288 | "msnbot/1.0 (+http://search.msn.com/msnbot.htm)", 289 | "msnbot/1.1 (+http://search.msn.com/msnbot.htm)", 290 | "msnbot/2.0b (+http://search.msn.com/msnbot.htm)", 291 | "msnbot/2.0b (+http://search.msn.com/msnbot.htm).", 292 | "msnbot/2.0b (+http://search.msn.com/msnbot.htm)._" 293 | ] 294 | } 295 | , 296 | { 297 | "pattern": "jyxobot", 298 | "instances": [] 299 | } 300 | , 301 | { 302 | "pattern": "FAST-WebCrawler", 303 | "instances": [ 304 | "FAST-WebCrawler/3.6/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", 305 | "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)", 306 | "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)", 307 | "FAST-WebCrawler/3.8" 308 | ] 309 | } 310 | , 311 | { 312 | "pattern": "FAST Enterprise Crawler", 313 | "instances": [ 314 | "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/", 315 | "FAST Enterprise Crawler 6 used by Schibsted (webcrawl@schibstedsok.no)" 316 | ] 317 | } 318 | , 319 | { 320 | "pattern": "BIGLOTRON", 321 | "instances": [ 322 | "BIGLOTRON (Beta 2;GNU/Linux)" 323 | ] 324 | } 325 | , 326 | { 327 | "pattern": "Teoma", 328 | "instances": [ 329 | "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", 330 | "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://about.ask.com/en/docs/about/webmasters.shtml)" 331 | ], 332 | "url": "http://about.ask.com/en/docs/about/webmasters.shtml" 333 | } 334 | , 335 | { 336 | "pattern": "convera", 337 | "instances": [ 338 | "ConveraCrawler/0.9e (+http://ews.converasearch.com/crawl.htm)" 339 | ], 340 | "url": "http://ews.converasearch.com/crawl.htm" 341 | } 342 | , 343 | { 344 | "pattern": "seekbot", 345 | "instances": [ 346 | "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2" 347 | ], 348 | "url": "http://www.seekbot.net/bot.html" 349 | } 350 | , 351 | { 352 | "pattern": "Gigabot", 353 | "instances": [ 354 | "Gigabot/1.0", 355 | "Gigabot/2.0 (http://www.gigablast.com/spider.html)" 356 | ], 357 | "url": "http://www.gigablast.com/spider.html" 358 | } 359 | , 360 | { 361 | "pattern": "Gigablast", 362 | "instances": [ 363 | "GigablastOpenSource/1.0" 364 | ], 365 | "url": "https://github.com/gigablast/open-source-search-engine" 366 | } 367 | , 368 | { 369 | "pattern": "exabot", 370 | "instances": [ 371 | "Mozilla/5.0 (compatible; Alexabot/1.0; +http://www.alexa.com/help/certifyscan; certifyscan@alexa.com)", 372 | "Mozilla/5.0 (compatible; Exabot PyExalead/3.0; +http://www.exabot.com/go/robot)", 373 | "Mozilla/5.0 (compatible; Exabot-Images/3.0; +http://www.exabot.com/go/robot)", 374 | "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)", 375 | "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", 376 | "Mozilla/5.0 (compatible; Exabot/3.0; http://www.exabot.com/go/robot)" 377 | ] 378 | } 379 | , 380 | { 381 | "pattern": "ia_archiver", 382 | "instances": [ 383 | "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)", 384 | "ia_archiver-web.archive.org" 385 | ] 386 | } 387 | , 388 | { 389 | "pattern": "GingerCrawler", 390 | "instances": [ 391 | "GingerCrawler/1.0 (Language Assistant for Dyslexics; www.gingersoftware.com/crawler_agent.htm; support at ginger software dot com)" 392 | ] 393 | } 394 | , 395 | { 396 | "pattern": "webmon ", 397 | "instances": [] 398 | } 399 | , 400 | { 401 | "pattern": "HTTrack", 402 | "instances": [ 403 | "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" 404 | ] 405 | } 406 | , 407 | { 408 | "pattern": "grub.org", 409 | "instances": [ 410 | "Mozilla/4.0 (compatible; grub-client-0.3.0; Crawl your own stuff with http://grub.org)", 411 | "Mozilla/4.0 (compatible; grub-client-1.0.4; Crawl your own stuff with http://grub.org)", 412 | "Mozilla/4.0 (compatible; grub-client-1.0.5; Crawl your own stuff with http://grub.org)", 413 | "Mozilla/4.0 (compatible; grub-client-1.0.6; Crawl your own stuff with http://grub.org)", 414 | "Mozilla/4.0 (compatible; grub-client-1.0.7; Crawl your own stuff with http://grub.org)", 415 | "Mozilla/4.0 (compatible; grub-client-1.1.1; Crawl your own stuff with http://grub.org)", 416 | "Mozilla/4.0 (compatible; grub-client-1.2.1; Crawl your own stuff with http://grub.org)", 417 | "Mozilla/4.0 (compatible; grub-client-1.3.1; Crawl your own stuff with http://grub.org)", 418 | "Mozilla/4.0 (compatible; grub-client-1.3.7; Crawl your own stuff with http://grub.org)", 419 | "Mozilla/4.0 (compatible; grub-client-1.4.3; Crawl your own stuff with http://grub.org)", 420 | "Mozilla/4.0 (compatible; grub-client-1.5.3; Crawl your own stuff with http://grub.org)" 421 | ] 422 | } 423 | , 424 | { 425 | "pattern": "UsineNouvelleCrawler", 426 | "instances": [] 427 | } 428 | , 429 | { 430 | "pattern": "antibot", 431 | "instances": [] 432 | } 433 | , 434 | { 435 | "pattern": "netresearchserver", 436 | "instances": [] 437 | } 438 | , 439 | { 440 | "pattern": "speedy", 441 | "instances": [ 442 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)", 443 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) Speedy Spider for SpeedyAds (http://www.entireweb.com/about/search_tech/speedy_spider/)", 444 | "Mozilla/5.0 (compatible; Speedy Spider; http://www.entireweb.com/about/search_tech/speedy_spider/)", 445 | "Speedy Spider (Entireweb; Beta/1.2; http://www.entireweb.com/about/search_tech/speedyspider/)", 446 | "Speedy Spider (http://www.entireweb.com/about/search_tech/speedy_spider/)" 447 | ] 448 | } 449 | , 450 | { 451 | "pattern": "fluffy", 452 | "instances": [] 453 | } 454 | , 455 | { 456 | "pattern": "findlink", 457 | "instances": [ 458 | "findlinks/1.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", 459 | "findlinks/1.1.3-beta8 (+http://wortschatz.uni-leipzig.de/findlinks/)", 460 | "findlinks/1.1.3-beta9 (+http://wortschatz.uni-leipzig.de/findlinks/)", 461 | "findlinks/1.1.5-beta7 (+http://wortschatz.uni-leipzig.de/findlinks/)", 462 | "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/)", 463 | "findlinks/1.1.6-beta1 (+http://wortschatz.uni-leipzig.de/findlinks/; YaCy 0.1; yacy.net)", 464 | "findlinks/1.1.6-beta2 (+http://wortschatz.uni-leipzig.de/findlinks/)", 465 | "findlinks/1.1.6-beta3 (+http://wortschatz.uni-leipzig.de/findlinks/)", 466 | "findlinks/1.1.6-beta4 (+http://wortschatz.uni-leipzig.de/findlinks/)", 467 | "findlinks/1.1.6-beta5 (+http://wortschatz.uni-leipzig.de/findlinks/)", 468 | "findlinks/1.1.6-beta6 (+http://wortschatz.uni-leipzig.de/findlinks/)", 469 | "findlinks/2.0 (+http://wortschatz.uni-leipzig.de/findlinks/)", 470 | "findlinks/2.0.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", 471 | "findlinks/2.0.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", 472 | "findlinks/2.0.4 (+http://wortschatz.uni-leipzig.de/findlinks/)", 473 | "findlinks/2.0.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", 474 | "findlinks/2.0.9 (+http://wortschatz.uni-leipzig.de/findlinks/)", 475 | "findlinks/2.1 (+http://wortschatz.uni-leipzig.de/findlinks/)", 476 | "findlinks/2.1.3 (+http://wortschatz.uni-leipzig.de/findlinks/)", 477 | "findlinks/2.1.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", 478 | "findlinks/2.2 (+http://wortschatz.uni-leipzig.de/findlinks/)", 479 | "findlinks/2.5 (+http://wortschatz.uni-leipzig.de/findlinks/)", 480 | "findlinks/2.6 (+http://wortschatz.uni-leipzig.de/findlinks/)" 481 | ] 482 | } 483 | , 484 | { 485 | "pattern": "msrbot", 486 | "instances": [] 487 | } 488 | , 489 | { 490 | "pattern": "panscient", 491 | "instances": [ 492 | "panscient.com" 493 | ] 494 | } 495 | , 496 | { 497 | "pattern": "yacybot", 498 | "instances": [ 499 | "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", 500 | "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", 501 | "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 502 | "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html", 503 | "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html", 504 | "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html", 505 | "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 506 | "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 507 | "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html", 508 | "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 509 | "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", 510 | "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", 511 | "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html", 512 | "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html", 513 | "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 514 | "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", 515 | "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 516 | "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 517 | "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 518 | "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html", 519 | "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html", 520 | "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", 521 | "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", 522 | "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", 523 | "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", 524 | "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", 525 | "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 526 | "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html", 527 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html", 528 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html", 529 | "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 530 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 531 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", 532 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 533 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html", 534 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", 535 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html", 536 | "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 537 | "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html", 538 | "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 539 | "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html", 540 | "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html", 541 | "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", 542 | "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", 543 | "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", 544 | "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", 545 | "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html" 546 | ] 547 | } 548 | , 549 | { 550 | "pattern": "AISearchBot", 551 | "instances": [] 552 | } 553 | , 554 | { 555 | "pattern": "ips-agent", 556 | "instances": [ 557 | "BlackBerry9000/4.6.0.167 Profile/MIDP-2.0 Configuration/CLDC-1.1 VendorID/102 ips-agent", 558 | "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.12; ips-agent) Gecko/20050922 Fedora/1.0.7-1.1.fc4 Firefox/1.0.7", 559 | "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.1.3; ips-agent) Gecko/20090824 Fedora/1.0.7-1.1.fc4 Firefox/3.5.3", 560 | "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.24; ips-agent) Gecko/20111107 Ubuntu/10.04 (lucid) Firefox/3.6.24", 561 | "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:14.0; ips-agent) Gecko/20100101 Firefox/14.0.1" 562 | ] 563 | } 564 | , 565 | { 566 | "pattern": "tagoobot", 567 | "instances": [] 568 | } 569 | , 570 | { 571 | "pattern": "MJ12bot", 572 | "instances": [ 573 | "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", 574 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", 575 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)", 576 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)", 577 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)", 578 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)", 579 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)", 580 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)", 581 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)", 582 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)", 583 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)", 584 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)", 585 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)", 586 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)", 587 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)", 588 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)", 589 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)", 590 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)", 591 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)", 592 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" 593 | ] 594 | } 595 | , 596 | { 597 | "pattern": "woriobot", 598 | "instances": [ 599 | "Mozilla/5.0 (compatible; woriobot +http://worio.com)", 600 | "Mozilla/5.0 (compatible; woriobot support [at] zite [dot] com +http://zite.com)" 601 | ] 602 | } 603 | , 604 | { 605 | "pattern": "yanga", 606 | "instances": [ 607 | "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)" 608 | ] 609 | } 610 | , 611 | { 612 | "pattern": "buzzbot", 613 | "instances": [ 614 | "Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)" 615 | ] 616 | } 617 | , 618 | { 619 | "pattern": "mlbot", 620 | "instances": [ 621 | "MLBot (www.metadatalabs.com/mlbot)" 622 | ] 623 | } 624 | , 625 | { 626 | "pattern": "yandex\\.com\\/bots", 627 | "url": "https://yandex.ru/support/webmaster/robot-workings/check-yandex-robots.html#robot-in-logs", 628 | "instances": [ 629 | "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", 630 | "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)", 631 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)", 632 | "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)", 633 | "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots", 634 | "Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots", 635 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)", 636 | "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)", 637 | "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots yabs01)", 638 | "Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)", 639 | "Mozilla/5.0 (compatible; YandexMetrika/4.0; +http://yandex.com/bots)", 640 | "Mozilla/5.0 (compatible; YandexTurbo/1.0; +http://yandex.com/bots)", 641 | "Mozilla/5.0 (compatible; YandexVideoParser/1.0; +http://yandex.com/bots)", 642 | "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)", 643 | "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)", 644 | "Mozilla/5.0 (compatible; YandexAdNet/1.0; +http://yandex.com/bots)", 645 | "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)", 646 | "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)", 647 | "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)", 648 | "Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots", 649 | "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)", 650 | "Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)", 651 | "Mozilla/5.0 (compatible; YandexForDomain/1.0; +http://yandex.com/bots)", 652 | "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)", 653 | "Mozilla/5.0 (compatible; YandexMarket/2.0; +http://yandex.com/bots)", 654 | "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)", 655 | "Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)", 656 | "Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)", 657 | "Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)", 658 | "Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)", 659 | "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)", 660 | "Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)", 661 | "Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)", 662 | "Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)", 663 | "Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)", 664 | "Mozilla/5.0 (compatible; YandexSpravBot/1.0; +http://yandex.com/bots)", 665 | "Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)", 666 | "Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)", 667 | "Mozilla/5.0 (compatible; YandexVerticals/1.0; +http://yandex.com/bots)", 668 | "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)", 669 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)" 670 | ], 671 | "addition_date": "2015/04/14" 672 | } 673 | , 674 | { 675 | "pattern": "purebot", 676 | "addition_date": "2010/01/19", 677 | "instances": [] 678 | } 679 | , 680 | { 681 | "pattern": "Linguee Bot", 682 | "addition_date": "2010/01/26", 683 | "url": "http://www.linguee.com/bot", 684 | "instances": [ 685 | "Linguee Bot (http://www.linguee.com/bot)", 686 | "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)" 687 | ] 688 | } 689 | , 690 | { 691 | "pattern": "CyberPatrol", 692 | "addition_date": "2010/02/11", 693 | "url": "http://www.cyberpatrol.com/cyberpatrolcrawler.asp", 694 | "instances": [ 695 | "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)" 696 | ] 697 | } 698 | , 699 | { 700 | "pattern": "voilabot", 701 | "addition_date": "2010/05/18", 702 | "instances": [ 703 | "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", 704 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)" 705 | ] 706 | } 707 | , 708 | { 709 | "pattern": "Baiduspider", 710 | "addition_date": "2010/07/15", 711 | "url": "http://www.baidu.jp/spider/", 712 | "instances": [ 713 | "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", 714 | "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)" 715 | ] 716 | } 717 | , 718 | { 719 | "pattern": "citeseerxbot", 720 | "addition_date": "2010/07/17", 721 | "instances": [] 722 | } 723 | , 724 | { 725 | "pattern": "spbot", 726 | "addition_date": "2010/07/31", 727 | "url": "http://www.seoprofiler.com/bot", 728 | "instances": [ 729 | "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", 730 | "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", 731 | "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )", 732 | "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )", 733 | "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )", 734 | "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )", 735 | "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )", 736 | "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )", 737 | "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )", 738 | "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )", 739 | "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )", 740 | "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )", 741 | "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )", 742 | "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )", 743 | "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )", 744 | "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )", 745 | "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )", 746 | "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )", 747 | "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )", 748 | "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )", 749 | "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )", 750 | "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )", 751 | "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )", 752 | "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )", 753 | "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )", 754 | "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )", 755 | "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )", 756 | "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )", 757 | "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )", 758 | "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )", 759 | "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )", 760 | "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )", 761 | "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )", 762 | "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )" 763 | ] 764 | } 765 | , 766 | { 767 | "pattern": "twengabot", 768 | "addition_date": "2010/08/03", 769 | "url": "http://www.twenga.com/bot.html", 770 | "instances": [] 771 | } 772 | , 773 | { 774 | "pattern": "postrank", 775 | "addition_date": "2010/08/03", 776 | "url": "http://www.postrank.com", 777 | "instances": [ 778 | "PostRank/2.0 (postrank.com)", 779 | "PostRank/2.0 (postrank.com; 1 subscribers)" 780 | ] 781 | } 782 | , 783 | { 784 | "pattern": "Turnitin", 785 | "addition_date": "2010/09/26", 786 | "url": "http://www.turnitin.com", 787 | "instances": [ 788 | "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", 789 | "Turnitin (https://bit.ly/2UvnfoQ)" 790 | ] 791 | } 792 | , 793 | { 794 | "pattern": "scribdbot", 795 | "addition_date": "2010/09/28", 796 | "url": "http://www.scribd.com", 797 | "instances": [] 798 | } 799 | , 800 | { 801 | "pattern": "page2rss", 802 | "addition_date": "2010/10/07", 803 | "url": "http://www.page2rss.com", 804 | "instances": [ 805 | "Mozilla/5.0 (compatible; Page2RSS/0.7; +http://page2rss.com/)" 806 | ] 807 | } 808 | , 809 | { 810 | "pattern": "sitebot", 811 | "addition_date": "2010/12/15", 812 | "url": "http://www.sitebot.org", 813 | "instances": [ 814 | "Mozilla/5.0 (compatible; Whoiswebsitebot/0.1; +http://www.whoiswebsite.net)" 815 | ] 816 | } 817 | , 818 | { 819 | "pattern": "linkdex", 820 | "addition_date": "2011/01/06", 821 | "url": "http://www.linkdex.com", 822 | "instances": [ 823 | "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", 824 | "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", 825 | "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", 826 | "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", 827 | "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", 828 | "linkdex.com/v2.0", 829 | "linkdexbot/Nutch-1.0-dev (http://www.linkdex.com/; crawl at linkdex dot com)" 830 | ] 831 | } 832 | , 833 | { 834 | "pattern": "Adidxbot", 835 | "url": "http://onlinehelp.microsoft.com/en-us/bing/hh204496.aspx", 836 | "instances": [] 837 | } 838 | , 839 | { 840 | "pattern": "ezooms", 841 | "addition_date": "2011/04/27", 842 | "url": "http://www.phpbb.com/community/viewtopic.php?f=64&t=935605&start=450#p12948289", 843 | "instances": [ 844 | "Mozilla/5.0 (compatible; Ezooms/1.0; ezooms.bot@gmail.com)" 845 | ] 846 | } 847 | , 848 | { 849 | "pattern": "dotbot", 850 | "addition_date": "2011/04/27", 851 | "instances": [ 852 | "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)", 853 | "dotbot" 854 | ] 855 | } 856 | , 857 | { 858 | "pattern": "Mail.RU_Bot", 859 | "addition_date": "2011/04/27", 860 | "instances": [ 861 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", 862 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/", 863 | "Mozilla/5.0 (compatible; Mail.RU_Bot/2.0; +http://go.mail.ru/", 864 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)" 865 | ] 866 | } 867 | , 868 | { 869 | "pattern": "discobot", 870 | "addition_date": "2011/05/03", 871 | "url": "http://discoveryengine.com/discobot.html", 872 | "instances": [ 873 | "Mozilla/5.0 (compatible; discobot/1.0; +http://discoveryengine.com/discobot.html)", 874 | "Mozilla/5.0 (compatible; discobot/2.0; +http://discoveryengine.com/discobot.html)", 875 | "mozilla/5.0 (compatible; discobot/1.1; +http://discoveryengine.com/discobot.html)" 876 | ] 877 | } 878 | , 879 | { 880 | "pattern": "heritrix", 881 | "addition_date": "2011/06/21", 882 | "url": "https://github.com/internetarchive/heritrix3/wiki", 883 | "instances": [ 884 | "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.webarchiv.cz)", 885 | "Mozilla/5.0 (compatible; heritrix/1.12.1b +http://netarkivet.dk/website/info.html)", 886 | "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://rjpower.org)", 887 | "Mozilla/5.0 (compatible; heritrix/1.14.2 +http://www.webarchiv.cz)", 888 | "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://archive.org)", 889 | "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.accelobot.com)", 890 | "Mozilla/5.0 (compatible; heritrix/1.14.3 +http://www.webarchiv.cz)", 891 | "Mozilla/5.0 (compatible; heritrix/1.14.3.r6601 +http://www.buddybuzz.net/yptrino)", 892 | "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://parsijoo.ir)", 893 | "Mozilla/5.0 (compatible; heritrix/1.14.4 +http://www.exif-search.com)", 894 | "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://aihit.com)", 895 | "Mozilla/5.0 (compatible; heritrix/2.0.2 +http://seekda.com)", 896 | "Mozilla/5.0 (compatible; heritrix/3.0.0-SNAPSHOT-20091120.021634 +http://crawler.archive.org)", 897 | "Mozilla/5.0 (compatible; heritrix/3.1.0-RC1 +http://boston.lti.cs.cmu.edu/crawler_12/)", 898 | "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://places.tomtom.com/crawlerinfo)", 899 | "Mozilla/5.0 (compatible; heritrix/3.1.1 +http://www.mixdata.com)", 900 | "Mozilla/5.0 (compatible; heritrix/3.1.1; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", 901 | "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.crim.ca)", 902 | "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.exif-search.com)", 903 | "Mozilla/5.0 (compatible; heritrix/3.2.0 +http://www.mixdata.com)", 904 | "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20160309-0050; UniLeipzigASV +http://corpora.informatik.uni-leipzig.de/crawler_faq.html)", 905 | "Mozilla/5.0 (compatible; sukibot_heritrix/3.1.1 +http://suki.ling.helsinki.fi/eng/webmasters.html)" 906 | ] 907 | } 908 | , 909 | { 910 | "pattern": "findthatfile", 911 | "addition_date": "2011/06/21", 912 | "url": "http://www.findthatfile.com/", 913 | "instances": [] 914 | } 915 | , 916 | { 917 | "pattern": "europarchive.org", 918 | "addition_date": "2011/06/21", 919 | "url": "", 920 | "instances": [ 921 | "Mozilla/5.0 (compatible; MSIE 7.0 +http://www.europarchive.org)" 922 | ] 923 | } 924 | , 925 | { 926 | "pattern": "NerdByNature.Bot", 927 | "addition_date": "2011/07/12", 928 | "url": "http://www.nerdbynature.net/bot", 929 | "instances": [ 930 | "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)" 931 | ] 932 | } 933 | , 934 | { 935 | "pattern": "sistrix crawler", 936 | "addition_date": "2011/08/02", 937 | "instances": [] 938 | } 939 | , 940 | { 941 | "pattern": "Ahrefs(Bot|SiteAudit)", 942 | "addition_date": "2011/08/28", 943 | "instances": [ 944 | "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", 945 | "Mozilla/5.0 (compatible; AhrefsSiteAudit/6.1; +http://ahrefs.com/robot/)", 946 | "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", 947 | "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", 948 | "Mozilla/5.0 (compatible; AhrefsSiteAudit/5.2; +http://ahrefs.com/robot/)", 949 | "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)", 950 | "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)" 951 | ] 952 | } 953 | , 954 | { 955 | "pattern": "fuelbot", 956 | "addition_date": "2018/06/28", 957 | "instances": [ 958 | "fuelbot" 959 | ] 960 | } 961 | , 962 | { 963 | "pattern": "CrunchBot", 964 | "addition_date": "2018/06/28", 965 | "instances": [ 966 | "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)" 967 | ] 968 | } 969 | , 970 | { 971 | "pattern": "IndeedBot", 972 | "addition_date": "2018/06/28", 973 | "instances": [ 974 | "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" 975 | ] 976 | } 977 | , 978 | { 979 | "pattern": "mappydata", 980 | "addition_date": "2018/06/28", 981 | "instances": [ 982 | "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)" 983 | ] 984 | } 985 | , 986 | { 987 | "pattern": "woobot", 988 | "addition_date": "2018/06/28", 989 | "instances": [ 990 | "woobot" 991 | ] 992 | } 993 | , 994 | { 995 | "pattern": "ZoominfoBot", 996 | "addition_date": "2018/06/28", 997 | "instances": [ 998 | "ZoominfoBot (zoominfobot at zoominfo dot com)" 999 | ] 1000 | } 1001 | , 1002 | { 1003 | "pattern": "PrivacyAwareBot", 1004 | "addition_date": "2018/06/28", 1005 | "instances": [ 1006 | "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)" 1007 | ] 1008 | } 1009 | , 1010 | { 1011 | "pattern": "Multiviewbot", 1012 | "addition_date": "2018/06/28", 1013 | "instances": [ 1014 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Multiviewbot" 1015 | ] 1016 | } 1017 | , 1018 | { 1019 | "pattern": "SWIMGBot", 1020 | "addition_date": "2018/06/28", 1021 | "instances": [ 1022 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36 SWIMGBot" 1023 | ] 1024 | } 1025 | , 1026 | { 1027 | "pattern": "Grobbot", 1028 | "addition_date": "2018/06/28", 1029 | "instances": [ 1030 | "Mozilla/5.0 (compatible; Grobbot/2.2; +https://grob.it)" 1031 | ] 1032 | } 1033 | , 1034 | { 1035 | "pattern": "eright", 1036 | "addition_date": "2018/06/28", 1037 | "instances": [ 1038 | "Mozilla/5.0 (compatible; eright/1.0; +bot@eright.com)" 1039 | ] 1040 | } 1041 | , 1042 | { 1043 | "pattern": "Apercite", 1044 | "addition_date": "2018/06/28", 1045 | "instances": [ 1046 | "Mozilla/5.0 (compatible; Apercite; +http://www.apercite.fr/robot/index.html)" 1047 | ] 1048 | } 1049 | , 1050 | { 1051 | "pattern": "semanticbot", 1052 | "addition_date": "2018/06/28", 1053 | "instances": [ 1054 | "semanticbot", 1055 | "semanticbot (info@semanticaudience.com)" 1056 | ] 1057 | } 1058 | , 1059 | { 1060 | "pattern": "Aboundex", 1061 | "addition_date": "2011/09/28", 1062 | "url": "http://www.aboundex.com/crawler/", 1063 | "instances": [ 1064 | "Aboundex/0.2 (http://www.aboundex.com/crawler/)", 1065 | "Aboundex/0.3 (http://www.aboundex.com/crawler/)" 1066 | ] 1067 | } 1068 | , 1069 | { 1070 | "pattern": "domaincrawler", 1071 | "addition_date": "2011/10/21", 1072 | "instances": [ 1073 | "CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)" 1074 | ] 1075 | } 1076 | , 1077 | { 1078 | "pattern": "wbsearchbot", 1079 | "addition_date": "2011/12/21", 1080 | "url": "http://www.warebay.com/bot.html", 1081 | "instances": [] 1082 | } 1083 | , 1084 | { 1085 | "pattern": "summify", 1086 | "addition_date": "2012/01/04", 1087 | "url": "http://summify.com", 1088 | "instances": [ 1089 | "Summify (Summify/1.0.1; +http://summify.com)" 1090 | ] 1091 | } 1092 | , 1093 | { 1094 | "pattern": "CCBot", 1095 | "addition_date": "2012/02/05", 1096 | "url": "http://www.commoncrawl.org/bot.html", 1097 | "instances": [ 1098 | "CCBot/2.0 (http://commoncrawl.org/faq/)", 1099 | "CCBot/2.0 (https://commoncrawl.org/faq/)" 1100 | ] 1101 | } 1102 | , 1103 | { 1104 | "pattern": "edisterbot", 1105 | "addition_date": "2012/02/25", 1106 | "instances": [] 1107 | } 1108 | , 1109 | { 1110 | "pattern": "seznambot", 1111 | "addition_date": "2012/03/14", 1112 | "instances": [ 1113 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", 1114 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", 1115 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)", 1116 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)", 1117 | "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)" 1118 | ] 1119 | } 1120 | , 1121 | { 1122 | "pattern": "ec2linkfinder", 1123 | "addition_date": "2012/03/22", 1124 | "instances": [ 1125 | "ec2linkfinder" 1126 | ] 1127 | } 1128 | , 1129 | { 1130 | "pattern": "gslfbot", 1131 | "addition_date": "2012/04/03", 1132 | "instances": [] 1133 | } 1134 | , 1135 | { 1136 | "pattern": "aiHitBot", 1137 | "addition_date": "2012/04/16", 1138 | "instances": [ 1139 | "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" 1140 | ] 1141 | } 1142 | , 1143 | { 1144 | "pattern": "intelium_bot", 1145 | "addition_date": "2012/05/07", 1146 | "instances": [] 1147 | } 1148 | , 1149 | { 1150 | "pattern": "facebookexternalhit", 1151 | "addition_date": "2012/05/07", 1152 | "instances": [ 1153 | "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", 1154 | "facebookexternalhit/1.1", 1155 | "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" 1156 | ], 1157 | "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler/" 1158 | } 1159 | , 1160 | { 1161 | "pattern": "Yeti", 1162 | "addition_date": "2012/05/07", 1163 | "url": "http://naver.me/bot", 1164 | "instances": [ 1165 | "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)" 1166 | ] 1167 | } 1168 | , 1169 | { 1170 | "pattern": "RetrevoPageAnalyzer", 1171 | "addition_date": "2012/05/07", 1172 | "instances": [ 1173 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)" 1174 | ] 1175 | } 1176 | , 1177 | { 1178 | "pattern": "lb-spider", 1179 | "addition_date": "2012/05/07", 1180 | "instances": [] 1181 | } 1182 | , 1183 | { 1184 | "pattern": "Sogou", 1185 | "addition_date": "2012/05/13", 1186 | "url": "http://www.sogou.com/docs/help/webmasters.htm#07", 1187 | "instances": [ 1188 | "Sogou News Spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", 1189 | "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", 1190 | "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 1191 | ] 1192 | } 1193 | , 1194 | { 1195 | "pattern": "lssbot", 1196 | "addition_date": "2012/05/15", 1197 | "instances": [] 1198 | } 1199 | , 1200 | { 1201 | "pattern": "careerbot", 1202 | "addition_date": "2012/05/23", 1203 | "url": "http://www.career-x.de/bot.html", 1204 | "instances": [] 1205 | } 1206 | , 1207 | { 1208 | "pattern": "wotbox", 1209 | "addition_date": "2012/06/12", 1210 | "url": "http://www.wotbox.com", 1211 | "instances": [ 1212 | "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)", 1213 | "Wotbox/2.01 (+http://www.wotbox.com/bot/)" 1214 | ] 1215 | } 1216 | , 1217 | { 1218 | "pattern": "wocbot", 1219 | "addition_date": "2012/07/25", 1220 | "url": "http://www.wocodi.com/crawler", 1221 | "instances": [] 1222 | } 1223 | , 1224 | { 1225 | "pattern": "ichiro", 1226 | "addition_date": "2012/08/28", 1227 | "url": "http://help.goo.ne.jp/help/article/1142", 1228 | "instances": [ 1229 | "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", 1230 | "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", 1231 | "DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", 1232 | "DoCoMo/2.0 P900i(c100;TB;W24H11)(compatible; ichiro/mobile goo;+http://help.goo.ne.jp/door/crawler.html)", 1233 | "DoCoMo/2.0 P901i(c100;TB;W24H11) (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/door/crawler.html)", 1234 | "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://help.goo.ne.jp/help/article/1142/)", 1235 | "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo; +http://search.goo.ne.jp/option/use/sub4/sub4-1/)", 1236 | "KDDI-CA31 UP.Browser/6.2.0.7.3.129 (GUI) MMP/2.0 (compatible; ichiro/mobile goo;+http://search.goo.ne.jp/option/use/sub4/sub4-1/)", 1237 | "ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html)", 1238 | "ichiro/2.0 (ichiro@nttr.co.jp)", 1239 | "ichiro/3.0 (http://help.goo.ne.jp/door/crawler.html)", 1240 | "ichiro/3.0 (http://help.goo.ne.jp/help/article/1142)", 1241 | "ichiro/3.0 (http://search.goo.ne.jp/option/use/sub4/sub4-1/)", 1242 | "ichiro/4.0 (http://help.goo.ne.jp/door/crawler.html)", 1243 | "ichiro/5.0 (http://help.goo.ne.jp/door/crawler.html)" 1244 | ] 1245 | } 1246 | , 1247 | { 1248 | "pattern": "DuckDuckBot", 1249 | "addition_date": "2012/09/19", 1250 | "url": "http://duckduckgo.com/duckduckbot.html", 1251 | "instances": [ 1252 | "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", 1253 | "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", 1254 | "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)", 1255 | "'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'" 1256 | ] 1257 | } 1258 | , 1259 | { 1260 | "pattern": "lssrocketcrawler", 1261 | "addition_date": "2012/09/24", 1262 | "instances": [] 1263 | } 1264 | , 1265 | { 1266 | "pattern": "drupact", 1267 | "addition_date": "2012/09/27", 1268 | "url": "http://www.arocom.de/drupact", 1269 | "instances": [ 1270 | "drupact/0.7; http://www.arocom.de/drupact" 1271 | ] 1272 | } 1273 | , 1274 | { 1275 | "pattern": "webcompanycrawler", 1276 | "addition_date": "2012/10/03", 1277 | "instances": [] 1278 | } 1279 | , 1280 | { 1281 | "pattern": "acoonbot", 1282 | "addition_date": "2012/10/07", 1283 | "url": "http://www.acoon.de/robot.asp", 1284 | "instances": [] 1285 | } 1286 | , 1287 | { 1288 | "pattern": "openindexspider", 1289 | "addition_date": "2012/10/26", 1290 | "url": "http://www.openindex.io/en/webmasters/spider.html", 1291 | "instances": [] 1292 | } 1293 | , 1294 | { 1295 | "pattern": "gnam gnam spider", 1296 | "addition_date": "2012/10/31", 1297 | "instances": [] 1298 | } 1299 | , 1300 | { 1301 | "pattern": "web-archive-net.com.bot", 1302 | "instances": [] 1303 | } 1304 | , 1305 | { 1306 | "pattern": "backlinkcrawler", 1307 | "addition_date": "2013/01/04", 1308 | "instances": [] 1309 | } 1310 | , 1311 | { 1312 | "pattern": "coccoc", 1313 | "addition_date": "2013/01/04", 1314 | "url": "http://help.coccoc.vn/", 1315 | "instances": [ 1316 | "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", 1317 | "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", 1318 | "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", 1319 | "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", 1320 | "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", 1321 | "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", 1322 | "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", 1323 | "coccoc", 1324 | "coccoc/1.0 ()", 1325 | "coccoc/1.0 (http://help.coccoc.com/)", 1326 | "coccoc/1.0 (http://help.coccoc.vn/)" 1327 | ] 1328 | } 1329 | , 1330 | { 1331 | "pattern": "integromedb", 1332 | "addition_date": "2013/01/10", 1333 | "url": "http://www.integromedb.org/Crawler", 1334 | "instances": [ 1335 | "www.integromedb.org/Crawler" 1336 | ] 1337 | } 1338 | , 1339 | { 1340 | "pattern": "content crawler spider", 1341 | "addition_date": "2013/01/11", 1342 | "instances": [] 1343 | } 1344 | , 1345 | { 1346 | "pattern": "toplistbot", 1347 | "addition_date": "2013/02/05", 1348 | "instances": [] 1349 | } 1350 | , 1351 | { 1352 | "pattern": "it2media-domain-crawler", 1353 | "addition_date": "2013/03/12", 1354 | "instances": [ 1355 | "it2media-domain-crawler/1.0 on crawler-prod.it2media.de", 1356 | "it2media-domain-crawler/2.0" 1357 | ] 1358 | } 1359 | , 1360 | { 1361 | "pattern": "ip-web-crawler.com", 1362 | "addition_date": "2013/03/22", 1363 | "instances": [] 1364 | } 1365 | , 1366 | { 1367 | "pattern": "siteexplorer.info", 1368 | "addition_date": "2013/05/01", 1369 | "instances": [ 1370 | "Mozilla/5.0 (compatible; SiteExplorer/1.0b; +http://siteexplorer.info/)", 1371 | "Mozilla/5.0 (compatible; SiteExplorer/1.1b; +http://siteexplorer.info/Backlink-Checker-Spider/)" 1372 | ] 1373 | } 1374 | , 1375 | { 1376 | "pattern": "elisabot", 1377 | "addition_date": "2013/06/27", 1378 | "instances": [] 1379 | } 1380 | , 1381 | { 1382 | "pattern": "proximic", 1383 | "addition_date": "2013/09/12", 1384 | "url": "http://www.proximic.com/info/spider.php", 1385 | "instances": [ 1386 | "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com)", 1387 | "Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)" 1388 | ] 1389 | } 1390 | , 1391 | { 1392 | "pattern": "changedetection", 1393 | "addition_date": "2013/09/13", 1394 | "url": "http://www.changedetection.com/bot.html", 1395 | "instances": [ 1396 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )" 1397 | ] 1398 | } 1399 | , 1400 | { 1401 | "pattern": "arabot", 1402 | "addition_date": "2013/10/09", 1403 | "instances": [] 1404 | } 1405 | , 1406 | { 1407 | "pattern": "WeSEE:Search", 1408 | "addition_date": "2013/11/18", 1409 | "instances": [ 1410 | "WeSEE:Search", 1411 | "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)" 1412 | ] 1413 | } 1414 | , 1415 | { 1416 | "pattern": "niki-bot", 1417 | "addition_date": "2014/01/01", 1418 | "instances": [] 1419 | } 1420 | , 1421 | { 1422 | "pattern": "CrystalSemanticsBot", 1423 | "addition_date": "2014/02/17", 1424 | "url": "http://www.crystalsemantics.com/user-agent/", 1425 | "instances": [] 1426 | } 1427 | , 1428 | { 1429 | "pattern": "rogerbot", 1430 | "addition_date": "2014/02/28", 1431 | "url": "http://moz.com/help/pro/what-is-rogerbot-", 1432 | "instances": [ 1433 | "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", 1434 | "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+partager@moz.com)", 1435 | "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)", 1436 | "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com", 1437 | "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)", 1438 | "rogerbot/1.0 (http://www.moz.com/dp/rogerbot, rogerbot-crawler@moz.com)", 1439 | "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler+shiny@seomoz.org)", 1440 | "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-crawler@seomoz.org)", 1441 | "rogerbot/1.0 (http://www.seomoz.org/dp/rogerbot, rogerbot-wherecat@moz.com)", 1442 | "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr2-crawler-05@moz.com)", 1443 | "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-11@moz.com)", 1444 | "rogerbot/1.1 (http://moz.com/help/guides/search-overview/crawl-diagnostics#more-help, rogerbot-crawler+pr4-crawler-15@moz.com)", 1445 | "rogerbot/1.2 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+phaser-testing-crawler-01@moz.com)" 1446 | ] 1447 | } 1448 | , 1449 | { 1450 | "pattern": "360Spider", 1451 | "addition_date": "2014/03/14", 1452 | "url": "http://needs-be.blogspot.co.uk/2013/02/how-to-block-spider360.html", 1453 | "instances": [ 1454 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider", 1455 | "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", 1456 | "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider", 1457 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; ) Firefox/1.5.0.11; 360Spider", 1458 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11; 360Spider", 1459 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Firefox/1.5.0.11 360Spider;", 1460 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.8.0.11) Gecko/20070312 Firefox/1.5.0.11; 360Spider", 1461 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider", 1462 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)", 1463 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider" 1464 | ] 1465 | } 1466 | , 1467 | { 1468 | "pattern": "psbot", 1469 | "addition_date": "2014/03/31", 1470 | "url": "http://www.picsearch.com/bot.html", 1471 | "instances": [ 1472 | "psbot-image (+http://www.picsearch.com/bot.html)", 1473 | "psbot-page (+http://www.picsearch.com/bot.html)", 1474 | "psbot/0.1 (+http://www.picsearch.com/bot.html)" 1475 | ] 1476 | } 1477 | , 1478 | { 1479 | "pattern": "InterfaxScanBot", 1480 | "addition_date": "2014/03/31", 1481 | "url": "http://scan-interfax.ru", 1482 | "instances": [] 1483 | } 1484 | , 1485 | { 1486 | "pattern": "CC Metadata Scaper", 1487 | "addition_date": "2014/04/01", 1488 | "url": "http://wiki.creativecommons.org/Metadata_Scraper", 1489 | "instances": [ 1490 | "CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper" 1491 | ] 1492 | } 1493 | , 1494 | { 1495 | "pattern": "g00g1e.net", 1496 | "addition_date": "2014/04/01", 1497 | "url": "http://www.g00g1e.net/", 1498 | "instances": [] 1499 | } 1500 | , 1501 | { 1502 | "pattern": "GrapeshotCrawler", 1503 | "addition_date": "2014/04/01", 1504 | "url": "http://www.grapeshot.co.uk/crawler.php", 1505 | "instances": [ 1506 | "Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)" 1507 | ] 1508 | } 1509 | , 1510 | { 1511 | "pattern": "urlappendbot", 1512 | "addition_date": "2014/05/10", 1513 | "url": "http://www.profound.net/urlappendbot.html", 1514 | "instances": [ 1515 | "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)" 1516 | ] 1517 | } 1518 | , 1519 | { 1520 | "pattern": "brainobot", 1521 | "addition_date": "2014/06/24", 1522 | "instances": [] 1523 | } 1524 | , 1525 | { 1526 | "pattern": "fr-crawler", 1527 | "addition_date": "2014/07/31", 1528 | "instances": [ 1529 | "Mozilla/5.0 (compatible; fr-crawler/1.1)" 1530 | ] 1531 | } 1532 | , 1533 | { 1534 | "pattern": "binlar", 1535 | "addition_date": "2014/09/12", 1536 | "instances": [ 1537 | "binlar_2.6.3 binlar2.6.3@unspecified.mail", 1538 | "binlar_2.6.3 binlar_2.6.3@unspecified.mail", 1539 | "binlar_2.6.3 larbin2.6.3@unspecified.mail", 1540 | "binlar_2.6.3 phanendra_kalapala@McAfee.com", 1541 | "binlar_2.6.3 test@mgmt.mic" 1542 | ] 1543 | } 1544 | , 1545 | { 1546 | "pattern": "SimpleCrawler", 1547 | "addition_date": "2014/09/12", 1548 | "instances": [ 1549 | "SimpleCrawler/0.1" 1550 | ] 1551 | } 1552 | , 1553 | { 1554 | "pattern": "Twitterbot", 1555 | "addition_date": "2014/09/12", 1556 | "url": "https://dev.twitter.com/cards/getting-started", 1557 | "instances": [ 1558 | "Twitterbot/0.1", 1559 | "Twitterbot/1.0" 1560 | ] 1561 | } 1562 | , 1563 | { 1564 | "pattern": "cXensebot", 1565 | "addition_date": "2014/10/05", 1566 | "instances": [ 1567 | "cXensebot/1.1a" 1568 | ], 1569 | "url": "http://www.cxense.com/bot.html" 1570 | } 1571 | , 1572 | { 1573 | "pattern": "smtbot", 1574 | "addition_date": "2014/10/04", 1575 | "instances": [ 1576 | "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", 1577 | "SMTBot (similartech.com/smtbot)", 1578 | "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", 1579 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)", 1580 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; http://www.similartech.com/smtbot)" 1581 | ], 1582 | "url": "http://www.similartech.com/smtbot" 1583 | } 1584 | , 1585 | { 1586 | "pattern": "bnf.fr_bot", 1587 | "addition_date": "2014/11/18", 1588 | "url": "http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html", 1589 | "instances": [ 1590 | "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)", 1591 | "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)" 1592 | ] 1593 | } 1594 | , 1595 | { 1596 | "pattern": "A6-Indexer", 1597 | "addition_date": "2014/12/05", 1598 | "url": "http://www.a6corp.com/a6-web-scraping-policy/", 1599 | "instances": [ 1600 | "A6-Indexer" 1601 | ] 1602 | } 1603 | , 1604 | { 1605 | "pattern": "ADmantX", 1606 | "addition_date": "2014/12/05", 1607 | "url": "http://www.admantx.com", 1608 | "instances": [ 1609 | "ADmantX Platform Semantic Analyzer - ADmantX Inc. - www.admantx.com - support@admantx.com" 1610 | ] 1611 | } 1612 | , 1613 | { 1614 | "pattern": "Facebot", 1615 | "url": "https://developers.facebook.com/docs/sharing/best-practices#crawl", 1616 | "addition_date": "2014/12/30", 1617 | "instances": [ 1618 | "Facebot/1.0" 1619 | ] 1620 | } 1621 | , 1622 | { 1623 | "pattern": "OrangeBot\\/", 1624 | "instances": [ 1625 | "Mozilla/5.0 (compatible; OrangeBot/2.0; support.orangebot@orange.com" 1626 | ], 1627 | "addition_date": "2015/01/12" 1628 | } 1629 | , 1630 | { 1631 | "pattern": "memorybot", 1632 | "url": "http://mignify.com/bot.htm", 1633 | "instances": [ 1634 | "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)" 1635 | ], 1636 | "addition_date": "2015/02/01" 1637 | } 1638 | , 1639 | { 1640 | "pattern": "AdvBot", 1641 | "url": "http://advbot.net/bot.html", 1642 | "instances": [ 1643 | "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)" 1644 | ], 1645 | "addition_date": "2015/02/01" 1646 | } 1647 | , 1648 | { 1649 | "pattern": "MegaIndex", 1650 | "url": "https://www.megaindex.ru/?tab=linkAnalyze", 1651 | "instances": [ 1652 | "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)", 1653 | "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" 1654 | ], 1655 | "addition_date": "2015/03/28" 1656 | } 1657 | , 1658 | { 1659 | "pattern": "SemanticScholarBot", 1660 | "url": "https://www.semanticscholar.org/crawler", 1661 | "instances": [ 1662 | "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)", 1663 | "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)" 1664 | ], 1665 | "addition_date": "2015/03/28" 1666 | } 1667 | , 1668 | { 1669 | "pattern": "ltx71", 1670 | "url": "http://ltx71.com/", 1671 | "instances": [ 1672 | "ltx71 - (http://ltx71.com/)" 1673 | ], 1674 | "addition_date": "2015/04/04" 1675 | } 1676 | , 1677 | { 1678 | "pattern": "nerdybot", 1679 | "url": "http://nerdybot.com/", 1680 | "instances": [ 1681 | "nerdybot" 1682 | ], 1683 | "addition_date": "2015/04/05" 1684 | } 1685 | , 1686 | { 1687 | "pattern": "xovibot", 1688 | "url": "http://www.xovibot.net/", 1689 | "instances": [ 1690 | "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)" 1691 | ], 1692 | "addition_date": "2015/04/05" 1693 | } 1694 | , 1695 | { 1696 | "pattern": "BUbiNG", 1697 | "url": "http://law.di.unimi.it/BUbiNG.html", 1698 | "instances": [ 1699 | "BUbiNG (+http://law.di.unimi.it/BUbiNG.html)" 1700 | ], 1701 | "addition_date": "2015/04/06" 1702 | } 1703 | , 1704 | { 1705 | "pattern": "Qwantify", 1706 | "url": "https://www.qwant.com/", 1707 | "instances": [ 1708 | "Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/*", 1709 | "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", 1710 | "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", 1711 | "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)" 1712 | ], 1713 | "addition_date": "2015/04/06" 1714 | } 1715 | , 1716 | { 1717 | "pattern": "archive.org_bot", 1718 | "url": "http://www.archive.org/details/archive.org_bot", 1719 | "depends_on": ["heritrix"], 1720 | "instances": [ 1721 | "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", 1722 | "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", 1723 | "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)", 1724 | "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)", 1725 | "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", 1726 | "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)" 1727 | ], 1728 | "addition_date": "2015/04/14" 1729 | } 1730 | , 1731 | { 1732 | "pattern": "Applebot", 1733 | "url": "http://www.apple.com/go/applebot", 1734 | "addition_date": "2015/04/15", 1735 | "instances": [ 1736 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", 1737 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", 1738 | "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", 1739 | "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", 1740 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)" 1741 | ] 1742 | } 1743 | , 1744 | { 1745 | "pattern": "TweetmemeBot", 1746 | "url": "http://datasift.com/bot.html", 1747 | "instances": [ 1748 | "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" 1749 | ], 1750 | "addition_date": "2015/04/15" 1751 | } 1752 | , 1753 | { 1754 | "pattern": "crawler4j", 1755 | "url": "https://github.com/yasserg/crawler4j", 1756 | "instances": [ 1757 | "crawler4j (http://code.google.com/p/crawler4j/)", 1758 | "crawler4j (https://github.com/yasserg/crawler4j/)" 1759 | ], 1760 | "addition_date": "2015/05/07" 1761 | } 1762 | , 1763 | { 1764 | "pattern": "findxbot", 1765 | "url": "http://www.findxbot.com", 1766 | "instances": [ 1767 | "Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)" 1768 | ], 1769 | "addition_date": "2015/05/07" 1770 | } 1771 | , 1772 | { 1773 | "pattern": "S[eE][mM]rushBot", 1774 | "url": "http://www.semrush.com/bot.html", 1775 | "instances": [ 1776 | "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)", 1777 | "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", 1778 | "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)", 1779 | "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)", 1780 | "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", 1781 | "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)", 1782 | "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", 1783 | "SEMrushBot" 1784 | ], 1785 | "addition_date": "2015/05/26" 1786 | } 1787 | , 1788 | { 1789 | "pattern": "yoozBot", 1790 | "url": "http://yooz.ir", 1791 | "instances": [ 1792 | "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)" 1793 | ], 1794 | "addition_date": "2015/05/26" 1795 | } 1796 | , 1797 | { 1798 | "pattern": "lipperhey", 1799 | "url": "http://www.lipperhey.com/", 1800 | "instances": [ 1801 | "Mozilla/5.0 (compatible; Lipperhey Link Explorer; http://www.lipperhey.com/)", 1802 | "Mozilla/5.0 (compatible; Lipperhey SEO Service; http://www.lipperhey.com/)", 1803 | "Mozilla/5.0 (compatible; Lipperhey Site Explorer; http://www.lipperhey.com/)", 1804 | "Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)" 1805 | ], 1806 | "addition_date": "2015/08/26" 1807 | } 1808 | , 1809 | { 1810 | "pattern": "Y!J", 1811 | "url": "https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/~/%E3%82%A6%E3%82%A7%E3%83%96%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AB%E3%82%A2%E3%82%AF%E3%82%BB%E3%82%B9%E3%81%99%E3%82%8B%E3%82%B7%E3%82%B9%E3%83%86%E3%83%A0%E3%81%AE%E3%83%A6%E3%83%BC%E3%82%B6%E3%83%BC%E3%82%A8%E3%83%BC%E3%82%B8%E3%82%A7%E3%83%B3%E3%83%88%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6", 1812 | "instances": [ 1813 | "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)", 1814 | "Y!J-BRJ/YATS crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", 1815 | "Y!J-PSC/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", 1816 | "Y!J-BRW/1.0 crawler (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", 1817 | "Mozilla/5.0 (iPhone; Y!J-BRY/YATSH crawler; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)", 1818 | "Mozilla/5.0 (compatible; Y!J SearchMonkey/1.0 (Y!J-AGENT; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html))" 1819 | ], 1820 | "addition_date": "2015/05/26" 1821 | } 1822 | , 1823 | { 1824 | "pattern": "Domain Re-Animator Bot", 1825 | "url": "http://domainreanimator.com", 1826 | "instances": [ 1827 | "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com" 1828 | ], 1829 | "addition_date": "2015/04/14" 1830 | } 1831 | , 1832 | { 1833 | "pattern": "AddThis", 1834 | "url": "https://www.addthis.com", 1835 | "instances": [ 1836 | "AddThis.com robot tech.support@clearspring.com" 1837 | ], 1838 | "addition_date": "2015/06/02" 1839 | } 1840 | , 1841 | { 1842 | "pattern": "Screaming Frog SEO Spider", 1843 | "url": "http://www.screamingfrog.co.uk/seo-spider", 1844 | "instances": [ 1845 | "Screaming Frog SEO Spider/5.1" 1846 | ], 1847 | "addition_date": "2016/01/08" 1848 | } 1849 | , 1850 | { 1851 | "pattern": "MetaURI", 1852 | "url": "http://www.useragentstring.com/MetaURI_id_17683.php", 1853 | "instances": [ 1854 | "MetaURI API/2.0 +metauri.com" 1855 | ], 1856 | "addition_date": "2016/01/02" 1857 | } 1858 | , 1859 | { 1860 | "pattern": "Scrapy", 1861 | "url": "http://scrapy.org/", 1862 | "instances": [ 1863 | "Scrapy/1.0.3 (+http://scrapy.org)" 1864 | ], 1865 | "addition_date": "2016/01/02" 1866 | } 1867 | , 1868 | { 1869 | "pattern": "Livelap[bB]ot", 1870 | "url": "http://site.livelap.com/crawler", 1871 | "instances": [ 1872 | "LivelapBot/0.2 (http://site.livelap.com/crawler)", 1873 | "Livelapbot/0.1" 1874 | ], 1875 | "addition_date": "2016/01/02" 1876 | } 1877 | , 1878 | { 1879 | "pattern": "OpenHoseBot", 1880 | "url": "http://www.openhose.org/bot.html", 1881 | "instances": [ 1882 | "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)" 1883 | ], 1884 | "addition_date": "2016/01/02" 1885 | } 1886 | , 1887 | { 1888 | "pattern": "CapsuleChecker", 1889 | "url": "http://www.capsulink.com/about", 1890 | "instances": [ 1891 | "CapsuleChecker (http://www.capsulink.com/)" 1892 | ], 1893 | "addition_date": "2016/01/02" 1894 | } 1895 | , 1896 | { 1897 | "pattern": "collection@infegy.com", 1898 | "url": "http://infegy.com/", 1899 | "instances": [ 1900 | "Mozilla/5.0 (compatible) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.73 Safari/537.36 collection@infegy.com" 1901 | ], 1902 | "addition_date": "2016/01/03" 1903 | } 1904 | , 1905 | { 1906 | "pattern": "IstellaBot", 1907 | "url": "http://www.tiscali.it/", 1908 | "instances": [ 1909 | "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)" 1910 | ], 1911 | "addition_date": "2016/01/09" 1912 | } 1913 | , 1914 | { 1915 | "pattern": "DeuSu\\/", 1916 | "addition_date": "2016/01/23", 1917 | "url": "https://deusu.de/robot.html", 1918 | "instances": [ 1919 | "Mozilla/5.0 (compatible; DeuSu/0.1.0; +https://deusu.org)", 1920 | "Mozilla/5.0 (compatible; DeuSu/5.0.2; +https://deusu.de/robot.html)" 1921 | ] 1922 | } 1923 | , 1924 | { 1925 | "pattern": "betaBot", 1926 | "addition_date": "2016/01/23", 1927 | "instances": [] 1928 | } 1929 | , 1930 | { 1931 | "pattern": "Cliqzbot\\/", 1932 | "addition_date": "2016/01/23", 1933 | "url": "http://cliqz.com/company/cliqzbot", 1934 | "instances": [ 1935 | "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)", 1936 | "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", 1937 | "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)", 1938 | "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)", 1939 | "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)" 1940 | ] 1941 | } 1942 | , 1943 | { 1944 | "pattern": "MojeekBot\\/", 1945 | "addition_date": "2016/01/23", 1946 | "url": "https://www.mojeek.com/bot.html", 1947 | "instances": [ 1948 | "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", 1949 | "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", 1950 | "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)", 1951 | "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)", 1952 | "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)", 1953 | "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)" 1954 | ] 1955 | } 1956 | , 1957 | { 1958 | "pattern": "netEstate NE Crawler", 1959 | "addition_date": "2016/01/23", 1960 | "url": "http://www.website-datenbank.de/", 1961 | "instances": [ 1962 | "netEstate NE Crawler (+http://www.sengine.info/)", 1963 | "netEstate NE Crawler (+http://www.website-datenbank.de/)" 1964 | ] 1965 | } 1966 | , 1967 | { 1968 | "pattern": "SafeSearch microdata crawler", 1969 | "addition_date": "2016/01/23", 1970 | "url": "https://safesearch.avira.com", 1971 | "instances": [ 1972 | "SafeSearch microdata crawler (https://safesearch.avira.com, safesearch-abuse@avira.com)" 1973 | ] 1974 | } 1975 | , 1976 | { 1977 | "pattern": "Gluten Free Crawler\\/", 1978 | "addition_date": "2016/01/23", 1979 | "url": "http://glutenfreepleasure.com/", 1980 | "instances": [ 1981 | "Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)" 1982 | ] 1983 | } 1984 | , 1985 | { 1986 | "pattern": "Sonic", 1987 | "addition_date": "2016/02/08", 1988 | "url": "http://www.yama.info.waseda.ac.jp/~crawler/info.html", 1989 | "instances": [ 1990 | "Mozilla/5.0 (compatible; RankSonicSiteAuditor/1.0; +https://ranksonic.com/ranksonic_sab.html)", 1991 | "Mozilla/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)", 1992 | "Mozzila/5.0 (compatible; Sonic/1.0; http://www.yama.info.waseda.ac.jp/~crawler/info.html)" 1993 | ] 1994 | } 1995 | , 1996 | { 1997 | "pattern": "Sysomos", 1998 | "addition_date": "2016/02/08", 1999 | "url": "http://www.sysomos.com", 2000 | "instances": [ 2001 | "Mozilla/5.0 (compatible; Sysomos/1.0; +http://www.sysomos.com/; Sysomos)" 2002 | ] 2003 | } 2004 | , 2005 | { 2006 | "pattern": "Trove", 2007 | "addition_date": "2016/02/08", 2008 | "url": "http://www.trove.com", 2009 | "instances": [] 2010 | } 2011 | , 2012 | { 2013 | "pattern": "deadlinkchecker", 2014 | "addition_date": "2016/02/08", 2015 | "url": "http://www.deadlinkchecker.com", 2016 | "instances": [ 2017 | "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36", 2018 | "www.deadlinkchecker.com XMLHTTP/1.0", 2019 | "www.deadlinkchecker.com XMLHTTP/1.0 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" 2020 | ] 2021 | } 2022 | , 2023 | { 2024 | "pattern": "Slack-ImgProxy", 2025 | "addition_date": "2016/04/25", 2026 | "url": "https://api.slack.com/robots", 2027 | "instances": [ 2028 | "Slack-ImgProxy (+https://api.slack.com/robots)", 2029 | "Slack-ImgProxy 0.59 (+https://api.slack.com/robots)", 2030 | "Slack-ImgProxy 0.66 (+https://api.slack.com/robots)", 2031 | "Slack-ImgProxy 1.106 (+https://api.slack.com/robots)", 2032 | "Slack-ImgProxy 1.138 (+https://api.slack.com/robots)", 2033 | "Slack-ImgProxy 149 (+https://api.slack.com/robots)" 2034 | ] 2035 | } 2036 | , 2037 | { 2038 | "pattern": "Embedly", 2039 | "addition_date": "2016/04/25", 2040 | "url": "http://support.embed.ly", 2041 | "instances": [ 2042 | "Embedly +support@embed.ly", 2043 | "Mozilla/5.0 (compatible; Embedly/0.2; +http://support.embed.ly/)", 2044 | "Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)" 2045 | ] 2046 | } 2047 | , 2048 | { 2049 | "pattern": "RankActiveLinkBot", 2050 | "addition_date": "2016/06/20", 2051 | "url": "https://rankactive.com/resources/rankactive-linkbot", 2052 | "instances": [ 2053 | "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)" 2054 | ] 2055 | } 2056 | , 2057 | { 2058 | "pattern": "iskanie", 2059 | "addition_date": "2016/09/02", 2060 | "url": "http://www.iskanie.com", 2061 | "instances": [ 2062 | "iskanie (+http://www.iskanie.com)" 2063 | ] 2064 | } 2065 | , 2066 | { 2067 | "pattern": "SafeDNSBot", 2068 | "addition_date": "2016/09/10", 2069 | "url": "https://www.safedns.com/searchbot", 2070 | "instances": [ 2071 | "SafeDNSBot (https://www.safedns.com/searchbot)" 2072 | ] 2073 | } 2074 | , 2075 | { 2076 | "pattern": "SkypeUriPreview", 2077 | "addition_date": "2016/10/10", 2078 | "instances": [ 2079 | "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" 2080 | ] 2081 | } 2082 | , 2083 | { 2084 | "pattern": "Veoozbot", 2085 | "addition_date": "2016/11/03", 2086 | "url": "http://www.veooz.com/veoozbot.html", 2087 | "instances": [ 2088 | "Mozilla/5.0 (compatible; Veoozbot/1.0; +http://www.veooz.com/veoozbot.html)" 2089 | ] 2090 | } 2091 | , 2092 | { 2093 | "pattern": "Slackbot", 2094 | "addition_date": "2016/11/03", 2095 | "url": "https://api.slack.com/robots", 2096 | "instances": [ 2097 | "Slackbot-LinkExpanding (+https://api.slack.com/robots)", 2098 | "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", 2099 | "Slackbot 1.0 (+https://api.slack.com/robots)" 2100 | ] 2101 | } 2102 | , 2103 | { 2104 | "pattern": "redditbot", 2105 | "addition_date": "2016/11/03", 2106 | "url": "http://www.reddit.com/feedback", 2107 | "instances": [ 2108 | "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)" 2109 | ] 2110 | } 2111 | , 2112 | { 2113 | "pattern": "datagnionbot", 2114 | "addition_date": "2016/11/03", 2115 | "url": "http://www.datagnion.com/bot.html", 2116 | "instances": [ 2117 | "datagnionbot (+http://www.datagnion.com/bot.html)" 2118 | ] 2119 | } 2120 | , 2121 | { 2122 | "pattern": "Google-Adwords-Instant", 2123 | "addition_date": "2016/11/03", 2124 | "url": "http://www.google.com/adsbot.html", 2125 | "instances": [ 2126 | "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" 2127 | ] 2128 | } 2129 | , 2130 | { 2131 | "pattern": "adbeat_bot", 2132 | "addition_date": "2016/11/04", 2133 | "instances": [ 2134 | "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", 2135 | "adbeat_bot" 2136 | ] 2137 | } 2138 | , 2139 | { 2140 | "pattern": "WhatsApp", 2141 | "addition_date": "2016/11/15", 2142 | "url": "https://www.whatsapp.com/", 2143 | "instances": [ 2144 | "WhatsApp", 2145 | "WhatsApp/0.3.4479 N", 2146 | "WhatsApp/0.3.4679 N", 2147 | "WhatsApp/0.3.4941 N", 2148 | "WhatsApp/2.12.15/i", 2149 | "WhatsApp/2.12.16/i", 2150 | "WhatsApp/2.12.17/i", 2151 | "WhatsApp/2.12.449 A", 2152 | "WhatsApp/2.12.453 A", 2153 | "WhatsApp/2.12.510 A", 2154 | "WhatsApp/2.12.540 A", 2155 | "WhatsApp/2.12.548 A", 2156 | "WhatsApp/2.12.555 A", 2157 | "WhatsApp/2.12.556 A", 2158 | "WhatsApp/2.16.1/i", 2159 | "WhatsApp/2.16.13 A", 2160 | "WhatsApp/2.16.2/i", 2161 | "WhatsApp/2.16.42 A", 2162 | "WhatsApp/2.16.57 A", 2163 | "WhatsApp/2.19.92 i", 2164 | "WhatsApp/2.19.175 A", 2165 | "WhatsApp/2.19.244 A", 2166 | "WhatsApp/2.19.258 A", 2167 | "WhatsApp/2.19.308 A", 2168 | "WhatsApp/2.19.330 A" 2169 | ] 2170 | } 2171 | , 2172 | { 2173 | "pattern": "contxbot", 2174 | "addition_date": "2017/02/25", 2175 | "instances": [ 2176 | "Mozilla/5.0 (compatible;contxbot/1.0)" 2177 | ] 2178 | } 2179 | , 2180 | { 2181 | "pattern": "pinterest.com.bot", 2182 | "addition_date": "2017/03/03", 2183 | "instances": [ 2184 | "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", 2185 | "Pinterest/0.2 (+http://www.pinterest.com/bot.html)" 2186 | ], 2187 | "url": "http://www.pinterest.com/bot.html" 2188 | } 2189 | , 2190 | { 2191 | "pattern": "electricmonk", 2192 | "addition_date": "2017/03/04", 2193 | "instances": [ 2194 | "Mozilla/5.0 (compatible; electricmonk/3.2.0 +https://www.duedil.com/our-crawler/)" 2195 | ], 2196 | "url": "https://www.duedil.com/our-crawler/" 2197 | } 2198 | , 2199 | { 2200 | "pattern": "GarlikCrawler", 2201 | "addition_date": "2017/03/18", 2202 | "instances": [ 2203 | "GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)" 2204 | ], 2205 | "url": "http://garlik.com/" 2206 | } 2207 | , 2208 | { 2209 | "pattern": "BingPreview\\/", 2210 | "addition_date": "2017/04/23", 2211 | "url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", 2212 | "instances": [ 2213 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b", 2214 | "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0; BingPreview/1.0b) like Gecko", 2215 | "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; WOW64; Trident/6.0; BingPreview/1.0b)", 2216 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; WOW64; Trident/5.0; BingPreview/1.0b)", 2217 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b" 2218 | ] 2219 | } 2220 | , 2221 | { 2222 | "pattern": "vebidoobot", 2223 | "addition_date": "2017/05/08", 2224 | "instances": [ 2225 | "Mozilla/5.0 (compatible; vebidoobot/1.0; +https://blog.vebidoo.de/vebidoobot/" 2226 | ], 2227 | "url": "https://blog.vebidoo.de/vebidoobot/" 2228 | } 2229 | , 2230 | { 2231 | "pattern": "FemtosearchBot", 2232 | "addition_date": "2017/05/16", 2233 | "instances": [ 2234 | "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)" 2235 | ], 2236 | "url": "http://femtosearch.com" 2237 | } 2238 | , 2239 | { 2240 | "pattern": "Yahoo Link Preview", 2241 | "addition_date": "2017/06/28", 2242 | "instances": [ 2243 | "Mozilla/5.0 (compatible; Yahoo Link Preview; https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html)" 2244 | ], 2245 | "url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html" 2246 | } 2247 | , 2248 | { 2249 | "pattern": "MetaJobBot", 2250 | "addition_date": "2017/08/16", 2251 | "instances": [ 2252 | "Mozilla/5.0 (compatible; MetaJobBot; http://www.metajob.de/crawler)" 2253 | ], 2254 | "url": "http://www.metajob.de/the/crawler" 2255 | } 2256 | , 2257 | { 2258 | "pattern": "DomainStatsBot", 2259 | "addition_date": "2017/08/16", 2260 | "instances": [ 2261 | "DomainStatsBot/1.0 (http://domainstats.io/our-bot)" 2262 | ], 2263 | "url": "http://domainstats.io/our-bot" 2264 | } 2265 | , 2266 | { 2267 | "pattern": "mindUpBot", 2268 | "addition_date": "2017/08/16", 2269 | "instances": [ 2270 | "mindUpBot (datenbutler.de)" 2271 | ], 2272 | "url": "http://www.datenbutler.de/" 2273 | } 2274 | , 2275 | { 2276 | "pattern": "Daum\\/", 2277 | "addition_date": "2017/08/16", 2278 | "instances": [ 2279 | "Mozilla/5.0 (compatible; Daum/4.1; +http://cs.daum.net/faq/15/4118.html?faqId=28966)" 2280 | ], 2281 | "url": "http://cs.daum.net/faq/15/4118.html?faqId=28966" 2282 | } 2283 | , 2284 | { 2285 | "pattern": "Jugendschutzprogramm-Crawler", 2286 | "addition_date": "2017/08/16", 2287 | "instances": [ 2288 | "Jugendschutzprogramm-Crawler; Info: http://www.jugendschutzprogramm.de" 2289 | ], 2290 | "url": "http://www.jugendschutzprogramm.de" 2291 | } 2292 | , 2293 | { 2294 | "pattern": "Xenu Link Sleuth", 2295 | "addition_date": "2017/08/19", 2296 | "instances": [ 2297 | "Xenu Link Sleuth/1.3.8" 2298 | ], 2299 | "url": "http://home.snafu.de/tilman/xenulink.html" 2300 | } 2301 | , 2302 | { 2303 | "pattern": "Pcore-HTTP", 2304 | "addition_date": "2017/08/19", 2305 | "instances": [ 2306 | "Pcore-HTTP/v0.40.3", 2307 | "Pcore-HTTP/v0.44.0" 2308 | ], 2309 | "url": "https://bitbucket.org/softvisio/pcore/overview" 2310 | } 2311 | , 2312 | { 2313 | "pattern": "moatbot", 2314 | "addition_date": "2017/09/16", 2315 | "instances": [ 2316 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 moatbot", 2317 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4 moatbot" 2318 | ], 2319 | "url": "https://moat.com" 2320 | } 2321 | , 2322 | { 2323 | "pattern": "KosmioBot", 2324 | "addition_date": "2017/09/16", 2325 | "instances": [ 2326 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)" 2327 | ], 2328 | "url": "http://kosm.io/bot.html" 2329 | } 2330 | , 2331 | { 2332 | "pattern": "[pP]ingdom", 2333 | "addition_date": "2017/09/16", 2334 | "instances": [ 2335 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/59.0.3071.109 Chrome/59.0.3071.109 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", 2336 | "Mozilla/5.0 (compatible; pingbot/2.0; +http://www.pingdom.com/)", 2337 | "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)", 2338 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)", 2339 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.1 Chrome/78.0.3904.130 Electron/7.3.2 Safari/537.36 PingdomTMS/2020.2", 2340 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.5 Chrome/78.0.3904.130 Electron/7.3.15 Safari/537.36 PingdomTMS/2020.2", 2341 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) browser/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", 2342 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) renderer/2020.2.0 Chrome/78.0.3904.130 Electron/7.1.7 Safari/537.36 PingdomTMS/2020.2", 2343 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; http://www.pingdom.com/)" 2344 | ], 2345 | "url": "http://www.pingdom.com" 2346 | } 2347 | , 2348 | { 2349 | "pattern": "AppInsights", 2350 | "addition_date": "2019/03/09", 2351 | "instances": [ 2352 | "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; AppInsights)" 2353 | ], 2354 | "url": "https://docs.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview" 2355 | } 2356 | , 2357 | { 2358 | "pattern": "PhantomJS", 2359 | "addition_date": "2017/09/18", 2360 | "instances": [ 2361 | "Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1 bl.uk_lddc_renderbot/2.0.0 (+ http://www.bl.uk/aboutus/legaldeposit/websites/websites/faqswebmaster/index.html)" 2362 | ], 2363 | "url": "http://phantomjs.org/" 2364 | } 2365 | , 2366 | { 2367 | "pattern": "Gowikibot", 2368 | "addition_date": "2017/10/26", 2369 | "instances": [ 2370 | "Mozilla/5.0 (compatible; Gowikibot/1.0; +http://www.gowikibot.com)" 2371 | ], 2372 | "url": "http://www.gowikibot.com" 2373 | } 2374 | , 2375 | { 2376 | "pattern": "PiplBot", 2377 | "addition_date": "2017/10/30", 2378 | "instances": [ 2379 | "PiplBot (+http://www.pipl.com/bot/)", 2380 | "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)" 2381 | ], 2382 | "url": "http://www.pipl.com/bot/" 2383 | } 2384 | , 2385 | { 2386 | "pattern": "Discordbot", 2387 | "addition_date": "2017/09/22", 2388 | "url": "https://discordapp.com", 2389 | "instances": [ 2390 | "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)" 2391 | ] 2392 | } 2393 | , 2394 | { 2395 | "pattern": "TelegramBot", 2396 | "addition_date": "2017/10/01", 2397 | "instances": [ 2398 | "TelegramBot (like TwitterBot)" 2399 | ] 2400 | } 2401 | , 2402 | { 2403 | "pattern": "Jetslide", 2404 | "addition_date": "2017/09/27", 2405 | "url": "http://jetsli.de/crawler", 2406 | "instances": [ 2407 | "Mozilla/5.0 (compatible; Jetslide; +http://jetsli.de/crawler)" 2408 | ] 2409 | } 2410 | , 2411 | { 2412 | "pattern": "newsharecounts", 2413 | "addition_date": "2017/09/30", 2414 | "url": "http://newsharecounts.com/crawler", 2415 | "instances": [ 2416 | "Mozilla/5.0 (compatible; NewShareCounts.com/1.0; +http://newsharecounts.com/crawler)" 2417 | ] 2418 | } 2419 | , 2420 | { 2421 | "pattern": "James BOT", 2422 | "addition_date": "2017/10/12", 2423 | "url": "http://cognitiveseo.com/bot.html", 2424 | "instances": [ 2425 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html" 2426 | ] 2427 | } 2428 | , 2429 | { 2430 | "pattern": "Bark[rR]owler", 2431 | "addition_date": "2017/10/09", 2432 | "url": "http://www.exensa.com/crawl", 2433 | "instances": [ 2434 | "Barkrowler/0.5.1 (experimenting / debugging - sorry for your logs ) http://www.exensa.com/crawl - admin@exensa.com -- based on BuBiNG", 2435 | "Barkrowler/0.7 (+http://www.exensa.com/crawl)", 2436 | "BarkRowler/0.7 (+http://www.exensa.com/crawling)", 2437 | "Barkrowler/0.9 (+http://www.exensa.com/crawl)" 2438 | ] 2439 | } 2440 | , 2441 | { 2442 | "pattern": "TinEye", 2443 | "addition_date": "2017/10/14", 2444 | "url": "http://www.tineye.com/crawler.html", 2445 | "instances": [ 2446 | "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", 2447 | "TinEye/1.1 (http://tineye.com/crawler.html)" 2448 | ] 2449 | } 2450 | , 2451 | { 2452 | "pattern": "SocialRankIOBot", 2453 | "addition_date": "2017/10/19", 2454 | "url": "http://socialrank.io/about", 2455 | "instances": [ 2456 | "SocialRankIOBot; http://socialrank.io/about" 2457 | ] 2458 | } 2459 | , 2460 | { 2461 | "pattern": "trendictionbot", 2462 | "addition_date": "2017/10/30", 2463 | "url": "http://www.trendiction.de/bot", 2464 | "instances": [ 2465 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11", 2466 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0" 2467 | ] 2468 | } 2469 | , 2470 | { 2471 | "pattern": "Ocarinabot", 2472 | "addition_date": "2017/09/27", 2473 | "instances": [ 2474 | "Ocarinabot" 2475 | ] 2476 | } 2477 | , 2478 | { 2479 | "pattern": "epicbot", 2480 | "addition_date": "2017/10/31", 2481 | "url": "http://www.epictions.com/epicbot", 2482 | "instances": [ 2483 | "Mozilla/5.0 (compatible; epicbot; +http://www.epictions.com/epicbot)" 2484 | ] 2485 | } 2486 | , 2487 | { 2488 | "pattern": "Primalbot", 2489 | "addition_date": "2017/09/27", 2490 | "url": "https://www.primal.com", 2491 | "instances": [ 2492 | "Mozilla/5.0 (compatible; Primalbot; +https://www.primal.com;)" 2493 | ] 2494 | } 2495 | , 2496 | { 2497 | "pattern": "DuckDuckGo-Favicons-Bot", 2498 | "addition_date": "2017/10/06", 2499 | "url": "http://duckduckgo.com", 2500 | "instances": [ 2501 | "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" 2502 | ] 2503 | } 2504 | , 2505 | { 2506 | "pattern": "GnowitNewsbot", 2507 | "addition_date": "2017/10/30", 2508 | "url": "http://www.gnowit.com", 2509 | "instances": [ 2510 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com" 2511 | ] 2512 | } 2513 | , 2514 | { 2515 | "pattern": "Leikibot", 2516 | "addition_date": "2017/09/24", 2517 | "url": "http://www.leiki.com", 2518 | "instances": [ 2519 | "Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0; +http://www.leiki.com)" 2520 | ] 2521 | } 2522 | , 2523 | { 2524 | "pattern": "LinkArchiver", 2525 | "addition_date": "2017/09/24", 2526 | "instances": [ 2527 | "@LinkArchiver twitter bot" 2528 | ] 2529 | } 2530 | , 2531 | { 2532 | "pattern": "YaK\\/", 2533 | "addition_date": "2017/09/25", 2534 | "url": "http://linkfluence.com", 2535 | "instances": [ 2536 | "Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)" 2537 | ] 2538 | } 2539 | , 2540 | { 2541 | "pattern": "PaperLiBot", 2542 | "addition_date": "2017/09/25", 2543 | "url": "http://support.paper.li/entries/20023257-what-is-paper-li", 2544 | "instances": [ 2545 | "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)", 2546 | "Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/entries/20023257-what-is-paper-li)" 2547 | 2548 | ] 2549 | } 2550 | , 2551 | { 2552 | "pattern": "Digg Deeper", 2553 | "addition_date": "2017/09/26", 2554 | "url": "http://digg.com/about", 2555 | "instances": [ 2556 | "Digg Deeper/v1 (http://digg.com/about)" 2557 | ] 2558 | } 2559 | , 2560 | { 2561 | "pattern": "dcrawl", 2562 | "addition_date": "2017/09/22", 2563 | "instances": [ 2564 | "dcrawl/1.0" 2565 | ] 2566 | } 2567 | , 2568 | { 2569 | "pattern": "Snacktory", 2570 | "addition_date": "2017/09/23", 2571 | "url": "https://github.com/karussell/snacktory", 2572 | "instances": [ 2573 | "Mozilla/5.0 (compatible; Snacktory; +https://github.com/karussell/snacktory)" 2574 | ] 2575 | } 2576 | , 2577 | { 2578 | "pattern": "AndersPinkBot", 2579 | "addition_date": "2017/09/24", 2580 | "url": "http://anderspink.com/bot.html", 2581 | "instances": [ 2582 | "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)" 2583 | ] 2584 | } 2585 | , 2586 | { 2587 | "pattern": "Fyrebot", 2588 | "addition_date": "2017/09/22", 2589 | "instances": [ 2590 | "Fyrebot/1.0" 2591 | ] 2592 | } 2593 | , 2594 | { 2595 | "pattern": "EveryoneSocialBot", 2596 | "addition_date": "2017/09/22", 2597 | "url": "http://everyonesocial.com", 2598 | "instances": [ 2599 | "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)" 2600 | ] 2601 | } 2602 | , 2603 | { 2604 | "pattern": "Mediatoolkitbot", 2605 | "addition_date": "2017/10/06", 2606 | "url": "http://mediatoolkit.com", 2607 | "instances": [ 2608 | "Mediatoolkitbot (complaints@mediatoolkit.com)" 2609 | ] 2610 | } 2611 | , 2612 | { 2613 | "pattern": "Luminator-robots", 2614 | "addition_date": "2017/09/22", 2615 | "instances": [ 2616 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.13 Luminator-robots/2.0" 2617 | ] 2618 | } 2619 | , 2620 | { 2621 | "pattern": "ExtLinksBot", 2622 | "addition_date": "2017/11/02", 2623 | "url": "https://extlinks.com/Bot.html", 2624 | "instances": [ 2625 | "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)" 2626 | ] 2627 | } 2628 | , 2629 | { 2630 | "pattern": "SurveyBot", 2631 | "addition_date": "2017/11/02", 2632 | "instances": [ 2633 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)" 2634 | ] 2635 | } 2636 | , 2637 | { 2638 | "pattern": "NING\\/", 2639 | "addition_date": "2017/11/02", 2640 | "instances": [ 2641 | "NING/1.0" 2642 | ] 2643 | } 2644 | , 2645 | { 2646 | "pattern": "okhttp", 2647 | "addition_date": "2017/11/02", 2648 | "instances": [ 2649 | "okhttp/2.5.0", 2650 | "okhttp/2.7.5", 2651 | "okhttp/3.2.0", 2652 | "okhttp/3.5.0", 2653 | "okhttp/4.1.0" 2654 | ] 2655 | } 2656 | , 2657 | { 2658 | "pattern": "Nuzzel", 2659 | "addition_date": "2017/11/02", 2660 | "instances": [ 2661 | "Nuzzel" 2662 | ] 2663 | } 2664 | , 2665 | { 2666 | "pattern": "omgili", 2667 | "addition_date": "2017/11/02", 2668 | "url": "http://omgili.com", 2669 | "instances": [ 2670 | "omgili/0.5 +http://omgili.com" 2671 | ] 2672 | } 2673 | , 2674 | { 2675 | "pattern": "PocketParser", 2676 | "addition_date": "2017/11/02", 2677 | "url": "https://getpocket.com/pocketparser_ua", 2678 | "instances": [ 2679 | "PocketParser/2.0 (+https://getpocket.com/pocketparser_ua)" 2680 | ] 2681 | } 2682 | , 2683 | { 2684 | "pattern": "YisouSpider", 2685 | "addition_date": "2017/11/02", 2686 | "instances": [ 2687 | "YisouSpider", 2688 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36" 2689 | ] 2690 | } 2691 | , 2692 | { 2693 | "pattern": "um-LN", 2694 | "addition_date": "2017/11/02", 2695 | "instances": [ 2696 | "Mozilla/5.0 (compatible; um-LN/1.0; mailto: techinfo@ubermetrics-technologies.com)" 2697 | ] 2698 | } 2699 | , 2700 | { 2701 | "pattern": "ToutiaoSpider", 2702 | "addition_date": "2017/11/02", 2703 | "url": "http://web.toutiao.com/media_cooperation/", 2704 | "instances": [ 2705 | "Mozilla/5.0 (compatible; ToutiaoSpider/1.0; http://web.toutiao.com/media_cooperation/;)" 2706 | ] 2707 | } 2708 | , 2709 | { 2710 | "pattern": "MuckRack", 2711 | "addition_date": "2017/11/02", 2712 | "url": "http://muckrack.com", 2713 | "instances": [ 2714 | "Mozilla/5.0 (compatible; MuckRack/1.0; +http://muckrack.com)" 2715 | ] 2716 | } 2717 | , 2718 | { 2719 | "pattern": "Jamie's Spider", 2720 | "addition_date": "2017/11/02", 2721 | "url": "http://jamiembrown.com/", 2722 | "instances": [ 2723 | "Jamie's Spider (http://jamiembrown.com/)" 2724 | ] 2725 | } 2726 | , 2727 | { 2728 | "pattern": "AHC\\/", 2729 | "addition_date": "2017/11/02", 2730 | "instances": [ 2731 | "AHC/2.0" 2732 | ] 2733 | } 2734 | , 2735 | { 2736 | "pattern": "NetcraftSurveyAgent", 2737 | "addition_date": "2017/11/02", 2738 | "instances": [ 2739 | "Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)" 2740 | ] 2741 | } 2742 | , 2743 | { 2744 | "pattern": "Laserlikebot", 2745 | "addition_date": "2017/11/02", 2746 | "instances": [ 2747 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)" 2748 | ] 2749 | } 2750 | , 2751 | { 2752 | "pattern": "^Apache-HttpClient", 2753 | "addition_date": "2017/11/02", 2754 | "instances": [ 2755 | "Apache-HttpClient/4.2.3 (java 1.5)", 2756 | "Apache-HttpClient/4.2.5 (java 1.5)", 2757 | "Apache-HttpClient/4.3.1 (java 1.5)", 2758 | "Apache-HttpClient/4.3.3 (java 1.5)", 2759 | "Apache-HttpClient/4.3.5 (java 1.5)", 2760 | "Apache-HttpClient/4.4.1 (Java/1.8.0_65)", 2761 | "Apache-HttpClient/4.5.2 (Java/1.8.0_65)", 2762 | "Apache-HttpClient/4.5.2 (Java/1.8.0_151)", 2763 | "Apache-HttpClient/4.5.2 (Java/1.8.0_161)", 2764 | "Apache-HttpClient/4.5.2 (Java/1.8.0_181)", 2765 | "Apache-HttpClient/4.5.3 (Java/1.8.0_121)", 2766 | "Apache-HttpClient/4.5.3-SNAPSHOT (Java/1.8.0_152)", 2767 | "Apache-HttpClient/4.5.7 (Java/11.0.3)", 2768 | "Apache-HttpClient/4.5.10 (Java/1.8.0_201)" 2769 | ] 2770 | } 2771 | , 2772 | { 2773 | "pattern": "AppEngine-Google", 2774 | "addition_date": "2017/11/02", 2775 | "instances": [ 2776 | "AppEngine-Google; (+http://code.google.com/appengine; appid: example)", 2777 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-nikon3)" 2778 | ] 2779 | } 2780 | , 2781 | { 2782 | "pattern": "Jetty", 2783 | "addition_date": "2017/11/02", 2784 | "instances": [ 2785 | "Jetty/9.3.z-SNAPSHOT" 2786 | ] 2787 | } 2788 | , 2789 | { 2790 | "pattern": "Upflow", 2791 | "addition_date": "2017/11/02", 2792 | "instances": [ 2793 | "Upflow/1.0" 2794 | ] 2795 | } 2796 | , 2797 | { 2798 | "pattern": "Thinklab", 2799 | "addition_date": "2017/11/02", 2800 | "url": "thinklab.com", 2801 | "instances": [ 2802 | "Thinklab (thinklab.com)" 2803 | ] 2804 | } 2805 | , 2806 | { 2807 | "pattern": "Traackr.com", 2808 | "addition_date": "2017/11/02", 2809 | "url": "Traackr.com", 2810 | "instances": [ 2811 | "Traackr.com" 2812 | ] 2813 | } 2814 | , 2815 | { 2816 | "pattern": "Twurly", 2817 | "addition_date": "2017/11/02", 2818 | "url": "http://twurly.org", 2819 | "instances": [ 2820 | "Ruby, Twurly v1.1 (http://twurly.org)" 2821 | ] 2822 | } 2823 | , 2824 | { 2825 | "pattern": "Mastodon", 2826 | "addition_date": "2017/11/02", 2827 | "instances": [ 2828 | "http.rb/2.2.2 (Mastodon/1.5.1; +https://example-masto-instance.org/)" 2829 | ] 2830 | } 2831 | , 2832 | { 2833 | "pattern": "http_get", 2834 | "addition_date": "2017/11/02", 2835 | "instances": [ 2836 | "http_get" 2837 | ] 2838 | } 2839 | , 2840 | { 2841 | "pattern": "DnyzBot", 2842 | "addition_date": "2017/11/20", 2843 | "instances": [ 2844 | "Mozilla/5.0 (compatible; DnyzBot/1.0)" 2845 | ] 2846 | } 2847 | , 2848 | { 2849 | "pattern": "botify", 2850 | "addition_date": "2018/02/01", 2851 | "instances": [ 2852 | "Mozilla/5.0 (compatible; botify; http://botify.com)" 2853 | ] 2854 | } 2855 | , 2856 | { 2857 | "pattern": "007ac9 Crawler", 2858 | "addition_date": "2018/02/09", 2859 | "instances": [ 2860 | "Mozilla/5.0 (compatible; 007ac9 Crawler; http://crawler.007ac9.net/)" 2861 | ] 2862 | } 2863 | , 2864 | { 2865 | "pattern": "BehloolBot", 2866 | "addition_date": "2018/02/09", 2867 | "instances": [ 2868 | "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)" 2869 | ] 2870 | } 2871 | , 2872 | { 2873 | "pattern": "BrandVerity", 2874 | "addition_date": "2018/02/27", 2875 | "instances": [ 2876 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/55.0 BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)", 2877 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11A465 Twitter for iPhone BrandVerity/1.0 (http://www.brandverity.com/why-is-brandverity-visiting-me)" 2878 | ], 2879 | "url": "http://www.brandverity.com/why-is-brandverity-visiting-me" 2880 | } 2881 | , 2882 | { 2883 | "pattern": "check_http", 2884 | "addition_date": "2018/02/09", 2885 | "instances": [ 2886 | "check_http/v2.2.1 (nagios-plugins 2.2.1)" 2887 | ] 2888 | } 2889 | , 2890 | { 2891 | "pattern": "BDCbot", 2892 | "addition_date": "2018/02/09", 2893 | "instances": [ 2894 | "Mozilla/5.0 (Windows NT 6.1; compatible; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36", 2895 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; BDCbot/1.0; +http://bigweb.bigdatacorp.com.br/faq.aspx) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" 2896 | ] 2897 | } 2898 | , 2899 | { 2900 | "pattern": "ZumBot", 2901 | "addition_date": "2018/02/09", 2902 | "instances": [ 2903 | "Mozilla/5.0 (compatible; ZumBot/1.0; http://help.zum.com/inquiry)" 2904 | ] 2905 | } 2906 | , 2907 | { 2908 | "pattern": "EZID", 2909 | "addition_date": "2018/02/09", 2910 | "instances": [ 2911 | "EZID (EZID link checker; https://ezid.cdlib.org/)" 2912 | ] 2913 | } 2914 | , 2915 | { 2916 | "pattern": "ICC-Crawler", 2917 | "addition_date": "2018/02/28", 2918 | "instances": [ 2919 | "ICC-Crawler/2.0 (Mozilla-compatible; ; http://ucri.nict.go.jp/en/icccrawler.html)" 2920 | ], 2921 | "url": "http://ucri.nict.go.jp/en/icccrawler.html" 2922 | } 2923 | , 2924 | { 2925 | "pattern": "ArchiveBot", 2926 | "addition_date": "2018/02/28", 2927 | "instances": [ 2928 | "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)" 2929 | ], 2930 | "url": "https://github.com/ArchiveTeam/ArchiveBot" 2931 | } 2932 | , 2933 | { 2934 | "pattern": "^LCC ", 2935 | "addition_date": "2018/02/28", 2936 | "instances": [ 2937 | "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" 2938 | ], 2939 | "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html" 2940 | } 2941 | , 2942 | { 2943 | "pattern": "filterdb.iss.net\\/crawler", 2944 | "addition_date": "2018/03/16", 2945 | "instances": [ 2946 | "Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)" 2947 | ], 2948 | "url": "http://filterdb.iss.net/crawler/" 2949 | } 2950 | , 2951 | { 2952 | "pattern": "BLP_bbot", 2953 | "addition_date": "2018/03/27", 2954 | "instances": [ 2955 | "BLP_bbot/0.1" 2956 | ] 2957 | } 2958 | , 2959 | { 2960 | "pattern": "BomboraBot", 2961 | "addition_date": "2018/03/27", 2962 | "instances": [ 2963 | "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)" 2964 | ], 2965 | "url": "http://www.bombora.com/bot" 2966 | } 2967 | , 2968 | { 2969 | "pattern": "Buck\\/", 2970 | "addition_date": "2018/03/27", 2971 | "instances": [ 2972 | "Buck/2.2; (+https://app.hypefactors.com/media-monitoring/about.html)" 2973 | ], 2974 | "url": "https://app.hypefactors.com/media-monitoring/about.html" 2975 | } 2976 | , 2977 | { 2978 | "pattern": "Companybook-Crawler", 2979 | "addition_date": "2018/03/27", 2980 | "instances": [ 2981 | "Companybook-Crawler (+https://www.companybooknetworking.com/)" 2982 | ], 2983 | "url": "https://www.companybooknetworking.com/" 2984 | } 2985 | , 2986 | { 2987 | "pattern": "Genieo", 2988 | "addition_date": "2018/03/27", 2989 | "instances": [ 2990 | "Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html)" 2991 | ], 2992 | "url": "http://www.genieo.com/webfilter.html" 2993 | } 2994 | , 2995 | { 2996 | "pattern": "magpie-crawler", 2997 | "addition_date": "2018/03/27", 2998 | "instances": [ 2999 | "magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)" 3000 | ], 3001 | "url": "http://www.brandwatch.net" 3002 | } 3003 | , 3004 | { 3005 | "pattern": "MeltwaterNews", 3006 | "addition_date": "2018/03/27", 3007 | "instances": [ 3008 | "MeltwaterNews www.meltwater.com" 3009 | ], 3010 | "url": "http://www.meltwater.com" 3011 | } 3012 | , 3013 | { 3014 | "pattern": "Moreover", 3015 | "addition_date": "2018/03/27", 3016 | "instances": [ 3017 | "Mozilla/5.0 Moreover/5.1 (+http://www.moreover.com)" 3018 | ], 3019 | "url": "http://www.moreover.com" 3020 | } 3021 | , 3022 | { 3023 | "pattern": "newspaper\\/", 3024 | "addition_date": "2018/03/27", 3025 | "instances": [ 3026 | "newspaper/0.1.0.7", 3027 | "newspaper/0.2.5", 3028 | "newspaper/0.2.6", 3029 | "newspaper/0.2.8" 3030 | ] 3031 | } 3032 | , 3033 | { 3034 | "pattern": "ScoutJet", 3035 | "addition_date": "2018/03/27", 3036 | "instances": [ 3037 | "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)" 3038 | ], 3039 | "url": "http://www.scoutjet.com/" 3040 | } 3041 | , 3042 | { 3043 | "pattern": "(^| )sentry\\/", 3044 | "addition_date": "2018/03/27", 3045 | "instances": [ 3046 | "sentry/8.22.0 (https://sentry.io)" 3047 | ], 3048 | "url": "https://sentry.io" 3049 | } 3050 | , 3051 | { 3052 | "pattern": "StorygizeBot", 3053 | "addition_date": "2018/03/27", 3054 | "instances": [ 3055 | "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)" 3056 | ], 3057 | "url": "http://www.storygize.com" 3058 | } 3059 | , 3060 | { 3061 | "pattern": "UptimeRobot", 3062 | "addition_date": "2018/03/27", 3063 | "instances": [ 3064 | "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" 3065 | ], 3066 | "url": "http://www.uptimerobot.com/" 3067 | } 3068 | , 3069 | { 3070 | "pattern": "OutclicksBot", 3071 | "addition_date": "2018/04/21", 3072 | "instances": [ 3073 | "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", 3074 | "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", 3075 | "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J", 3076 | "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h" 3077 | ], 3078 | "url": "https://www.outclicks.net" 3079 | } 3080 | , 3081 | { 3082 | "pattern": "seoscanners", 3083 | "addition_date": "2018/05/27", 3084 | "instances": [ 3085 | "Mozilla/5.0 (compatible; seoscanners.net/1; +spider@seoscanners.net)" 3086 | ], 3087 | "url": "http://www.seoscanners.net/" 3088 | } 3089 | , 3090 | { 3091 | "pattern": "Hatena", 3092 | "addition_date": "2018/05/29", 3093 | "instances": [ 3094 | "Hatena Antenna/0.3", 3095 | "Hatena::Russia::Crawler/0.01", 3096 | "Hatena-Favicon/2 (http://www.hatena.ne.jp/faq/)", 3097 | "Hatena::Scissors/0.01", 3098 | "HatenaBookmark/4.0 (Hatena::Bookmark; Analyzer)", 3099 | "Hatena::Fetcher/0.01 (master) Furl/3.13" 3100 | ] 3101 | } 3102 | , 3103 | { 3104 | "pattern": "Google Web Preview", 3105 | "addition_date": "2018/05/31", 3106 | "instances": [ 3107 | "Mozilla/5.0 (Linux; U; Android 2.3.4; generic) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Version/4.0 Mobile Safari/537.36", 3108 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview) Chrome/27.0.1453 Safari/537.36" 3109 | ] 3110 | } 3111 | , 3112 | { 3113 | "pattern": "MauiBot", 3114 | "addition_date": "2018/06/06", 3115 | "instances": [ 3116 | "MauiBot (crawler.feedback+wc@gmail.com)" 3117 | ] 3118 | } 3119 | , 3120 | { 3121 | "pattern": "AlphaBot", 3122 | "addition_date": "2018/05/27", 3123 | "instances": [ 3124 | "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)" 3125 | ], 3126 | "url": "http://alphaseobot.com/bot.html" 3127 | } 3128 | , 3129 | { 3130 | "pattern": "SBL-BOT", 3131 | "addition_date": "2018/06/06", 3132 | "instances": [ 3133 | "SBL-BOT (http://sbl.net)" 3134 | ], 3135 | "url": "http://sbl.net", 3136 | "description" : "Bot of SoftByte BlackWidow" 3137 | } 3138 | , 3139 | { 3140 | "pattern": "IAS crawler", 3141 | "addition_date": "2018/06/06", 3142 | "instances": [ 3143 | "IAS crawler (ias_crawler; http://integralads.com/site-indexing-policy/)" 3144 | ], 3145 | "url": "http://integralads.com/site-indexing-policy/", 3146 | "description" : "Bot of Integral Ad Science, Inc." 3147 | } 3148 | , 3149 | { 3150 | "pattern": "adscanner", 3151 | "addition_date": "2018/06/24", 3152 | "instances": [ 3153 | "Mozilla/5.0 (compatible; adscanner/)" 3154 | ] 3155 | } 3156 | , 3157 | { 3158 | "pattern": "Netvibes", 3159 | "addition_date": "2018/06/24", 3160 | "instances": [ 3161 | "Netvibes (crawler/bot; http://www.netvibes.com", 3162 | "Netvibes (crawler; http://www.netvibes.com)" 3163 | ], 3164 | "url": "http://www.netvibes.com" 3165 | } 3166 | , 3167 | { 3168 | "pattern": "acapbot", 3169 | "addition_date": "2018/06/27", 3170 | "instances": [ 3171 | "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)", 3172 | "Mozilla/5.0 (compatible;acapbot/0.1.;treat like Googlebot)" 3173 | ] 3174 | } 3175 | , 3176 | { 3177 | "pattern": "Baidu-YunGuanCe", 3178 | "addition_date": "2018/06/27", 3179 | "instances": [ 3180 | "Baidu-YunGuanCe-Bot(ce.baidu.com)", 3181 | "Baidu-YunGuanCe-SLABot(ce.baidu.com)", 3182 | "Baidu-YunGuanCe-ScanBot(ce.baidu.com)", 3183 | "Baidu-YunGuanCe-PerfBot(ce.baidu.com)", 3184 | "Baidu-YunGuanCe-VSBot(ce.baidu.com)" 3185 | ], 3186 | "url": "https://ce.baidu.com/topic/topic20150908", 3187 | "description": "Baidu Cloud Watch" 3188 | } 3189 | , 3190 | { 3191 | "pattern": "bitlybot", 3192 | "addition_date": "2018/06/27", 3193 | "instances": [ 3194 | "bitlybot/3.0 (+http://bit.ly/)", 3195 | "bitlybot/2.0", 3196 | "bitlybot" 3197 | ], 3198 | "url": "http://bit.ly/" 3199 | } 3200 | , 3201 | { 3202 | "pattern": "blogmuraBot", 3203 | "addition_date": "2018/06/27", 3204 | "instances": [ 3205 | "blogmuraBot (+http://www.blogmura.com)" 3206 | ], 3207 | "url": "http://www.blogmura.com", 3208 | "description": "A blog ranking site which links to blogs on just about every theme possible." 3209 | } 3210 | , 3211 | { 3212 | "pattern": "Bot.AraTurka.com", 3213 | "addition_date": "2018/06/27", 3214 | "instances": [ 3215 | "Bot.AraTurka.com/0.0.1" 3216 | ], 3217 | "url": "http://www.araturka.com" 3218 | } 3219 | , 3220 | { 3221 | "pattern": "bot-pge.chlooe.com", 3222 | "addition_date": "2018/06/27", 3223 | "instances": [ 3224 | "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)" 3225 | ] 3226 | } 3227 | , 3228 | { 3229 | "pattern": "BoxcarBot", 3230 | "addition_date": "2018/06/27", 3231 | "instances": [ 3232 | "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)" 3233 | ], 3234 | "url": "https://boxcar.io/" 3235 | } 3236 | , 3237 | { 3238 | "pattern": "BTWebClient", 3239 | "addition_date": "2018/06/27", 3240 | "instances": [ 3241 | "BTWebClient/180B(9704)" 3242 | ], 3243 | "url": "http://www.utorrent.com/", 3244 | "description": "µTorrent BitTorrent Client" 3245 | } 3246 | , 3247 | { 3248 | "pattern": "ContextAd Bot", 3249 | "addition_date": "2018/06/27", 3250 | "instances": [ 3251 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", 3252 | "ContextAd Bot 1.0" 3253 | ] 3254 | } 3255 | , 3256 | { 3257 | "pattern": "Digincore bot", 3258 | "addition_date": "2018/06/27", 3259 | "instances": [ 3260 | "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)" 3261 | ], 3262 | "url": "http://www.digincore.com/crawler.html" 3263 | } 3264 | , 3265 | { 3266 | "pattern": "Disqus", 3267 | "addition_date": "2018/06/27", 3268 | "instances": [ 3269 | "Disqus/1.0" 3270 | ], 3271 | "url": "https://disqus.com/", 3272 | "description": "validate and quality check pages." 3273 | } 3274 | , 3275 | { 3276 | "pattern": "Feedly", 3277 | "addition_date": "2018/06/27", 3278 | "instances": [ 3279 | "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)", 3280 | "FeedlyBot/1.0 (http://feedly.com)" 3281 | ], 3282 | "url": "https://www.feedly.com/fetcher.html", 3283 | "description": "Feedly Fetcher is how Feedly grabs RSS or Atom feeds when users choose to add them to their Feedly or any of the other applications built on top of the feedly cloud." 3284 | } 3285 | , 3286 | { 3287 | "pattern": "Fetch\\/", 3288 | "addition_date": "2018/06/27", 3289 | "instances": [ 3290 | "Fetch/2.0a (CMS Detection/Web/SEO analysis tool, see http://guess.scritch.org)" 3291 | ] 3292 | } 3293 | , 3294 | { 3295 | "pattern": "Fever", 3296 | "addition_date": "2018/06/27", 3297 | "instances": [ 3298 | "Fever/1.38 (Feed Parser; http://feedafever.com; Allow like Gecko)" 3299 | ], 3300 | "url": "http://feedafever.com" 3301 | } 3302 | , 3303 | { 3304 | "pattern": "Flamingo_SearchEngine", 3305 | "addition_date": "2018/06/27", 3306 | "instances": [ 3307 | "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)" 3308 | ] 3309 | } 3310 | , 3311 | { 3312 | "pattern": "FlipboardProxy", 3313 | "addition_date": "2018/06/27", 3314 | "instances": [ 3315 | "Mozilla/5.0 (compatible; FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", 3316 | "Mozilla/5.0 (compatible; FlipboardProxy/1.2; +http://flipboard.com/browserproxy)", 3317 | "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", 3318 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)", 3319 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)" 3320 | ], 3321 | "url": "https://about.flipboard.com/browserproxy/", 3322 | "description": "a proxy service to fetch, validate, and prepare certain elements of websites for presentation through the Flipboard Application" 3323 | } 3324 | , 3325 | { 3326 | "pattern": "g2reader-bot", 3327 | "addition_date": "2018/06/27", 3328 | "instances": [ 3329 | "g2reader-bot/1.0 (+http://www.g2reader.com/)" 3330 | ], 3331 | "url": "http://www.g2reader.com/" 3332 | } 3333 | , 3334 | { 3335 | "pattern": "G2 Web Services", 3336 | "addition_date": "2019/03/01", 3337 | "instances": [ 3338 | "G2 Web Services/1.0 (built with StormCrawler Archetype 1.8; https://www.g2webservices.com/; developers@g2llc.com)" 3339 | ], 3340 | "url": "https://www.g2webservices.com/" 3341 | } 3342 | , 3343 | { 3344 | "pattern": "imrbot", 3345 | "addition_date": "2018/06/27", 3346 | "instances": [ 3347 | "Mozilla/5.0 (compatible; imrbot/1.10.8 +http://www.mignify.com)" 3348 | ], 3349 | "url": "http://www.mignify.com" 3350 | } 3351 | , 3352 | { 3353 | "pattern": "K7MLWCBot", 3354 | "addition_date": "2018/06/27", 3355 | "instances": [ 3356 | "K7MLWCBot/1.0 (+http://www.k7computing.com)" 3357 | ], 3358 | "url": "http://www.k7computing.com", 3359 | "description": "Virus scanner" 3360 | } 3361 | , 3362 | { 3363 | "pattern": "Kemvibot", 3364 | "addition_date": "2018/06/27", 3365 | "instances": [ 3366 | "Kemvibot/1.0 (http://kemvi.com, marco@kemvi.com)" 3367 | ], 3368 | "url": "http://kemvi.com" 3369 | } 3370 | , 3371 | { 3372 | "pattern": "Landau-Media-Spider", 3373 | "addition_date": "2018/06/27", 3374 | "instances": [ 3375 | "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)" 3376 | ], 3377 | "url": "http://bots.landaumedia.de/bot.html" 3378 | } 3379 | , 3380 | { 3381 | "pattern": "linkapediabot", 3382 | "addition_date": "2018/06/27", 3383 | "instances": [ 3384 | "linkapediabot (+http://www.linkapedia.com)" 3385 | ], 3386 | "url": "http://www.linkapedia.com" 3387 | } 3388 | , 3389 | { 3390 | "pattern": "vkShare", 3391 | "addition_date": "2018/07/02", 3392 | "instances": [ 3393 | "Mozilla/5.0 (compatible; vkShare; +http://vk.com/dev/Share)" 3394 | ], 3395 | "url": "http://vk.com/dev/Share" 3396 | } 3397 | , 3398 | { 3399 | "pattern": "Siteimprove.com", 3400 | "addition_date": "2018/06/22", 3401 | "instances": [ 3402 | "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com", 3403 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) Match by Siteimprove.com", 3404 | "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) SiteCheck-sitecrawl by Siteimprove.com", 3405 | "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.0) LinkCheck by Siteimprove.com" 3406 | ] 3407 | } 3408 | , 3409 | { 3410 | "pattern": "BLEXBot\\/", 3411 | "addition_date": "2018/07/07", 3412 | "instances": [ 3413 | "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" 3414 | ], 3415 | "url": "http://webmeup-crawler.com" 3416 | } 3417 | , 3418 | { 3419 | "pattern": "DareBoost", 3420 | "addition_date": "2018/07/07", 3421 | "instances": [ 3422 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36 DareBoost" 3423 | ], 3424 | "url": "https://www.dareboost.com/", 3425 | "description": "Bot to test, Analyze and Optimize website" 3426 | } 3427 | , 3428 | { 3429 | "pattern": "ZuperlistBot\\/", 3430 | "addition_date": "2018/07/07", 3431 | "instances": [ 3432 | "Mozilla/5.0 (compatible; ZuperlistBot/1.0)" 3433 | ] 3434 | } 3435 | , 3436 | { 3437 | "pattern": "Miniflux\\/", 3438 | "addition_date": "2018/07/07", 3439 | "instances": [ 3440 | "Mozilla/5.0 (compatible; Miniflux/2.0.x-dev; +https://miniflux.net)", 3441 | "Mozilla/5.0 (compatible; Miniflux/2.0.3; +https://miniflux.net)", 3442 | "Mozilla/5.0 (compatible; Miniflux/2.0.7; +https://miniflux.net)", 3443 | "Mozilla/5.0 (compatible; Miniflux/2.0.10; +https://miniflux.net)", 3444 | "Mozilla/5.0 (compatibl$; Miniflux/2.0.x-dev; +https://miniflux.app)", 3445 | "Mozilla/5.0 (compatible; Miniflux/2.0.11; +https://miniflux.app)", 3446 | "Mozilla/5.0 (compatible; Miniflux/2.0.12; +https://miniflux.app)", 3447 | "Mozilla/5.0 (compatible; Miniflux/ae1dc1a; +https://miniflux.app)", 3448 | "Mozilla/5.0 (compatible; Miniflux/3b6e44c; +https://miniflux.app)" 3449 | ], 3450 | "url": "https://miniflux.net", 3451 | "description": "Miniflux is a minimalist and opinionated feed reader." 3452 | } 3453 | , 3454 | { 3455 | "pattern": "Feedspot", 3456 | "addition_date": "2018/07/07", 3457 | "instances": [ 3458 | "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", 3459 | "Mozilla/5.0 (compatible; Feedspot/1.0 (+https://www.feedspot.com/fs/fetcher; like FeedFetcher-Google)" 3460 | ], 3461 | "url": "http://www.feedspot.com/fs/bot" 3462 | } 3463 | , 3464 | { 3465 | "pattern": "Diffbot\\/", 3466 | "addition_date": "2018/07/07", 3467 | "instances": [ 3468 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)" 3469 | ], 3470 | "url": "http://www.diffbot.com" 3471 | } 3472 | , 3473 | { 3474 | "pattern": "SEOkicks", 3475 | "addition_date": "2018/08/22", 3476 | "instances": [ 3477 | "Mozilla/5.0 (compatible; SEOkicks; +https://www.seokicks.de/robot.html)" 3478 | ], 3479 | "url": "https://www.seokicks.de/robot.html" 3480 | } 3481 | , 3482 | { 3483 | "pattern": "tracemyfile", 3484 | "addition_date": "2018/08/23", 3485 | "instances": [ 3486 | "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" 3487 | ] 3488 | } 3489 | , 3490 | { 3491 | "pattern": "Nimbostratus-Bot", 3492 | "addition_date": "2018/08/29", 3493 | "instances": [ 3494 | "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" 3495 | ] 3496 | } 3497 | , 3498 | { 3499 | "pattern": "zgrab", 3500 | "addition_date": "2018/08/30", 3501 | "instances": [ 3502 | "Mozilla/5.0 zgrab/0.x" 3503 | ], 3504 | "url": "https://zmap.io/" 3505 | } 3506 | , 3507 | { 3508 | "pattern": "PR-CY.RU", 3509 | "addition_date": "2018/08/30", 3510 | "instances": [ 3511 | "Mozilla/5.0 (compatible; PR-CY.RU; + https://a.pr-cy.ru)" 3512 | ], 3513 | "url": "https://a.pr-cy.ru/" 3514 | } 3515 | , 3516 | { 3517 | "pattern": "AdsTxtCrawler", 3518 | "addition_date": "2018/08/30", 3519 | "instances": [ 3520 | "AdsTxtCrawler/1.0" 3521 | ] 3522 | }, 3523 | { 3524 | "pattern": "Datafeedwatch", 3525 | "addition_date": "2018/09/05", 3526 | "instances": [ 3527 | "Datafeedwatch/2.1.x" 3528 | ], 3529 | "url": "https://www.datafeedwatch.com/" 3530 | } 3531 | , 3532 | { 3533 | "pattern": "Zabbix", 3534 | "addition_date": "2018/09/05", 3535 | "instances": [ 3536 | "Zabbix" 3537 | ], 3538 | "url": "https://www.zabbix.com/documentation/3.4/manual/web_monitoring" 3539 | } 3540 | , 3541 | { 3542 | "pattern": "TangibleeBot", 3543 | "addition_date": "2018/09/05", 3544 | "instances": [ 3545 | "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)" 3546 | ], 3547 | "url": "http://tangiblee.com/bot" 3548 | } 3549 | , 3550 | { 3551 | "pattern": "google-xrawler", 3552 | "addition_date": "2018/09/05", 3553 | "instances": [ 3554 | "google-xrawler" 3555 | ], 3556 | "url": "https://webmasters.stackexchange.com/questions/105560/what-is-the-google-xrawler-user-agent-used-for" 3557 | } 3558 | , 3559 | { 3560 | "pattern": "axios", 3561 | "addition_date": "2018/09/06", 3562 | "instances": [ 3563 | "axios/0.18.0", 3564 | "axios/0.19.0" 3565 | ], 3566 | "url": "https://github.com/axios/axios" 3567 | } 3568 | , 3569 | { 3570 | "pattern": "Amazon CloudFront", 3571 | "addition_date": "2018/09/07", 3572 | "instances": [ 3573 | "Amazon CloudFront" 3574 | ], 3575 | "url": "https://aws.amazon.com/cloudfront/" 3576 | } 3577 | , 3578 | { 3579 | "pattern": "Pulsepoint", 3580 | "addition_date": "2018/09/24", 3581 | "instances": [ 3582 | "Pulsepoint XT3 web scraper" 3583 | ] 3584 | } 3585 | , 3586 | { 3587 | "pattern": "CloudFlare-AlwaysOnline", 3588 | "addition_date": "2018/09/27", 3589 | "instances": [ 3590 | "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34", 3591 | "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +https://www.cloudflare.com/always-online) AppleWebKit/534.34" 3592 | ], 3593 | "url" : "https://www.cloudflare.com/always-online/" 3594 | } 3595 | , 3596 | { 3597 | "pattern": "Google-Structured-Data-Testing-Tool", 3598 | "addition_date": "2018/10/02", 3599 | "instances": [ 3600 | "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +https://search.google.com/structured-data/testing-tool)", 3601 | "Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)" 3602 | ], 3603 | "url": "https://search.google.com/structured-data/testing-tool" 3604 | } 3605 | , 3606 | { 3607 | "pattern": "WordupInfoSearch", 3608 | "addition_date": "2018/10/07", 3609 | "instances": [ 3610 | "WordupInfoSearch/1.0" 3611 | ] 3612 | } 3613 | , 3614 | { 3615 | "pattern": "WebDataStats", 3616 | "addition_date": "2018/10/08", 3617 | "instances": [ 3618 | "Mozilla/5.0 (compatible; WebDataStats/1.0 ; +https://webdatastats.com/policy.html)" 3619 | ], 3620 | "url": "https://webdatastats.com/" 3621 | } 3622 | , 3623 | { 3624 | "pattern": "HttpUrlConnection", 3625 | "addition_date": "2018/10/08", 3626 | "instances": [ 3627 | "Jersey/2.25.1 (HttpUrlConnection 1.8.0_141)" 3628 | ] 3629 | } 3630 | , 3631 | { 3632 | "pattern": "Seekport Crawler", 3633 | "addition_date": "2018/10/08", 3634 | "instances": [ 3635 | "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/)" 3636 | ], 3637 | "url": "http://seekport.com/" 3638 | } 3639 | , 3640 | { 3641 | "pattern": "ZoomBot", 3642 | "addition_date": "2018/10/10", 3643 | "instances": [ 3644 | "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)" 3645 | ], 3646 | "url": "http://suite.seozoom.it/bot.html" 3647 | } 3648 | , 3649 | { 3650 | "pattern": "VelenPublicWebCrawler", 3651 | "addition_date": "2018/10/09", 3652 | "instances": [ 3653 | "VelenPublicWebCrawler (velen.io)" 3654 | ] 3655 | } 3656 | , 3657 | { 3658 | "pattern": "MoodleBot", 3659 | "addition_date": "2018/10/10", 3660 | "instances": [ 3661 | "MoodleBot/1.0" 3662 | ] 3663 | } 3664 | , 3665 | { 3666 | "pattern": "jpg-newsbot", 3667 | "addition_date": "2018/10/10", 3668 | "instances": [ 3669 | "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)" 3670 | ], 3671 | "url": "https://vipnytt.no/bots/" 3672 | } 3673 | , 3674 | { 3675 | "pattern": "outbrain", 3676 | "addition_date": "2018/10/14", 3677 | "instances": [ 3678 | "Mozilla/5.0 (Java) outbrain" 3679 | ], 3680 | "url": "https://www.outbrain.com/help/advertisers/invalid-url/" 3681 | } 3682 | , 3683 | { 3684 | "pattern": "W3C_Validator", 3685 | "addition_date": "2018/10/14", 3686 | "instances": [ 3687 | "W3C_Validator/1.3" 3688 | ], 3689 | "url": "https://validator.w3.org/services" 3690 | } 3691 | , 3692 | { 3693 | "pattern": "Validator\\.nu", 3694 | "addition_date": "2018/10/14", 3695 | "instances": [ 3696 | "Validator.nu/LV" 3697 | ], 3698 | "url": "https://validator.w3.org/services" 3699 | } 3700 | , 3701 | { 3702 | "pattern": "W3C-checklink", 3703 | "addition_date": "2018/10/14", 3704 | "depends_on": ["libwww-perl"], 3705 | "instances": [ 3706 | "W3C-checklink/2.90 libwww-perl/5.64", 3707 | "W3C-checklink/3.6.2.3 libwww-perl/5.64", 3708 | "W3C-checklink/4.2 [4.20] libwww-perl/5.803", 3709 | "W3C-checklink/4.2.1 [4.21] libwww-perl/5.803", 3710 | "W3C-checklink/4.3 [4.42] libwww-perl/5.805", 3711 | "W3C-checklink/4.3 [4.42] libwww-perl/5.808", 3712 | "W3C-checklink/4.3 [4.42] libwww-perl/5.820", 3713 | "W3C-checklink/4.5 [4.154] libwww-perl/5.823", 3714 | "W3C-checklink/4.5 [4.160] libwww-perl/5.823" 3715 | ], 3716 | "url": "https://validator.w3.org/services" 3717 | } 3718 | , 3719 | { 3720 | "pattern": "W3C-mobileOK", 3721 | "addition_date": "2018/10/14", 3722 | "instances": [ 3723 | "W3C-mobileOK/DDC-1.0" 3724 | ], 3725 | "url": "https://validator.w3.org/services" 3726 | } 3727 | , 3728 | { 3729 | "pattern": "W3C_I18n-Checker", 3730 | "addition_date": "2018/10/14", 3731 | "instances": [ 3732 | "W3C_I18n-Checker/1.0" 3733 | ], 3734 | "url": "https://validator.w3.org/services" 3735 | } 3736 | , 3737 | { 3738 | "pattern": "FeedValidator", 3739 | "addition_date": "2018/10/14", 3740 | "instances": [ 3741 | "FeedValidator/1.3" 3742 | ], 3743 | "url": "https://validator.w3.org/services" 3744 | } 3745 | , 3746 | { 3747 | "pattern": "W3C_CSS_Validator", 3748 | "addition_date": "2018/10/14", 3749 | "instances": [ 3750 | "Jigsaw/2.3.0 W3C_CSS_Validator_JFouffa/2.0" 3751 | ], 3752 | "url": "https://validator.w3.org/services" 3753 | } 3754 | , 3755 | { 3756 | "pattern": "W3C_Unicorn", 3757 | "addition_date": "2018/10/14", 3758 | "instances": [ 3759 | "W3C_Unicorn/1.0" 3760 | ], 3761 | "url": "https://validator.w3.org/services" 3762 | } 3763 | , 3764 | { 3765 | "pattern": "Google-PhysicalWeb", 3766 | "addition_date": "2018/10/21", 3767 | "instances": [ 3768 | "Mozilla/5.0 (Google-PhysicalWeb)" 3769 | ] 3770 | } 3771 | , 3772 | { 3773 | "pattern": "Blackboard", 3774 | "addition_date": "2018/10/28", 3775 | "instances": [ 3776 | "Blackboard Safeassign" 3777 | ], 3778 | "url": "https://help.blackboard.com/Learn/Administrator/Hosting/Tools_Management/SafeAssign" 3779 | }, 3780 | { 3781 | "pattern": "ICBot\\/", 3782 | "addition_date": "2018/10/23", 3783 | "instances": [ 3784 | "Mozilla/5.0 (compatible; ICBot/0.1; +https://ideasandcode.xyz" 3785 | ], 3786 | "url": "https://ideasandcode.xyz" 3787 | }, 3788 | { 3789 | "pattern": "BazQux", 3790 | "addition_date": "2018/10/23", 3791 | "instances": [ 3792 | "Mozilla/5.0 (compatible; BazQux/2.4; +https://bazqux.com/fetcher; 1 subscribers)" 3793 | ], 3794 | "url": "https://bazqux.com/fetcher" 3795 | }, 3796 | { 3797 | "pattern": "Twingly", 3798 | "addition_date": "2018/10/23", 3799 | "instances": [ 3800 | "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)" 3801 | ], 3802 | "url": "https://twingly.com" 3803 | }, 3804 | { 3805 | "pattern": "Rivva", 3806 | "addition_date": "2018/10/23", 3807 | "instances": [ 3808 | "Mozilla/5.0 (compatible; Rivva; http://rivva.de)" 3809 | ], 3810 | "url": "http://rivva.de" 3811 | }, 3812 | { 3813 | "pattern": "Experibot", 3814 | "addition_date": "2018/11/03", 3815 | "instances": [ 3816 | "Experibot-v2 http://goo.gl/ZAr8wX", 3817 | "Experibot-v3 http://goo.gl/ZAr8wX" 3818 | ], 3819 | "url": "https://amirkr.wixsite.com/experibot" 3820 | }, 3821 | { 3822 | "pattern": "awesomecrawler", 3823 | "addition_date": "2018/11/24", 3824 | "instances": [ 3825 | "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.5 Safari/537.22 +awesomecrawler" 3826 | ] 3827 | }, 3828 | { 3829 | "pattern": "Dataprovider.com", 3830 | "addition_date": "2018/11/24", 3831 | "instances": [ 3832 | "Mozilla/5.0 (compatible; Dataprovider.com)" 3833 | ], 3834 | "url": "https://www.dataprovider.com/" 3835 | }, 3836 | { 3837 | "pattern": "GroupHigh\\/", 3838 | "addition_date": "2018/11/24", 3839 | "instances": [ 3840 | "Mozilla/5.0 (compatible; GroupHigh/1.0; +http://www.grouphigh.com/" 3841 | ], 3842 | "url": "http://www.grouphigh.com/" 3843 | }, 3844 | { 3845 | "pattern": "theoldreader.com", 3846 | "addition_date": "2018/12/02", 3847 | "instances": [ 3848 | "Mozilla/5.0 (compatible; theoldreader.com)" 3849 | ], 3850 | "url": "https://www.theoldreader.com/" 3851 | } 3852 | , 3853 | { 3854 | "pattern": "AnyEvent", 3855 | "addition_date": "2018/12/07", 3856 | "instances": [ 3857 | "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/2.24; +http://software.schmorp.de/pkg/AnyEvent)" 3858 | ], 3859 | "url": "http://software.schmorp.de/pkg/AnyEvent.html" 3860 | } 3861 | , 3862 | { 3863 | "pattern": "Uptimebot\\.org", 3864 | "addition_date": "2019/01/17", 3865 | "instances": [ 3866 | "Uptimebot.org - Free website monitoring" 3867 | ], 3868 | "url": "http://uptimebot.org/" 3869 | } 3870 | , 3871 | { 3872 | "pattern": "Nmap Scripting Engine", 3873 | "addition_date": "2019/02/04", 3874 | "instances": [ 3875 | "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)" 3876 | ], 3877 | "url": "https://nmap.org/book/nse.html" 3878 | } 3879 | , 3880 | { 3881 | "pattern": "2ip.ru", 3882 | "addition_date": "2019/02/12", 3883 | "instances": [ 3884 | "2ip.ru CMS Detector (https://2ip.ru/cms/)" 3885 | ], 3886 | "url": "https://2ip.ru/cms/" 3887 | }, 3888 | { 3889 | "pattern": "Clickagy", 3890 | "addition_date": "2019/02/19", 3891 | "instances": [ 3892 | "Clickagy Intelligence Bot v2" 3893 | ], 3894 | "url": "https://www.clickagy.com" 3895 | }, 3896 | { 3897 | "pattern": "Caliperbot", 3898 | "addition_date": "2019/03/02", 3899 | "instances": [ 3900 | "Caliperbot/1.0 (+http://www.conductor.com/caliperbot)" 3901 | ], 3902 | "url": "http://www.conductor.com/caliperbot" 3903 | }, 3904 | { 3905 | "pattern": "MBCrawler", 3906 | "addition_date": "2019/03/02", 3907 | "instances": [ 3908 | "MBCrawler/1.0 (https://monitorbacklinks.com)" 3909 | ], 3910 | "url": "https://monitorbacklinks.com" 3911 | }, 3912 | { 3913 | "pattern": "online-webceo-bot", 3914 | "addition_date": "2019/03/02", 3915 | "instances": [ 3916 | "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)" 3917 | ], 3918 | "url": "http://online.webceo.com" 3919 | }, 3920 | { 3921 | "pattern": "B2B Bot", 3922 | "addition_date": "2019/03/02", 3923 | "instances": [ 3924 | "B2B Bot" 3925 | ] 3926 | }, 3927 | { 3928 | "pattern": "AddSearchBot", 3929 | "addition_date": "2019/03/02", 3930 | "instances": [ 3931 | "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)" 3932 | ], 3933 | "url": "http://www.addsearch.com/bot" 3934 | }, 3935 | { 3936 | "pattern": "Google Favicon", 3937 | "addition_date": "2019/03/14", 3938 | "instances": [ 3939 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" 3940 | ] 3941 | }, 3942 | { 3943 | "pattern": "HubSpot", 3944 | "addition_date": "2019/04/15", 3945 | "instances": [ 3946 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler - web-crawlers@hubspot.com", 3947 | "Mozilla/5.0 (X11; Linux x86_64; HubSpot Single Page link check; web-crawlers+links@hubspot.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", 3948 | "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)", 3949 | "HubSpot Connect 2.0 (http://dev.hubspot.com/) - BizOpsCompanies-Tq2-BizCoDomainValidationAudit" 3950 | ] 3951 | }, 3952 | { 3953 | "pattern": "Chrome-Lighthouse", 3954 | "addition_date": "2019/03/15", 3955 | "instances": [ 3956 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Mobile Safari/537.36 Chrome-Lighthouse", 3957 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse", 3958 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Safari/537.36 Chrome-Lighthouse", 3959 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3694.0 Mobile Safari/537.36 Chrome-Lighthouse" 3960 | ], 3961 | "url": "https://developers.google.com/speed/pagespeed/insights" 3962 | }, 3963 | { 3964 | "pattern": "HeadlessChrome", 3965 | "url": "https://developers.google.com/web/updates/2017/04/headless-chrome", 3966 | "addition_date": "2019/06/17", 3967 | "instances": [ 3968 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/74.0.3729.169 Safari/537.36", 3969 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/69.0.3494.0 Safari/537.36", 3970 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/76.0.3803.0 Safari/537.36" 3971 | ] 3972 | }, 3973 | { 3974 | "pattern": "CheckMarkNetwork\\/", 3975 | "addition_date": "2019/06/30", 3976 | "instances": [ 3977 | "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" 3978 | ], 3979 | "url": "https://www.checkmarknetwork.com/" 3980 | }, 3981 | { 3982 | "pattern": "www\\.uptime\\.com", 3983 | "addition_date": "2019/07/21", 3984 | "instances": [ 3985 | "Mozilla/5.0 (compatible; Uptimebot/1.0; +http://www.uptime.com/uptimebot)" 3986 | ], 3987 | "url": "http://www.uptime.com/uptimebot" 3988 | } 3989 | , 3990 | { 3991 | "pattern": "Streamline3Bot\\/", 3992 | "addition_date": "2019/07/21", 3993 | "instances": [ 3994 | "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", 3995 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0" 3996 | ], 3997 | "url": "https://www.ubtsupport.com/legal/Streamline3Bot.php" 3998 | } 3999 | , 4000 | { 4001 | "pattern": "serpstatbot\\/", 4002 | "addition_date": "2019/07/25", 4003 | "instances": [ 4004 | "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", 4005 | "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)" 4006 | ], 4007 | "url": "http://serpstatbot.com" 4008 | } 4009 | , 4010 | { 4011 | "pattern": "MixnodeCache\\/", 4012 | "addition_date": "2019/08/04", 4013 | "instances": [ 4014 | "MixnodeCache/1.8(+https://cache.mixnode.com/)" 4015 | ], 4016 | "url": "https://cache.mixnode.com/" 4017 | } 4018 | , 4019 | { 4020 | "pattern": "^curl", 4021 | "addition_date": "2019/08/15", 4022 | "instances": [ 4023 | "curl", 4024 | "curl/7.29.0", 4025 | "curl/7.47.0", 4026 | "curl/7.54.0", 4027 | "curl/7.55.1", 4028 | "curl/7.64.0", 4029 | "curl/7.64.1", 4030 | "curl/7.65.3" 4031 | ], 4032 | "url": "https://curl.haxx.se/" 4033 | } 4034 | , 4035 | { 4036 | "pattern": "SimpleScraper", 4037 | "addition_date": "2019/08/16", 4038 | "instances": [ 4039 | "Mozilla/5.0 (compatible; SimpleScraper)" 4040 | ], 4041 | "url": "https://github.com/ramonkcom/simple-scraper/" 4042 | } 4043 | , 4044 | { 4045 | "pattern": "RSSingBot", 4046 | "addition_date": "2019/09/15", 4047 | "instances": [ 4048 | "RSSingBot (http://www.rssing.com)" 4049 | ], 4050 | "url": "http://www.rssing.com" 4051 | } 4052 | , 4053 | { 4054 | "pattern": "Jooblebot", 4055 | "addition_date": "2019/09/25", 4056 | "instances": [ 4057 | "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36" 4058 | ], 4059 | "url": "http://jooble.org/jooble-bot" 4060 | } 4061 | , 4062 | { 4063 | "pattern": "fedoraplanet", 4064 | "addition_date": "2019/09/28", 4065 | "instances": [ 4066 | "venus/fedoraplanet" 4067 | ], 4068 | "url": "http://fedoraplanet.org/" 4069 | } 4070 | , 4071 | { 4072 | "pattern": "Friendica", 4073 | "addition_date": "2019/09/28", 4074 | "instances": [ 4075 | "Friendica 'The Tazmans Flax-lily' 2019.01-1293; https://hoyer.xyz" 4076 | ], 4077 | "url": "https://hoyer.xyz" 4078 | } 4079 | , 4080 | { 4081 | "pattern": "NextCloud", 4082 | "addition_date": "2019/09/30", 4083 | "instances": [ 4084 | "NextCloud-News/1.0" 4085 | ], 4086 | "url": "https://nextcloud.com/" 4087 | } 4088 | , 4089 | { 4090 | "pattern": "Tiny Tiny RSS", 4091 | "addition_date": "2019/10/04", 4092 | "instances": [ 4093 | "Tiny Tiny RSS/1.15.3 (http://tt-rss.org/)", 4094 | "Tiny Tiny RSS/17.12 (a2d1fa5) (http://tt-rss.org/)", 4095 | "Tiny Tiny RSS/19.2 (b68db2d) (http://tt-rss.org/)", 4096 | "Tiny Tiny RSS/19.8 (http://tt-rss.org/)" 4097 | ], 4098 | "url": "http://tt-rss.org/" 4099 | } 4100 | , 4101 | { 4102 | "pattern": "RegionStuttgartBot", 4103 | "addition_date": "2019/10/17", 4104 | "instances": [ 4105 | "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)" 4106 | ], 4107 | "url": "http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/" 4108 | } 4109 | , 4110 | { 4111 | "pattern": "Bytespider", 4112 | "addition_date": "2019/11/11", 4113 | "instances": [ 4114 | "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.3754.1902 Mobile Safari/537.36; Bytespider", 4115 | "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.4454.1745 Mobile Safari/537.36; Bytespider", 4116 | "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.7597.1164 Mobile Safari/537.36; Bytespider;bytespider@bytedance.com", 4117 | "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2988.1545 Mobile Safari/537.36; Bytespider", 4118 | "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4141.1682 Mobile Safari/537.36; Bytespider", 4119 | "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.3478.1649 Mobile Safari/537.36; Bytespider", 4120 | "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.5267.1259 Mobile Safari/537.36; Bytespider", 4121 | "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.7990.1979 Mobile Safari/537.36; Bytespider", 4122 | "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.2268.1523 Mobile Safari/537.36; Bytespider", 4123 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2576.1836 Mobile Safari/537.36; Bytespider", 4124 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.9681.1227 Mobile Safari/537.36; Bytespider", 4125 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.6023.1635 Mobile Safari/537.36; Bytespider", 4126 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.4944.1981 Mobile Safari/537.36; Bytespider", 4127 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.3613.1739 Mobile Safari/537.36; Bytespider", 4128 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.4022.1033 Mobile Safari/537.36; Bytespider", 4129 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.3248.1547 Mobile Safari/537.36; Bytespider", 4130 | "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.5527.1507 Mobile Safari/537.36; Bytespider", 4131 | "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.5216.1326 Mobile Safari/537.36; Bytespider", 4132 | "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.9038.1080 Mobile Safari/537.36; Bytespider" 4133 | ], 4134 | "url": "https://stackoverflow.com/questions/57908900/what-is-the-bytespider-user-agent" 4135 | } 4136 | , 4137 | { 4138 | "pattern": "Datanyze", 4139 | "addition_date": "2019/11/17", 4140 | "instances": [ 4141 | "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" 4142 | ], 4143 | "url": "https://www.datanyze.com/dnyzbot/" 4144 | } 4145 | , 4146 | { 4147 | "pattern": "Google-Site-Verification", 4148 | "addition_date": "2019/12/11", 4149 | "instances": [ 4150 | "Mozilla/5.0 (compatible; Google-Site-Verification/1.0)" 4151 | ], 4152 | "url": "https://support.google.com/webmasters/answer/9008080" 4153 | } 4154 | , 4155 | { 4156 | "pattern": "TrendsmapResolver", 4157 | "addition_date": "2020/02/24", 4158 | "instances": [ 4159 | "Mozilla/5.0 (compatible; TrendsmapResolver/0.1)" 4160 | ], 4161 | "url": "https://www.trendsmap.com/" 4162 | } 4163 | , 4164 | { 4165 | "pattern": "tweetedtimes", 4166 | "addition_date": "2020/02/24", 4167 | "instances": [ 4168 | "Mozilla/5.0 (compatible; +http://tweetedtimes.com)" 4169 | ], 4170 | "url": "https://tweetedtimes.com/" 4171 | }, 4172 | { 4173 | "pattern": "NTENTbot", 4174 | "addition_date": "2020/02/24", 4175 | "instances": [ 4176 | "Mozilla/5.0 (compatible; NTENTbot; +http://www.ntent.com/ntentbot)" 4177 | ], 4178 | "url": "https://ntent.com/ntentbot/" 4179 | }, 4180 | { 4181 | "pattern": "Gwene", 4182 | "addition_date": "2020/02/24", 4183 | "instances": [ 4184 | "Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot" 4185 | ], 4186 | "url": "https://gwene.org" 4187 | }, 4188 | { 4189 | "pattern": "SimplePie", 4190 | "addition_date": "2020/02/24", 4191 | "instances": [ 4192 | "SimplePie/1.3-dev (Feed Parser; http://simplepie.org; Allow like Gecko)" 4193 | ], 4194 | "url": "http://simplepie.org" 4195 | }, 4196 | { 4197 | "pattern": "SearchAtlas", 4198 | "addition_date": "2020/03/02", 4199 | "instances": [ 4200 | "SearchAtlas.com SEO Crawler" 4201 | ], 4202 | "url": "http://SearchAtlas.com" 4203 | }, 4204 | { 4205 | "pattern": "Superfeedr", 4206 | "addition_date": "2020/03/02", 4207 | "instances": [ 4208 | "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860" 4209 | ], 4210 | "url": "http://superfeedr.com" 4211 | }, 4212 | { 4213 | "pattern": "feedbot", 4214 | "addition_date": "2020/03/02", 4215 | "instances": [ 4216 | "wp.com feedbot/1.0 (+https://wp.com)" 4217 | ], 4218 | "url": "http://wp.com" 4219 | }, 4220 | { 4221 | "pattern": "UT-Dorkbot", 4222 | "addition_date": "2020/03/02", 4223 | "instances": [ 4224 | "UT-Dorkbot/1.0" 4225 | ], 4226 | "url": "https://security.utexas.edu/dorkbot" 4227 | }, 4228 | { 4229 | "pattern": "Amazonbot", 4230 | "addition_date": "2020/03/02", 4231 | "instances": [ 4232 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" 4233 | ], 4234 | "url": "https://developer.amazon.com/support/amazonbot" 4235 | }, 4236 | { 4237 | "pattern": "SerendeputyBot", 4238 | "addition_date": "2020/03/02", 4239 | "instances": [ 4240 | "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)" 4241 | ], 4242 | "url": "http://serendeputy.com/about/serendeputy-bot" 4243 | }, 4244 | { 4245 | "pattern": "Eyeotabot", 4246 | "addition_date": "2020/03/02", 4247 | "instances": [ 4248 | "Mozilla/5.0 (compatible; Eyeotabot/1.0; +http://www.eyeota.com)" 4249 | ], 4250 | "url": "http://www.eyeota.com" 4251 | }, 4252 | { 4253 | "pattern": "officestorebot", 4254 | "addition_date": "2020/03/02", 4255 | "instances": [ 4256 | "Mozilla/5.0 (compatible; officestorebot/1.0; +https://aka.ms/officestorebot)" 4257 | ], 4258 | "url": "https://aka.ms/officestorebot" 4259 | }, 4260 | { 4261 | "pattern": "Neticle Crawler", 4262 | "addition_date": "2020/03/02", 4263 | "instances": [ 4264 | "Neticle Crawler v1.0 ( https://neticle.com/bot/en/ )" 4265 | ], 4266 | "url": "https://neticle.com/bot/en/" 4267 | }, 4268 | { 4269 | "pattern": "SurdotlyBot", 4270 | "addition_date": "2020/03/02", 4271 | "instances": [ 4272 | "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)" 4273 | ], 4274 | "url": "http://sur.ly/bot.html" 4275 | }, 4276 | { 4277 | "pattern": "LinkisBot", 4278 | "addition_date": "2020/03/02", 4279 | "instances": [ 4280 | "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" 4281 | ] 4282 | }, 4283 | { 4284 | "pattern": "AwarioSmartBot", 4285 | "addition_date": "2020/03/02", 4286 | "instances": [ 4287 | "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" 4288 | ], 4289 | "url": "https://awario.com/bots.html" 4290 | }, 4291 | { 4292 | "pattern": "AwarioRssBot", 4293 | "addition_date": "2020/03/02", 4294 | "instances": [ 4295 | "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)" 4296 | ], 4297 | "url": "https://awario.com/bots.html" 4298 | }, 4299 | { 4300 | "pattern": "RyteBot", 4301 | "addition_date": "2020/03/02", 4302 | "instances": [ 4303 | "RyteBot/1.0.0 (+https://bot.ryte.com/)" 4304 | ], 4305 | "url": "https://bot.ryte.com/" 4306 | }, 4307 | { 4308 | "pattern": "FreeWebMonitoring SiteChecker", 4309 | "addition_date": "2020/03/02", 4310 | "instances": [ 4311 | "FreeWebMonitoring SiteChecker/0.2 (+https://www.freewebmonitoring.com/bot.html)" 4312 | ], 4313 | "url": "https://www.freewebmonitoring.com/bot.html" 4314 | }, 4315 | { 4316 | "pattern": "AspiegelBot", 4317 | "addition_date": "2020/03/16", 4318 | "instances": [ 4319 | "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)" 4320 | ], 4321 | "url": "https://aspiegel.com" 4322 | }, 4323 | { 4324 | "pattern": "NAVER Blog Rssbot", 4325 | "addition_date": "2020/03/16", 4326 | "instances": [ 4327 | "NAVER Blog Rssbot" 4328 | ], 4329 | "url": "http://www.naver.com" 4330 | }, 4331 | { 4332 | "pattern": "zenback bot", 4333 | "addition_date": "2020/03/16", 4334 | "instances": [ 4335 | "Mozilla/5.0 (compatible; zenback bot; powered by logly +http://corp.logly.co.jp/)" 4336 | ], 4337 | "url": "http://corp.logly.co.jp/" 4338 | }, 4339 | { 4340 | "pattern": "SentiBot", 4341 | "addition_date": "2020/03/16", 4342 | "instances": [ 4343 | "SentiBot www.sentibot.eu (compatible with Googlebot)" 4344 | ], 4345 | "url": "https://www.sentibot.eu" 4346 | }, 4347 | { 4348 | "pattern": "Domains Project\\/", 4349 | "addition_date": "2020/03/16", 4350 | "instances": [ 4351 | "Mozilla/5.0 (compatible; Domains Project/1.0.3; +https://github.com/tb0hdan/domains)" 4352 | ], 4353 | "url": "https://github.com/tb0hdan/domains" 4354 | }, 4355 | { 4356 | "pattern": "Pandalytics", 4357 | "addition_date": "2020/03/16", 4358 | "instances": [ 4359 | "Pandalytics/1.0 (https://domainsbot.com/pandalytics/)" 4360 | ], 4361 | "url": "https://domainsbot.com/pandalytics/" 4362 | }, 4363 | { 4364 | "pattern": "VKRobot", 4365 | "addition_date": "2020/03/16", 4366 | "instances": [ 4367 | "Mozilla/5.0 (compatible; VKRobot/1.0)" 4368 | ] 4369 | }, 4370 | { 4371 | "pattern": "bidswitchbot", 4372 | "addition_date": "2020/03/16", 4373 | "instances": [ 4374 | "bidswitchbot/1.0" 4375 | ], 4376 | "url": "https://www.bidswitch.com/about-us/" 4377 | }, 4378 | { 4379 | "pattern": "tigerbot", 4380 | "addition_date": "2020/03/16", 4381 | "instances": [ 4382 | "tigerbot" 4383 | ] 4384 | }, 4385 | { 4386 | "pattern": "NIXStatsbot", 4387 | "addition_date": "2020/03/16", 4388 | "instances": [ 4389 | "Mozilla/5.0 (compatible; NIXStatsbot/1.1; +http://www.nixstats.com/bot.html)" 4390 | ], 4391 | "url": "http://www.nixstats.com/bot.html" 4392 | }, 4393 | { 4394 | "pattern": "Atom Feed Robot", 4395 | "addition_date": "2020/03/16", 4396 | "instances": [ 4397 | "RSSMicro.com RSS/Atom Feed Robot" 4398 | ], 4399 | "url": "https://rssmicro.com" 4400 | }, 4401 | { 4402 | "pattern": "[Cc]urebot", 4403 | "addition_date": "2020/03/16", 4404 | "instances": [ 4405 | "Curebot/1.0", 4406 | "curebot-feed-fetcher" 4407 | ] 4408 | }, 4409 | { 4410 | "pattern": "PagePeeker\\/", 4411 | "addition_date": "2020/03/16", 4412 | "instances": [ 4413 | "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36 (compatible; PagePeeker/3.0; +https://pagepeeker.com/robots/)" 4414 | ], 4415 | "url": "https://pagepeeker.com/robots/" 4416 | }, 4417 | { 4418 | "pattern": "Vigil\\/", 4419 | "addition_date": "2020/03/16", 4420 | "instances": [ 4421 | "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)" 4422 | ], 4423 | "url": "http://vigil-app.com/bot.html" 4424 | }, 4425 | { 4426 | "pattern": "rssbot\\/", 4427 | "addition_date": "2020/03/16", 4428 | "instances": [ 4429 | "rssbot/1.4.3 (+https://t.me/RustRssBot)" 4430 | ], 4431 | "url": "https://t.me/RustRssBot" 4432 | }, 4433 | { 4434 | "pattern": "startmebot\\/", 4435 | "addition_date": "2020/03/16", 4436 | "instances": [ 4437 | "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)" 4438 | ], 4439 | "url": "https://start.me/bot" 4440 | }, 4441 | { 4442 | "pattern": "JobboerseBot", 4443 | "addition_date": "2020/03/16", 4444 | "instances": [ 4445 | "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0" 4446 | ], 4447 | "url": "http://www.jobboerse.com/bot.htm" 4448 | }, 4449 | { 4450 | "pattern": "seewithkids", 4451 | "addition_date": "2020/03/16", 4452 | "instances": [ 4453 | "http://seewithkids.com/bot" 4454 | ], 4455 | "url": "http://seewithkids.com/bot" 4456 | }, 4457 | { 4458 | "pattern": "NINJA bot", 4459 | "addition_date": "2020/03/16", 4460 | "instances": [ 4461 | "NINJA bot" 4462 | ] 4463 | }, 4464 | { 4465 | "pattern": "Cutbot", 4466 | "addition_date": "2020/03/16", 4467 | "instances": [ 4468 | "Cutbot; 1.5; http://cutbot.net/" 4469 | ], 4470 | "url": "http://cutbot.net/" 4471 | }, 4472 | { 4473 | "pattern": "BublupBot", 4474 | "addition_date": "2020/03/16", 4475 | "instances": [ 4476 | "BublupBot (+https://www.bublup.com/bublup-bot.html)" 4477 | ], 4478 | "url": "https://www.bublup.com/bublup-bot.html" 4479 | }, 4480 | { 4481 | "pattern": "BrandONbot", 4482 | "addition_date": "2020/03/16", 4483 | "instances": [ 4484 | "BrandONbot (http://brandonmedia.net)" 4485 | ], 4486 | "url": "http://brandonmedia.net" 4487 | }, 4488 | { 4489 | "pattern": "RidderBot", 4490 | "addition_date": "2020/03/16", 4491 | "instances": [ 4492 | "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)", 4493 | "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321" 4494 | ], 4495 | "url": "http://brandonmedia.net" 4496 | }, 4497 | { 4498 | "pattern": "Taboolabot", 4499 | "addition_date": "2020/03/16", 4500 | "instances": [ 4501 | "Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com)" 4502 | ], 4503 | "url": "http://www.taboola.com" 4504 | }, 4505 | { 4506 | "pattern": "Dubbotbot", 4507 | "addition_date": "2020/03/16", 4508 | "instances": [ 4509 | "Mozilla/5.0 (compatible; Dubbotbot/0.2; +http://dubbot.com)" 4510 | ], 4511 | "url": "http://dubbot.com" 4512 | }, 4513 | { 4514 | "pattern": "FindITAnswersbot", 4515 | "addition_date": "2020/03/16", 4516 | "instances": [ 4517 | "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)" 4518 | ], 4519 | "url": "http://search.it-influentials.com/bot.htm" 4520 | }, 4521 | { 4522 | "pattern": "infoobot", 4523 | "addition_date": "2020/03/16", 4524 | "instances": [ 4525 | "infoobot/0.1 (https://www.infoo.nl/bot.html)" 4526 | ], 4527 | "url": "https://www.infoo.nl/bot.html" 4528 | }, 4529 | { 4530 | "pattern": "Refindbot", 4531 | "addition_date": "2020/03/16", 4532 | "instances": [ 4533 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 (Refindbot/1.0)" 4534 | ], 4535 | "url": "https://refind.com/about" 4536 | }, 4537 | { 4538 | "pattern": "BlogTraffic\\/\\d\\.\\d+ Feed-Fetcher", 4539 | "addition_date": "2020/03/16", 4540 | "instances": [ 4541 | "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)" 4542 | ], 4543 | "url": "http://www.blogtraffic.de/rss-bot.html" 4544 | }, 4545 | { 4546 | "pattern": "SeobilityBot", 4547 | "addition_date": "2020/03/16", 4548 | "instances": [ 4549 | "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)" 4550 | ], 4551 | "url": "https://www.seobility.net/sites/bot.html" 4552 | }, 4553 | { 4554 | "pattern": "Cincraw", 4555 | "addition_date": "2020/03/16", 4556 | "instances": [ 4557 | "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" 4558 | ], 4559 | "url": "http://cincrawdata.net/bot/" 4560 | }, 4561 | { 4562 | "pattern": "Dragonbot", 4563 | "addition_date": "2020/03/16", 4564 | "instances": [ 4565 | "Mozilla/5.0 (Windows NT 6.1; rv:34.0) Gecko/20100101 Firefox/34.0; Dragonbot; http://www.dragonmetrics.com" 4566 | ], 4567 | "url": "http://www.dragonmetrics.com" 4568 | }, 4569 | { 4570 | "pattern": "VoluumDSP-content-bot", 4571 | "addition_date": "2020/03/16", 4572 | "instances": [ 4573 | "Mozilla/5.0 (compatible; VoluumDSP-content-bot/2.0; +dsp-dev@codewise.com)" 4574 | ], 4575 | "url": "https://codewise.com" 4576 | }, 4577 | { 4578 | "pattern": "FreshRSS", 4579 | "addition_date": "2020/03/16", 4580 | "instances": [ 4581 | "FreshRSS/1.11.2 (Linux; https://freshrss.org) like Googlebot" 4582 | ], 4583 | "url": "https://freshrss.org" 4584 | }, 4585 | { 4586 | "pattern": "BitBot", 4587 | "addition_date": "2020/03/16", 4588 | "instances": [ 4589 | "Mozilla/5.0 (compatible; BitBot/v1.19.0; +https://bitbot.dev)" 4590 | ], 4591 | "url": "https://bitbot.dev" 4592 | }, 4593 | { 4594 | "pattern": "^PHP-Curl-Class", 4595 | "addition_date": "2020/12/10", 4596 | "instances": [ 4597 | "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.2.24 curl/7.61.1", 4598 | "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.19 curl/7.66.0", 4599 | "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.3.23 curl/7.66.0", 4600 | "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.7 curl/7.69.1", 4601 | "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.9 curl/7.69.1", 4602 | "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.10 curl/7.69.1", 4603 | "PHP-Curl-Class/4.13.0 (+https://github.com/php-curl-class/php-curl-class) PHP/7.4.11 curl/7.69.1" 4604 | ], 4605 | "url": "https://github.com/php-curl-class/php-curl-class" 4606 | }, 4607 | { 4608 | "pattern": "Google-Certificates-Bridge", 4609 | "addition_date": "2020/12/23", 4610 | "instances": [ 4611 | "Google-Certificates-Bridge" 4612 | ] 4613 | }, 4614 | { 4615 | "pattern": "centurybot", 4616 | "addition_date": "2022/04/26", 4617 | "instances": [ 4618 | "Mozilla/5.0 (compatible; Go-http-client/1.1; +centurybot9@gmail.com)" 4619 | ], 4620 | "url": "unknown" 4621 | }, 4622 | { 4623 | "pattern": "Viber", 4624 | "addition_date": "2021/04/27", 4625 | "instances": [ 4626 | "Viber" 4627 | ], 4628 | "url": "https://www.viber.com/" 4629 | }, 4630 | { 4631 | "pattern": "e\\.ventures Investment Crawler", 4632 | "addition_date": "2021/06/05", 4633 | "instances": [ 4634 | "e.ventures Investment Crawler (eventures.vc)" 4635 | ] 4636 | }, 4637 | { 4638 | "pattern": "evc-batch", 4639 | "addition_date": "2021/06/07", 4640 | "instances": [ 4641 | "Mozilla/5.0 (compatible; evc-batch/2.0)" 4642 | ] 4643 | }, 4644 | { 4645 | "pattern": "PetalBot", 4646 | "addition_date": "2021/06/07", 4647 | "instances": [ 4648 | "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", 4649 | "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" 4650 | ], 4651 | "url": "https://webmaster.petalsearch.com/site/petalbot" 4652 | }, 4653 | { 4654 | "pattern": "virustotal", 4655 | "addition_date": "2021/09/22", 4656 | "instances": [ 4657 | "Mozilla\/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) AppEngine-Google; (+http:\/\/code.google.com\/appengine; appid: s~virustotalcloud)", 4658 | "AppEngine-Google; (+http:\/\/code.google.com\/appengine; appid: s~virustotalcloud)" 4659 | ], 4660 | "url": "https://www.virustotal.com/gui/home/url" 4661 | }, 4662 | { 4663 | "pattern": "(^| )PTST\\/", 4664 | "addition_date": "2021/12/05", 4665 | "instances": [ 4666 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 PTST/211202.211915", 4667 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0 PTST/211202.211915" 4668 | ], 4669 | "url": "https://www.webpagetest.org" 4670 | }, 4671 | { 4672 | "pattern": "minicrawler", 4673 | "addition_date": "2022/01/12", 4674 | "instances": [ 4675 | "Testomatobot/1.0 (Linux x86_64; +https://www.testomato.com/testomatobot) minicrawler/5.2.2" 4676 | ], 4677 | "url": "https://www.testomato.com/bot" 4678 | }, 4679 | { 4680 | "pattern": "Cookiebot", 4681 | "addition_date": "2022/01/23", 4682 | "url": "https://www.cookiebot.com/", 4683 | "instances": [ 4684 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Cookiebot/1.0; +http://cookiebot.com/) Chrome/97.0.4692.71 Safari/537.36" 4685 | ] 4686 | }, 4687 | { 4688 | "pattern": "trovitBot", 4689 | "addition_date": "2022/06/08", 4690 | "url": "http://www.trovit.com/bot.html", 4691 | "instances": [ 4692 | "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)" 4693 | ] 4694 | }, 4695 | { 4696 | "pattern": "seostar\\.co", 4697 | "addition_date": "2022/08/04", 4698 | "url": "https://seostar.co/robot/", 4699 | "instances": [ 4700 | "Mozilla/5.0 (compatible; Adsbot/3.1; +https://seostar.co/robot/)" 4701 | ] 4702 | }, 4703 | { 4704 | "pattern": "IonCrawl", 4705 | "addition_date": "2022/08/04", 4706 | "url": "https://www.ionos.de/terms-gtc/faq-crawler-en", 4707 | "instances": [ 4708 | "IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)" 4709 | ] 4710 | }, 4711 | { 4712 | "pattern": "Uptime-Kuma", 4713 | "addition_date": "2022/10/17", 4714 | "url": "https://uptime.kuma.pet/", 4715 | "instances": [ 4716 | "Uptime-Kuma/1.18.0" 4717 | ] 4718 | }, 4719 | { 4720 | "pattern": "SeekportBot", 4721 | "addition_date": "2022/10/17", 4722 | "url": "https://bot.seekport.com", 4723 | "instances": [ 4724 | "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)" 4725 | ] 4726 | }, 4727 | { 4728 | "pattern": "FreshpingBot", 4729 | "addition_date": "2022/10/17", 4730 | "url": "https://www.freshworks.com/website-monitoring/", 4731 | "instances": [ 4732 | "FreshpingBot/1.0 (+https://freshping.io/)" 4733 | ] 4734 | }, 4735 | { 4736 | "pattern": "Feedbin", 4737 | "addition_date": "2022/11/05", 4738 | "url": "https://feedbin.com/", 4739 | "instances": [ 4740 | "Feedbin feed-id:2005098 - 2 subscribers" 4741 | ] 4742 | }, 4743 | { 4744 | "pattern": "CriteoBot", 4745 | "addition_date": "2022/11/13", 4746 | "url": "https://www.criteo.com/", 4747 | "instances": [ 4748 | "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)" 4749 | ] 4750 | }, 4751 | { 4752 | "pattern": "Snap URL Preview Service", 4753 | "addition_date": "2022/11/13", 4754 | "url": "https://snap.com/", 4755 | "instances": [ 4756 | "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" 4757 | ] 4758 | }, 4759 | { 4760 | "pattern": "Better Uptime Bot", 4761 | "addition_date": "2022/11/13", 4762 | "url": "https://betteruptime.com/", 4763 | "instances": [ 4764 | "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" 4765 | ] 4766 | }, 4767 | { 4768 | "pattern": "RuxitSynthetic", 4769 | "addition_date": "2023/02/16", 4770 | "url": "https://www.dynatrace.com/support/help/platform-modules/digital-experience/synthetic-monitoring/browser-monitors/configure-browser-monitors#expand--default-user-agent", 4771 | "instances" : ["RuxitSynthetic/1.0"] 4772 | }, 4773 | { 4774 | "pattern": "Google-Read-Aloud", 4775 | "addition_date": "2023/02/16", 4776 | "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers", 4777 | "instances": [ 4778 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)", 4779 | "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 (compatible; Google-Read-Aloud; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)" 4780 | ] 4781 | }, 4782 | { 4783 | "pattern": "Valve\\/Steam", 4784 | "addition_date": "2023/05/24", 4785 | "instances": [ 4786 | "Valve/Steam HTTP Client 1.0 (SteamChatURLLookup)" 4787 | ] 4788 | }, 4789 | { 4790 | "pattern": "OdklBot\\/", 4791 | "addition_date": "2023/05/24", 4792 | "instances": [ 4793 | "OdklBot/1.0 (share@odnoklassniki.ru)", 4794 | "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)" 4795 | ], 4796 | "url": "https://odnoklassniki.ru/" 4797 | }, 4798 | { 4799 | "pattern": "GPTBot", 4800 | "addition_date": "2023/08/09", 4801 | "instances": [ 4802 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)" 4803 | ], 4804 | "url": "https://platform.openai.com/docs/gptbot" 4805 | }, 4806 | { 4807 | "pattern": "YandexRenderResourcesBot\\/", 4808 | "addition_date": "2023/08/16", 4809 | "instances": [ 4810 | "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0" 4811 | ], 4812 | "url": "http://yandex.com/bots" 4813 | }, 4814 | { 4815 | "pattern": "LightspeedSystemsCrawler", 4816 | "addition_date": "2023/08/16", 4817 | "instances": [ 4818 | "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US" 4819 | ] 4820 | }, 4821 | { 4822 | "pattern": "ev-crawler\\/", 4823 | "addition_date": "2023/08/16", 4824 | "instances": [ 4825 | "Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)" 4826 | ], 4827 | "url": "https://headline.com/legal/crawler" 4828 | }, 4829 | { 4830 | "pattern": "BitSightBot\\/", 4831 | "addition_date": "2023/08/16", 4832 | "instances": [ 4833 | "Mozilla/5.0 (compatible; BitSightBot/1.0)" 4834 | ], 4835 | "url": "https://www.bitsight.com" 4836 | }, 4837 | { 4838 | "pattern": "woorankreview\\/", 4839 | "addition_date": "2023/08/16", 4840 | "instances": [ 4841 | "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)", 4842 | "Mozilla/5.0 (compatible; woorankreview/2.0; +https://www.woorank.com/)" 4843 | ], 4844 | "url": "https://www.woorank.com/" 4845 | }, 4846 | { 4847 | "pattern": "Google-Safety", 4848 | "addition_date": "2023/08/17", 4849 | "instances": [ 4850 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Mobile Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", 4851 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)", 4852 | "Google-Safety" 4853 | ], 4854 | "url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers" 4855 | }, 4856 | { 4857 | "pattern": "AwarioBot", 4858 | "addition_date": "2023/08/23", 4859 | "instances": [ 4860 | "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)" 4861 | ], 4862 | "url": "https://awario.com/bots.html" 4863 | }, 4864 | { 4865 | "pattern": "DataForSeoBot", 4866 | "addition_date": "2023/08/23", 4867 | "instances": [ 4868 | "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)" 4869 | ], 4870 | "url": "https://dataforseo.com/dataforseo-bot" 4871 | }, 4872 | { 4873 | "pattern": "Linespider", 4874 | "addition_date": "2023/08/24", 4875 | "instances": [ 4876 | "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", 4877 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36" 4878 | ], 4879 | "url": "https://help2.line.me/linesearchbot/web/?contentId=50006055&lang=en" 4880 | }, 4881 | { 4882 | "pattern": "WellKnownBot", 4883 | "addition_date": "2023/08/29", 4884 | "instances": [ 4885 | "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)" 4886 | ], 4887 | "url": "https://well-known.dev/about/#bot)" 4888 | }, 4889 | { 4890 | "pattern": "A Patent Crawler", 4891 | "addition_date": "2023/08/29", 4892 | "instances": [ 4893 | "E. Orliac, G. Fourestey/2.3 (A Patent Crawler; http://scitas.epfl.ch/; etienne.orliac@epfl.ch, gilles.fourestey@epfl.ch)" 4894 | ], 4895 | "url": "http://scitas.epfl.ch/" 4896 | }, 4897 | { 4898 | "pattern": "StractBot", 4899 | "addition_date": "2023/09/06", 4900 | "instances": [ 4901 | "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)" 4902 | ], 4903 | "url": "https://trystract.com/webmasters" 4904 | }, 4905 | { 4906 | "pattern": "search\\.marginalia\\.nu", 4907 | "addition_date": "2023/09/08", 4908 | "instances": [ 4909 | "search.marginalia.nu" 4910 | ], 4911 | "url": "https://search.marginalia.nu" 4912 | }, 4913 | { 4914 | "pattern": "YouBot", 4915 | "addition_date": "2023/09/08", 4916 | "instances": [ 4917 | "YouBot (+http://www.you.com)" 4918 | ], 4919 | "url": "https://you.com/" 4920 | }, 4921 | { 4922 | "pattern": "Nicecrawler", 4923 | "addition_date": "2023/09/08", 4924 | "instances": [ 4925 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Nicecrawler/1.1; +http://www.nicecrawler.com/) Chrome/90.0.4430.97 Safari/537.36" 4926 | ], 4927 | "url": "http://www.nicecrawler.com/" 4928 | }, 4929 | { 4930 | "pattern": "Neevabot", 4931 | "addition_date": "2023/09/08", 4932 | "instances": [ 4933 | "Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)" 4934 | ], 4935 | "url": "https://neeva.com/neevabot" 4936 | }, 4937 | { 4938 | "pattern": "BrightEdge Crawler", 4939 | "addition_date": "2023/09/08", 4940 | "instances": [ 4941 | "BrightEdge Crawler/1.0 (crawler@brightedge.com)" 4942 | ], 4943 | "url": "https://www.brightedge.com/" 4944 | }, 4945 | { 4946 | "pattern": "SiteCheckerBotCrawler", 4947 | "addition_date": "2023/09/08", 4948 | "instances": [ 4949 | "SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)" 4950 | ], 4951 | "url": "http://sitechecker.pro" 4952 | }, 4953 | { 4954 | "pattern": "TombaPublicWebCrawler", 4955 | "addition_date": "2023/09/08", 4956 | "instances": [ 4957 | "Mozilla/5.0 (compatible; TombaPublicWebCrawler/1.0; +https://tombascraper.com)" 4958 | ], 4959 | "url": "https://tombascraper.com" 4960 | }, 4961 | { 4962 | "pattern": "CrawlyProjectCrawler", 4963 | "addition_date": "2023/09/08", 4964 | "instances": [ 4965 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; crawlyproject@digitaldragon.dev +https://crawlyproject.digitaldragon.dev/)" 4966 | ], 4967 | "url": "https://crawlyproject.digitaldragon.dev/" 4968 | }, 4969 | { 4970 | "pattern": "KomodiaBot", 4971 | "addition_date": "2023/09/08", 4972 | "instances": [ 4973 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0" 4974 | ], 4975 | "url": "http://www.komodia.com/newwiki/index.php/URL_server_crawler" 4976 | }, 4977 | { 4978 | "pattern": "KStandBot", 4979 | "addition_date": "2023/09/08", 4980 | "instances": [ 4981 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://url-classification.io/wiki/index.php?title=URL_server_crawler) KStandBot/1.0" 4982 | ], 4983 | "url": "http://url-classification.io" 4984 | }, 4985 | { 4986 | "pattern": "CISPA Webcrawler", 4987 | "addition_date": "2023/09/08", 4988 | "instances": [ 4989 | "CISPA Webcrawler (https://vuln-notify-checker.cispa.saarland)" 4990 | ], 4991 | "url": "https://vuln-notify-checker.cispa.saarland" 4992 | }, 4993 | { 4994 | "pattern": "MTRobot", 4995 | "addition_date": "2023/09/08", 4996 | "instances": [ 4997 | "MTRobot/0.2 (Metrics Tools Analytics Crawler; https://metrics-tools.de/robot.html; crawler@metrics-tools.de)" 4998 | ], 4999 | "url": "https://metrics-tools.de/robot.html" 5000 | }, 5001 | { 5002 | "pattern": "hyscore.io", 5003 | "addition_date": "2023/09/08", 5004 | "instances": [ 5005 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1. 4 (compatible; HyScore/1.0; +https://hyscore.io/crawler/)" 5006 | ], 5007 | "url": "https://hyscore.io/crawler/" 5008 | }, 5009 | { 5010 | "pattern": "AlexandriaOrgBot", 5011 | "addition_date": "2023/09/08", 5012 | "instances": [ 5013 | "Mozilla/5.0 (Linux) (compatible; AlexandriaOrgBot/1.0; +https://www.alexandria.org/bot.html)" 5014 | ], 5015 | "url": "https://www.alexandria.org/bot.html" 5016 | }, 5017 | { 5018 | "pattern": "2ip bot", 5019 | "addition_date": "2023/09/08", 5020 | "instances": [ 5021 | "2ip bot/1.1 (+http://2ip.io)" 5022 | ], 5023 | "url": "http://2ip.io" 5024 | }, 5025 | { 5026 | "pattern": "Yellowbrandprotectionbot", 5027 | "addition_date": "2023/09/08", 5028 | "instances": [ 5029 | "Mozilla/5.0 (compatible; Yellowbrandprotectionbot/1.0; +https://www.yellowbp.com/bot.html)" 5030 | ], 5031 | "url": "https://www.yellowbp.com/bot.html" 5032 | }, 5033 | { 5034 | "pattern": "SEOlizer", 5035 | "addition_date": "2023/09/08", 5036 | "instances": [ 5037 | "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" 5038 | ], 5039 | "url": "https://www.seolizer.de/bot.html" 5040 | }, 5041 | { 5042 | "pattern": "vuhuvBot", 5043 | "addition_date": "2023/09/08", 5044 | "instances": [ 5045 | "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)" 5046 | ], 5047 | "url": "http://vuhuv.com/bot.html" 5048 | }, 5049 | { 5050 | "pattern": "INETDEX-BOT", 5051 | "addition_date": "2023/09/08", 5052 | "instances": [ 5053 | "INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/bot.html)" 5054 | ], 5055 | "url": "https://inetdex.com/bot.html" 5056 | }, 5057 | { 5058 | "pattern": "Synapse", 5059 | "addition_date": "2023/09/08", 5060 | "instances": [ 5061 | "Synapse (bot; +https://github.com/matrix-org/synapse)" 5062 | ], 5063 | "url": "https://github.com/matrix-org/synapse" 5064 | }, 5065 | { 5066 | "pattern": "t3versionsBot", 5067 | "addition_date": "2023/09/08", 5068 | "instances": [ 5069 | "Mozilla/5.0 (compatible; t3versionsBot/1.0; +https://www.t3versions.com/bot)" 5070 | ], 5071 | "url": "https://www.t3versions.com/bot" 5072 | }, 5073 | { 5074 | "pattern": "deepnoc", 5075 | "addition_date": "2023/09/08", 5076 | "instances": [ 5077 | "deepnoc - https://deepnoc.com/bot" 5078 | ], 5079 | "url": "https://deepnoc.com/bot" 5080 | }, 5081 | { 5082 | "pattern": "Cocolyzebot", 5083 | "addition_date": "2023/09/08", 5084 | "instances": [ 5085 | "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)" 5086 | ], 5087 | "url": "https://cocolyze.com/bot" 5088 | }, 5089 | { 5090 | "pattern": "hypestat", 5091 | "addition_date": "2023/09/08", 5092 | "instances": [ 5093 | "Mozilla/5.0 (compatible; hypestat/1.0; +https://hypestat.com/bot)" 5094 | ], 5095 | "url": "https://hypestat.com/bot" 5096 | }, 5097 | { 5098 | "pattern": "ReverseEngineeringBot", 5099 | "addition_date": "2023/09/08", 5100 | "instances": [ 5101 | "Mozilla/5.0 (compatible; ReverseEngineeringBot/0.1; +https://torus.company/bot.html)" 5102 | ], 5103 | "url": "https://torus.company/bot.html" 5104 | }, 5105 | { 5106 | "pattern": "sempi.tech", 5107 | "addition_date": "2023/09/08", 5108 | "instances": [ 5109 | "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)" 5110 | ], 5111 | "url": "http://sempi.tech/bot.html" 5112 | }, 5113 | { 5114 | "pattern": "Iframely", 5115 | "addition_date": "2023/09/08", 5116 | "instances": [ 5117 | "Iframely/1.3.1 (+https://iframely.com/docs/about) Atlassian" 5118 | ], 5119 | "url": "https://iframely.com/docs/about" 5120 | }, 5121 | { 5122 | "pattern": "MetaInspector", 5123 | "addition_date": "2023/09/08", 5124 | "instances": [ 5125 | "MetaInspector/5.6.0 (+https://github.com/jaimeiniesta/metainspector)" 5126 | ], 5127 | "url": "https://github.com/jaimeiniesta/metainspector" 5128 | }, 5129 | { 5130 | "pattern": "node-fetch", 5131 | "addition_date": "2023/09/08", 5132 | "instances": [ 5133 | "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)" 5134 | ], 5135 | "url": "https://github.com/bitinn/node-fetch" 5136 | }, 5137 | { 5138 | "pattern": "lkxscan", 5139 | "addition_date": "2023/09/08", 5140 | "instances": [ 5141 | "lkxscan/v0.1.0 (+https://leakix.net) l9explore/v1.0.0 (+https://github.com/LeakIX/l9explore)" 5142 | ], 5143 | "url": "https://github.com/LeakIX/l9explore" 5144 | }, 5145 | { 5146 | "pattern": "python-opengraph", 5147 | "addition_date": "2023/09/08", 5148 | "instances": [ 5149 | "python-opengraph-jaywink/0.2.0 (+https://github.com/jaywink/python-opengraph)" 5150 | ], 5151 | "url": "https://github.com/jaywink/python-opengraph" 5152 | }, 5153 | { 5154 | "pattern": "OpenGraphCheck", 5155 | "addition_date": "2023/09/08", 5156 | "instances": [ 5157 | "OpenGraphCheck/2.1 (+https://opengraphcheck.com)" 5158 | ], 5159 | "url": "https://opengraphcheck.com" 5160 | }, 5161 | { 5162 | "pattern": "developers.google.com\\/\\+\\/web\\/snippet", 5163 | "addition_date": "2023/09/08", 5164 | "instances": [ 5165 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)", 5166 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/" 5167 | ], 5168 | "url": "https://developers.google.com/+/web/snippet" 5169 | }, 5170 | { 5171 | "pattern": "SenutoBot", 5172 | "addition_date": "2023/09/08", 5173 | "instances": [ 5174 | "SenutoBot/1.0 (compatible; SenutoBot/1.0; +https://www.senuto.com/)" 5175 | ], 5176 | "url": "https://www.senuto.com" 5177 | }, 5178 | { 5179 | "pattern": "MaCoCu", 5180 | "addition_date": "2023/09/08", 5181 | "instances": [ 5182 | "Mozilla/5.0 (compatible; MaCoCu; +https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/)" 5183 | ], 5184 | "url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data" 5185 | }, 5186 | { 5187 | "pattern": "NewsBlur", 5188 | "addition_date": "2023/09/08", 5189 | "instances": [ 5190 | "NewsBlur Feed Fetcher - 1 subscriber - http://www.newsblur.com/site/0000000/webpage (Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.1 Safari/605.1.15)" 5191 | ], 5192 | "url": "http://www.newsblur.com" 5193 | }, 5194 | { 5195 | "pattern": "inoreader", 5196 | "addition_date": "2023/09/08", 5197 | "instances": [ 5198 | "Mozilla/5.0 (compatible; inoreader.com; 1 subscribers)" 5199 | ], 5200 | "url": "http://inoreader.com" 5201 | }, 5202 | { 5203 | "pattern": "NetSystemsResearch", 5204 | "addition_date": "2023/09/08", 5205 | "instances": [ 5206 | "NetSystemsResearch studies the availability of various services across the internet. Our website is netsystemsresearch.com" 5207 | ], 5208 | "url": "http://netsystemsresearch.com" 5209 | }, 5210 | { 5211 | "pattern": "PageThing", 5212 | "addition_date": "2023/09/08", 5213 | "instances": [ 5214 | "PageThing http://pagething.com curl www" 5215 | ], 5216 | "url": "http://pagething.com" 5217 | } 5218 | ] 5219 | -------------------------------------------------------------------------------- /custom.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pattern": "^[\\w]+$" 4 | }, 5 | { 6 | "pattern": "java(?!;)" 7 | }, 8 | { 9 | "pattern": "((?