├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── NetCrawlerDetect ├── .gitignore ├── NetCrawlerDetect.Tests │ ├── NetCrawlerDetect.Tests.csproj │ ├── Tests.cs │ ├── crawlers.txt │ └── devices.txt ├── NetCrawlerDetect.sln └── NetCrawlerDetect │ ├── CrawlerDetect.cs │ ├── Fixtures │ ├── AbstractProvider.cs │ ├── Crawlers.cs │ ├── Exclusions.cs │ └── Headers.cs │ └── NetCrawlerDetect.csproj └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .DS_Store 3 | NetCrawlerDetect/.vs/* -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: required 3 | language: csharp 4 | mono: none 5 | dotnet: 2.0.0 6 | script: 7 | - cd NetCrawlerDetect 8 | - dotnet restore 9 | - dotnet build -c Release 10 | - cd NetCrawlerDetect.Tests 11 | - dotnet test 12 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ### Contributing 2 | If you find a bot/spider/crawler user agent that NetCrawlerDetect fails to detect, please submit a pull request with the regex pattern added to the `_data` List in `Fixtures/Crawlers.cs` and add the failing user agent to `NetCrawlerDetect.Tests/crawlers.txt`. 3 | 4 | Please also consider submitting a pull request to our parent project [(https://github.com/JayBizzle/Crawler-Detect)](https://github.com/JayBizzle/Crawler-Detect). :) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Graham "Gee" Plumb 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NetCrawlerDetect/.gitignore: -------------------------------------------------------------------------------- 1 | # Autosave files 2 | *~ 3 | 4 | # build 5 | [Oo]bj/ 6 | [Bb]in/ 7 | packages/ 8 | TestResults/ 9 | 10 | # globs 11 | Makefile.in 12 | *.DS_Store 13 | *.sln.cache 14 | *.suo 15 | *.cache 16 | *.pidb 17 | *.userprefs 18 | *.usertasks 19 | config.log 20 | config.make 21 | config.status 22 | aclocal.m4 23 | install-sh 24 | autom4te.cache/ 25 | *.user 26 | *.tar.gz 27 | tarballs/ 28 | test-results/ 29 | Thumbs.db 30 | 31 | # Mac bundle stuff 32 | *.dmg 33 | *.app 34 | 35 | # resharper 36 | *_Resharper.* 37 | *.Resharper 38 | 39 | # dotCover 40 | *.dotCover 41 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect.Tests/NetCrawlerDetect.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.0 5 | false 6 | 1.2.113 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | PreserveNewest 21 | 22 | 23 | PreserveNewest 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect.Tests/Tests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Net; 4 | using System.Text.RegularExpressions; 5 | using Xunit; 6 | 7 | namespace NetCrawlerDetect.Tests 8 | { 9 | /// 10 | /// Ported unit tests with some additional coverage 11 | /// 12 | public class Tests 13 | { 14 | CrawlerDetect _detector; 15 | 16 | 17 | /// 18 | /// Test setup 19 | /// 20 | public Tests() 21 | { 22 | _detector = new CrawlerDetect(); 23 | } 24 | 25 | 26 | [Fact] 27 | public void UserAgentsAreBots() 28 | { 29 | bool result = false; 30 | 31 | using (var filestream = File.OpenRead(@"crawlers.txt")) 32 | { 33 | using (var reader = new StreamReader(filestream)) 34 | { 35 | var entry = reader.ReadLine(); 36 | result = _detector.IsCrawler(entry); 37 | 38 | Assert.True(result, $"Misidentified bot: {entry}"); 39 | } 40 | } 41 | } 42 | 43 | 44 | [Fact] 45 | public void UserAgentsAreDevices() 46 | { 47 | bool result = false; 48 | 49 | using (var filestream = File.OpenRead(@"devices.txt")) 50 | { 51 | using (var reader = new StreamReader(filestream)) 52 | { 53 | var entry = reader.ReadLine(); 54 | result = _detector.IsCrawler(entry); 55 | 56 | Assert.False(result, $"Misidentified device: {entry}"); 57 | } 58 | } 59 | } 60 | 61 | 62 | [Fact] 63 | public void ReturnCorrectlyMatchedBotName() 64 | { 65 | var result = _detector.IsCrawler("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)"); 66 | Assert.True(result, "Yahoo Ad monitoring IS a bot!"); 67 | Assert.Equal("monitoring", _detector.Matches[0].Value); 68 | } 69 | 70 | 71 | [Fact] 72 | public void NoMatchesWhenNoBotMatched() 73 | { 74 | var result = _detector.IsCrawler("nothing to see here!"); 75 | Assert.False(result); 76 | Assert.Equal(0, _detector.Matches.Count); 77 | } 78 | 79 | 80 | [Fact] 81 | public void EmptyUserAgent() 82 | { 83 | Assert.Throws(typeof(ArgumentException), () => 84 | { 85 | var result = _detector.IsCrawler(" \t"); 86 | }); 87 | } 88 | 89 | 90 | [Fact] 91 | public void NullAllTheWay() 92 | { 93 | var cd = new CrawlerDetect(null, null); 94 | 95 | Assert.Throws(typeof(ArgumentException), () => 96 | { 97 | var result = cd.IsCrawler(null); 98 | }); 99 | } 100 | 101 | 102 | [Fact] 103 | public void EmptyAllTheWay() 104 | { 105 | var cd = new CrawlerDetect(null, string.Empty); 106 | 107 | Assert.Throws(typeof(ArgumentException), () => 108 | { 109 | var result = cd.IsCrawler(string.Empty); 110 | }); 111 | } 112 | 113 | 114 | [Fact] 115 | public void InferBotViaUserAgentHeader() 116 | { 117 | var headers = new WebHeaderCollection() 118 | { 119 | {"accept", "*/*"}, 120 | {"accept-encoding", "DEFLATE"}, 121 | {"cache-control", "no-cache"}, 122 | {"connection", "Keep-Alive"}, 123 | // {"from", "bingbot(at)microsoft.com"}, 124 | {"host", "www.test.com"}, 125 | {"pragma", "no-cache"}, 126 | {"user-agent", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"} 127 | }; 128 | 129 | var cd = new CrawlerDetect(headers); 130 | var result = cd.IsCrawler(); 131 | Assert.True(result); 132 | } 133 | 134 | 135 | [Fact] 136 | public void BotUserAgentPassedViaConstructor() 137 | { 138 | var cd = new CrawlerDetect(null, "Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) AppleWebKit (KHTML, like Gecko) Mobile (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)"); 139 | var result = cd.IsCrawler(); 140 | Assert.True(result); 141 | } 142 | 143 | 144 | [Fact] 145 | public void InferBotViaFromHeader() 146 | { 147 | var headers = new WebHeaderCollection() 148 | { 149 | {"accept", "*/*"}, 150 | {"accept-encoding", "DEFLATE"}, 151 | {"cache-control", "no-cache"}, 152 | {"connection", "Keep-Alive"}, 153 | {"from", "googlebot(at)googlebot.com"}, 154 | {"host", "www.test.com"}, 155 | {"pragma", "no-cache"}, 156 | {"user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36"} 157 | }; 158 | 159 | var cd = new CrawlerDetect(headers); 160 | var result = cd.IsCrawler(); 161 | Assert.True(result); 162 | } 163 | 164 | 165 | [Fact] 166 | public void NoRegexCollisions() 167 | { 168 | var crawlers = new Fixtures.Crawlers(); 169 | 170 | foreach (var key1 in crawlers.GetAll()) 171 | { 172 | foreach (var key2 in crawlers.GetAll()) 173 | { 174 | if (key1 == key2) 175 | continue; 176 | 177 | var regex = new Regex(key1); 178 | Assert.False(regex.IsMatch(key2)); 179 | } 180 | } 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2012 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetCrawlerDetect", "NetCrawlerDetect\NetCrawlerDetect.csproj", "{2537AACE-3138-47C5-8B02-8E7CA001DA28}" 5 | EndProject 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetCrawlerDetect.Tests", "NetCrawlerDetect.Tests\NetCrawlerDetect.Tests.csproj", "{BA60A81E-2293-4FC6-89E8-B44BBE8C20BA}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {2537AACE-3138-47C5-8B02-8E7CA001DA28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {2537AACE-3138-47C5-8B02-8E7CA001DA28}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {2537AACE-3138-47C5-8B02-8E7CA001DA28}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {2537AACE-3138-47C5-8B02-8E7CA001DA28}.Release|Any CPU.Build.0 = Release|Any CPU 18 | {BA60A81E-2293-4FC6-89E8-B44BBE8C20BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 19 | {BA60A81E-2293-4FC6-89E8-B44BBE8C20BA}.Debug|Any CPU.Build.0 = Debug|Any CPU 20 | {BA60A81E-2293-4FC6-89E8-B44BBE8C20BA}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {BA60A81E-2293-4FC6-89E8-B44BBE8C20BA}.Release|Any CPU.Build.0 = Release|Any CPU 22 | EndGlobalSection 23 | GlobalSection(MonoDevelopProperties) = preSolution 24 | description = A .net standard port of JayBizzle's CrawlerDetect project (https://github.com/JayBizzle/Crawler-Detect). 25 | version = 1.2.111 26 | Policies = $0 27 | $0.VersionControlPolicy = $1 28 | $0.DotNetNamingPolicy = $2 29 | $2.DirectoryNamespaceAssociation = PrefixedHierarchical 30 | $0.StandardHeader = $3 31 | $0.TextStylePolicy = $4 32 | $4.inheritsSet = null 33 | $4.scope = text/x-csharp 34 | $0.CSharpFormattingPolicy = $5 35 | $5.scope = text/x-csharp 36 | $0.TextStylePolicy = $6 37 | $6.FileWidth = 80 38 | $6.TabsToSpaces = True 39 | $6.scope = text/plain 40 | EndGlobalSection 41 | EndGlobal 42 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect/CrawlerDetect.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Net; 4 | using System.Text; 5 | using System.Text.RegularExpressions; 6 | 7 | using NetCrawlerDetect.Fixtures; 8 | 9 | namespace NetCrawlerDetect 10 | { 11 | /// 12 | /// Crawler Detect 13 | /// 14 | public class CrawlerDetect 15 | { 16 | /// 17 | /// The user-agent string to test 18 | /// 19 | protected string _userAgent = null; 20 | 21 | 22 | /// 23 | /// Headers that contain a user agent 24 | /// 25 | protected WebHeaderCollection _headers = new WebHeaderCollection(); 26 | 27 | 28 | /// 29 | /// Store regex matches 30 | /// 31 | protected MatchCollection _matches = null; 32 | 33 | 34 | /// 35 | /// Crawlers object 36 | /// 37 | protected Crawlers _crawlers = new Crawlers(); 38 | 39 | 40 | /// 41 | /// Exclusions object 42 | /// 43 | protected Exclusions _exclusions = new Exclusions(); 44 | 45 | 46 | /// 47 | /// Headers object 48 | /// 49 | protected Headers _uaHttpHeaders = new Headers(); 50 | 51 | 52 | /// 53 | /// A compilation of regex user agent snippets that belong to crawlers 54 | /// 55 | protected static Regex _compiledRegex = null; 56 | 57 | 58 | /// 59 | /// A compilation of regex user agent snippets to ignore 60 | /// 61 | protected static Regex _compiledExclusions = null; 62 | 63 | 64 | /// 65 | /// Expose any matches 66 | /// 67 | public MatchCollection Matches => _matches; 68 | 69 | 70 | /// 71 | /// Fetch the user agent 72 | /// 73 | public string UserAgent => _userAgent; 74 | 75 | 76 | /// 77 | /// Constructor 78 | /// 79 | public CrawlerDetect() 80 | : this(null, null) 81 | { 82 | } 83 | 84 | 85 | /// 86 | /// Constructor 87 | /// 88 | public CrawlerDetect(WebHeaderCollection headers, string userAgent = null) 89 | { 90 | // Crude way to get an empty match collection 91 | var regex = new Regex(@".", RegexOptions.Compiled); 92 | _matches = regex.Matches(""); 93 | 94 | if (_compiledRegex == null) 95 | _compiledRegex = CompileRegex(_crawlers.GetAll()); 96 | 97 | if (_compiledExclusions == null) 98 | _compiledExclusions = CompileRegex(_exclusions.GetAll()); 99 | 100 | SetHttpHeaders(headers); 101 | SetUserAgent(userAgent); 102 | } 103 | 104 | 105 | /// 106 | /// Compile the given list of expressions into a single Regex 107 | /// 108 | private Regex CompileRegex(IEnumerable expressions) 109 | { 110 | string patterns = "(" + string.Join("|", expressions) + ")"; 111 | return new Regex(patterns, RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); 112 | } 113 | 114 | 115 | /// 116 | /// Set the HTTP headers to inspect for a crawler user agent 117 | /// 118 | private void SetHttpHeaders(WebHeaderCollection headers) 119 | { 120 | // Bail if no headers passed in 121 | if (headers == null) 122 | return; 123 | 124 | // Only stash user agent HTTP headers 125 | foreach (var key in GetUserAgentHeaders()) 126 | { 127 | if (Array.Exists(headers.AllKeys, (x) => x.Equals(key, StringComparison.InvariantCultureIgnoreCase))) 128 | { 129 | _headers.Add(key, headers[key]); 130 | } 131 | } 132 | } 133 | 134 | 135 | /// 136 | /// Return user agent headers 137 | /// 138 | private IEnumerable GetUserAgentHeaders() 139 | { 140 | return _uaHttpHeaders.GetAll(); 141 | } 142 | 143 | 144 | /// 145 | /// Set the user agent string to evaluate 146 | /// 147 | private void SetUserAgent(string userAgent) 148 | { 149 | var result = userAgent; 150 | 151 | if (string.IsNullOrEmpty(userAgent)) 152 | { 153 | var builder = new StringBuilder(); 154 | 155 | // Only stash user agent HTTP headers 156 | foreach (var key in GetUserAgentHeaders()) 157 | { 158 | if (Array.Exists(_headers.AllKeys, (x) => x.Equals(key, StringComparison.InvariantCultureIgnoreCase))) 159 | { 160 | builder.Append(_headers[key]); 161 | builder.Append(" "); 162 | } 163 | } 164 | 165 | result = builder.ToString(); 166 | } 167 | 168 | _userAgent = result; 169 | } 170 | 171 | 172 | /// 173 | /// Check user agent string against the regex 174 | /// 175 | public bool IsCrawler(string userAgent = null) 176 | { 177 | var agent = userAgent ?? _userAgent; 178 | 179 | if (string.IsNullOrWhiteSpace(agent)) 180 | throw new ArgumentException("Cannot test a null or empty user agent!"); 181 | 182 | agent = _compiledExclusions.Replace(agent, ""); 183 | 184 | if (agent.Length == 0) 185 | return false; 186 | 187 | _matches = _compiledRegex.Matches(agent); 188 | 189 | return _matches.Count > 0; 190 | } 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect/Fixtures/AbstractProvider.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCrawlerDetect.Fixtures 4 | { 5 | /// 6 | /// Generic provider of data 7 | /// 8 | public abstract class AbstractProvider 9 | { 10 | /// 11 | /// The data 12 | /// 13 | protected List _data; 14 | 15 | 16 | /// 17 | /// Get an enumeration of the data 18 | /// 19 | public IEnumerable GetAll() 20 | { 21 | return _data; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect/Fixtures/Crawlers.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCrawlerDetect.Fixtures 4 | { 5 | /// 6 | /// A provider of crawler user-agent regex snippets 7 | /// 8 | public class Crawlers : AbstractProvider 9 | { 10 | /// 11 | /// Constructor 12 | /// 13 | public Crawlers() 14 | { 15 | // Collection of user-agent regex snippets 16 | _data = new List() 17 | { 18 | @" YLT", 19 | @"^Aether", 20 | @"^Amazon Simple Notification Service Agent$", 21 | @"^Amazon-Route53-Health-Check-Service", 22 | @"^b0t$", 23 | @"^bluefish ", 24 | @"^Calypso v\/", 25 | @"^COMODO DCV", 26 | @"^Corax", 27 | @"^DangDang", 28 | @"^DavClnt", 29 | @"^DHSH", 30 | @"^docker\/[0-9]", 31 | @"^Expanse", 32 | @"^FDM ", 33 | @"^git\/", 34 | @"^Goose\/", 35 | @"^Grabber", 36 | @"^Gradle\/", 37 | @"^HTTPClient\/", 38 | @"^HTTPing", 39 | @"^Java\/", 40 | @"^Jeode\/", 41 | @"^Jetty\/", 42 | @"^Mail\/", 43 | @"^Mget", 44 | @"^Microsoft URL Control", 45 | @"^Mikrotik\/", 46 | @"^Netlab360", 47 | @"^NG\/[0-9\.]", 48 | @"^NING\/", 49 | @"^npm\/", 50 | @"^Nuclei", 51 | @"^PHP-AYMAPI\/", 52 | @"^PHP\/", 53 | @"^pip\/", 54 | @"^pnpm\/", 55 | @"^RMA\/", 56 | @"^Ruby|Ruby\/[0-9]", 57 | @"^Swurl ", 58 | @"^TLS tester ", 59 | @"^twine\/", 60 | @"^ureq", 61 | @"^VSE\/[0-9]", 62 | @"^WordPress\.com", 63 | @"^XRL\/[0-9]", 64 | @"^ZmEu", 65 | @"008\/", 66 | @"13TABS", 67 | @"192\.comAgent", 68 | @"2GDPR\/", 69 | @"2ip\.ru", 70 | @"404enemy", 71 | @"7Siters", 72 | @"80legs", 73 | @"a3logics\.in", 74 | @"A6-Indexer", 75 | @"Abonti", 76 | @"Aboundex", 77 | @"aboutthedomain", 78 | @"Accoona-AI-Agent", 79 | @"acebookexternalhit\/", 80 | @"acoon", 81 | @"acrylicapps\.com\/pulp", 82 | @"Acunetix", 83 | @"AdAuth\/", 84 | @"adbeat", 85 | @"AddThis", 86 | @"ADmantX", 87 | @"AdminLabs", 88 | @"adressendeutschland", 89 | @"adreview\/", 90 | @"adscanner", 91 | @"adstxt-worker", 92 | @"Adstxtaggregator", 93 | @"adstxt\.com", 94 | @"Adyen HttpClient", 95 | @"AffiliateLabz\/", 96 | @"affilimate-puppeteer", 97 | @"agentslug", 98 | @"AHC", 99 | @"aihit", 100 | @"aiohttp\/", 101 | @"Airmail", 102 | @"akka-http\/", 103 | @"akula\/", 104 | @"alertra", 105 | @"alexa site audit", 106 | @"Alibaba\.Security\.Heimdall", 107 | @"Alligator", 108 | @"allloadin", 109 | @"AllSubmitter", 110 | @"alyze\.info", 111 | @"amagit", 112 | @"Anarchie", 113 | @"AndroidDownloadManager", 114 | @"Anemone", 115 | @"AngleSharp", 116 | @"annotate_google", 117 | @"Anthill", 118 | @"Anturis Agent", 119 | @"Ant\.com", 120 | @"AnyEvent-HTTP\/", 121 | @"Apache Ant\/", 122 | @"Apache Droid", 123 | @"Apache OpenOffice", 124 | @"Apache-HttpAsyncClient", 125 | @"Apache-HttpClient", 126 | @"ApacheBench", 127 | @"Apexoo", 128 | @"apimon\.de", 129 | @"APIs-Google", 130 | @"AportWorm\/", 131 | @"AppBeat\/", 132 | @"AppEngine-Google", 133 | @"AppleSyndication", 134 | @"Aprc\/[0-9]", 135 | @"Arachmo", 136 | @"arachnode", 137 | @"Arachnophilia", 138 | @"aria2", 139 | @"Arukereso", 140 | @"asafaweb", 141 | @"Asana\/", 142 | @"Ask Jeeves", 143 | @"AskQuickly", 144 | @"ASPSeek", 145 | @"Asterias", 146 | @"Astute", 147 | @"asynchttp", 148 | @"Attach", 149 | @"attohttpc", 150 | @"autocite", 151 | @"AutomaticWPTester", 152 | @"Autonomy", 153 | @"awin\.com", 154 | @"AWS Security Scanner", 155 | @"axios\/", 156 | @"a\.pr-cy\.ru", 157 | @"B-l-i-t-z-B-O-T", 158 | @"Backlink-Ceck", 159 | @"backlink-check", 160 | @"BacklinkHttpStatus", 161 | @"BackStreet", 162 | @"BackupLand", 163 | @"BackWeb", 164 | @"Bad-Neighborhood", 165 | @"Badass", 166 | @"baidu\.com", 167 | @"Bandit", 168 | @"basicstate", 169 | @"BatchFTP", 170 | @"Battleztar Bazinga", 171 | @"baypup\/", 172 | @"BazQux", 173 | @"BBBike", 174 | @"BCKLINKS", 175 | @"BDFetch", 176 | @"BegunAdvertising", 177 | @"Bewica-security-scan", 178 | @"Bidtellect", 179 | @"BigBozz", 180 | @"Bigfoot", 181 | @"biglotron", 182 | @"BingLocalSearch", 183 | @"BingPreview", 184 | @"binlar", 185 | @"biNu image cacher", 186 | @"Bitacle", 187 | @"Bitrix link preview", 188 | @"biz_Directory", 189 | @"BKCTwitterUnshortener\/", 190 | @"Black Hole", 191 | @"Blackboard Safeassign", 192 | @"BlackWidow", 193 | @"BlockNote\.Net", 194 | @"BlogBridge", 195 | @"Bloglines", 196 | @"Bloglovin", 197 | @"BlogPulseLive", 198 | @"BlogSearch", 199 | @"Blogtrottr", 200 | @"BlowFish", 201 | @"boitho\.com-dc", 202 | @"Boost\.Beast", 203 | @"BPImageWalker", 204 | @"Braintree-Webhooks", 205 | @"Branch Metrics API", 206 | @"Branch-Passthrough", 207 | @"Brandprotect", 208 | @"BrandVerity", 209 | @"Brandwatch", 210 | @"Brodie\/", 211 | @"Browsershots", 212 | @"BUbiNG", 213 | @"Buck\/", 214 | @"Buddy", 215 | @"BuiltWith", 216 | @"Bullseye", 217 | @"BunnySlippers", 218 | @"Burf Search", 219 | @"Butterfly\/", 220 | @"BuzzSumo", 221 | @"CAAM\/[0-9]", 222 | @"CakePHP", 223 | @"Calculon", 224 | @"Canary%20Mail", 225 | @"CaretNail", 226 | @"catexplorador", 227 | @"CC Metadata Scaper", 228 | @"Cegbfeieh", 229 | @"censys", 230 | @"centuryb.o.t9[at]gmail.com", 231 | @"Cerberian Drtrs", 232 | @"CERT\.at-Statistics-Survey", 233 | @"cf-facebook", 234 | @"cg-eye", 235 | @"changedetection", 236 | @"ChangesMeter", 237 | @"Charlotte", 238 | @"chatterino-api-cache", 239 | @"CheckHost", 240 | @"checkprivacy", 241 | @"CherryPicker", 242 | @"ChinaClaw", 243 | @"Chirp\/", 244 | @"chkme\.com", 245 | @"Chlooe", 246 | @"Chromaxa", 247 | @"CirrusExplorer", 248 | @"CISPA Vulnerability Notification", 249 | @"CISPA Web Analyser", 250 | @"Citoid", 251 | @"CJNetworkQuality", 252 | @"Clarsentia", 253 | @"clips\.ua\.ac\.be", 254 | @"Cloud mapping", 255 | @"CloudEndure", 256 | @"CloudFlare-AlwaysOnline", 257 | @"Cloudflare-Healthchecks", 258 | @"Cloudinary", 259 | @"cmcm\.com", 260 | @"coccoc", 261 | @"cognitiveseo", 262 | @"ColdFusion", 263 | @"colly -", 264 | @"CommaFeed", 265 | @"Commons-HttpClient", 266 | @"commonscan", 267 | @"contactbigdatafr", 268 | @"contentkingapp", 269 | @"Contextual Code Sites Explorer", 270 | @"convera", 271 | @"CookieReports", 272 | @"copyright sheriff", 273 | @"CopyRightCheck", 274 | @"Copyscape", 275 | @"cortex\/", 276 | @"Cosmos4j\.feedback", 277 | @"Covario-IDS", 278 | @"Craw\/", 279 | @"Crescent", 280 | @"Criteo", 281 | @"Crowsnest", 282 | @"CSHttp", 283 | @"CSSCheck", 284 | @"Cula\/", 285 | @"curb", 286 | @"Curious George", 287 | @"curl", 288 | @"cuwhois\/", 289 | @"cybo\.com", 290 | @"DAP\/NetHTTP", 291 | @"DareBoost", 292 | @"DatabaseDriverMysqli", 293 | @"DataCha0s", 294 | @"DatadogSynthetics", 295 | @"Datafeedwatch", 296 | @"Datanyze", 297 | @"DataparkSearch", 298 | @"dataprovider", 299 | @"DataXu", 300 | @"Daum(oa)?[ \/][0-9]", 301 | @"dBpoweramp", 302 | @"ddline", 303 | @"deeris", 304 | @"delve\.ai", 305 | @"Demon", 306 | @"DeuSu", 307 | @"developers\.google\.com\/\+\/web\/snippet\/", 308 | @"Devil", 309 | @"Digg", 310 | @"Digincore", 311 | @"DigitalPebble", 312 | @"Dirbuster", 313 | @"Discourse Forum Onebox", 314 | @"Dispatch\/", 315 | @"Disqus\/", 316 | @"DittoSpyder", 317 | @"dlvr", 318 | @"DMBrowser", 319 | @"DNSPod-reporting", 320 | @"docoloc", 321 | @"Dolphin http client", 322 | @"DomainAppender", 323 | @"DomainLabz", 324 | @"Domains Project\/", 325 | @"Donuts Content Explorer", 326 | @"dotMailer content retrieval", 327 | @"dotSemantic", 328 | @"downforeveryoneorjustme", 329 | @"Download Wonder", 330 | @"downnotifier", 331 | @"DowntimeDetector", 332 | @"Drip", 333 | @"drupact", 334 | @"Drupal \(\+http:\/\/drupal\.org\/\)", 335 | @"DTS Agent", 336 | @"dubaiindex", 337 | @"DuplexWeb-Google", 338 | @"DynatraceSynthetic", 339 | @"EARTHCOM", 340 | @"Easy-Thumb", 341 | @"EasyDL", 342 | @"Ebingbong", 343 | @"ec2linkfinder", 344 | @"eCairn-Grabber", 345 | @"eCatch", 346 | @"ECCP", 347 | @"eContext\/", 348 | @"Ecxi", 349 | @"EirGrabber", 350 | @"ElectricMonk", 351 | @"elefent", 352 | @"EMail Exractor", 353 | @"EMail Wolf", 354 | @"EmailWolf", 355 | @"Embarcadero", 356 | @"Embed PHP Library", 357 | @"Embedly", 358 | @"endo\/", 359 | @"europarchive\.org", 360 | @"evc-batch", 361 | @"EventMachine HttpClient", 362 | @"Everwall Link Expander", 363 | @"Evidon", 364 | @"Evrinid", 365 | @"ExactSearch", 366 | @"ExaleadCloudview", 367 | @"Excel\/", 368 | @"exif", 369 | @"ExoRank", 370 | @"Exploratodo", 371 | @"Express WebPictures", 372 | @"Extreme Picture Finder", 373 | @"EyeNetIE", 374 | @"ezooms", 375 | @"facebookexternalhit", 376 | @"facebookexternalua", 377 | @"facebookplatform", 378 | @"fairshare", 379 | @"Faraday v", 380 | @"fasthttp", 381 | @"Faveeo", 382 | @"Favicon downloader", 383 | @"faviconarchive", 384 | @"faviconkit", 385 | @"FavOrg", 386 | @"Feed Wrangler", 387 | @"Feedable\/", 388 | @"Feedbin", 389 | @"FeedBooster", 390 | @"FeedBucket", 391 | @"FeedBunch\/", 392 | @"FeedBurner", 393 | @"feeder", 394 | @"Feedly", 395 | @"FeedshowOnline", 396 | @"Feedshow\/", 397 | @"Feedspot", 398 | @"FeedViewer\/", 399 | @"Feedwind\/", 400 | @"FeedZcollector", 401 | @"feeltiptop", 402 | @"Fetch API", 403 | @"Fetch\/[0-9]", 404 | @"Fever\/[0-9]", 405 | @"FHscan", 406 | @"Fiery%20Feeds", 407 | @"Filestack", 408 | @"Fimap", 409 | @"findlink", 410 | @"findthatfile", 411 | @"FlashGet", 412 | @"FlipboardBrowserProxy", 413 | @"FlipboardProxy", 414 | @"FlipboardRSS", 415 | @"Flock\/", 416 | @"Florienzh\/", 417 | @"fluffy", 418 | @"Flunky", 419 | @"flynxapp", 420 | @"forensiq", 421 | @"ForusP", 422 | @"FoundSeoTool", 423 | @"free thumbnails", 424 | @"Freeuploader", 425 | @"FreshRSS", 426 | @"frontman", 427 | @"Funnelback", 428 | @"Fuzz Faster U Fool", 429 | @"G-i-g-a-b-o-t", 430 | @"g00g1e\.net", 431 | @"ganarvisitas", 432 | @"gdnplus\.com", 433 | @"geek-tools", 434 | @"Genieo", 435 | @"GentleSource", 436 | @"GetCode", 437 | @"Getintent", 438 | @"GetLinkInfo", 439 | @"getprismatic", 440 | @"GetRight", 441 | @"getroot", 442 | @"GetURLInfo\/", 443 | @"GetWeb", 444 | @"Geziyor", 445 | @"Ghost Inspector", 446 | @"GigablastOpenSource", 447 | @"GIS-LABS", 448 | @"github-camo", 449 | @"GitHub-Hookshot", 450 | @"github\.com", 451 | @"Go http package", 452 | @"Go [\d\.]* package http", 453 | @"Go!Zilla", 454 | @"Go-Ahead-Got-It", 455 | @"Go-http-client", 456 | @"go-mtasts\/", 457 | @"gobuster", 458 | @"gobyus", 459 | @"Gofeed", 460 | @"gofetch", 461 | @"Goldfire Server", 462 | @"GomezAgent", 463 | @"gooblog", 464 | @"Goodzer\/", 465 | @"Google AppsViewer", 466 | @"Google Desktop", 467 | @"Google favicon", 468 | @"Google Keyword Suggestion", 469 | @"Google Keyword Tool", 470 | @"Google Page Speed Insights", 471 | @"Google PP Default", 472 | @"Google Search Console", 473 | @"Google Web Preview", 474 | @"Google-Ads-Creatives-Assistant", 475 | @"Google-Ads-Overview", 476 | @"Google-Adwords", 477 | @"Google-Apps-Script", 478 | @"Google-Calendar-Importer", 479 | @"Google-HotelAdsVerifier", 480 | @"Google-HTTP-Java-Client", 481 | @"Google-Podcast", 482 | @"Google-Publisher-Plugin", 483 | @"Google-Read-Aloud", 484 | @"Google-SearchByImage", 485 | @"Google-Site-Verification", 486 | @"Google-SMTP-STS", 487 | @"Google-speakr", 488 | @"Google-Structured-Data-Testing-Tool", 489 | @"Google-Transparency-Report", 490 | @"google-xrawler", 491 | @"Google-Youtube-Links", 492 | @"GoogleDocs", 493 | @"GoogleHC\/", 494 | @"GoogleProber", 495 | @"GoogleProducer", 496 | @"GoogleSites", 497 | @"Gookey", 498 | @"GoSpotCheck", 499 | @"gosquared-thumbnailer", 500 | @"Gotit", 501 | @"GoZilla", 502 | @"grabify", 503 | @"GrabNet", 504 | @"Grafula", 505 | @"Grammarly", 506 | @"GrapeFX", 507 | @"GreatNews", 508 | @"Gregarius", 509 | @"GRequests", 510 | @"grokkit", 511 | @"grouphigh", 512 | @"grub-client", 513 | @"gSOAP\/", 514 | @"GT::WWW", 515 | @"GTmetrix", 516 | @"GuzzleHttp", 517 | @"gvfs\/", 518 | @"HAA(A)?RTLAND http client", 519 | @"Haansoft", 520 | @"hackney\/", 521 | @"Hadi Agent", 522 | @"HappyApps-WebCheck", 523 | @"Hardenize", 524 | @"Hatena", 525 | @"Havij", 526 | @"HaxerMen", 527 | @"HeadlessChrome", 528 | @"HEADMasterSEO", 529 | @"HeartRails_Capture", 530 | @"help@dataminr\.com", 531 | @"heritrix", 532 | @"Hexometer", 533 | @"historious", 534 | @"hkedcity", 535 | @"hledejLevne\.cz", 536 | @"Hloader", 537 | @"HMView", 538 | @"Holmes", 539 | @"HonesoSearchEngine", 540 | @"HootSuite Image proxy", 541 | @"Hootsuite-WebFeed", 542 | @"hosterstats", 543 | @"HostTracker", 544 | @"ht:\/\/check", 545 | @"htdig", 546 | @"HTMLparser", 547 | @"htmlyse", 548 | @"HTTP Banner Detection", 549 | @"http-get", 550 | @"HTTP-Header-Abfrage", 551 | @"http-kit", 552 | @"http-request\/", 553 | @"HTTP-Tiny", 554 | @"HTTP::Lite", 555 | 556 | // Francis [Bot] 557 | @"http:\/\/www.neomo.de\/", 558 | @"HttpComponents", 559 | @"httphr", 560 | @"HTTPie", 561 | @"HTTPMon", 562 | @"httpRequest", 563 | @"httpscheck", 564 | @"httpssites_power", 565 | @"httpunit", 566 | @"HttpUrlConnection", 567 | @"http\.rb\/", 568 | @"HTTP_Compression_Test", 569 | @"http_get", 570 | @"http_request2", 571 | @"http_requester", 572 | @"httrack", 573 | @"huaweisymantec", 574 | @"HubSpot ", 575 | @"HubSpot-Link-Resolver", 576 | @"Humanlinks", 577 | @"i2kconnect\/", 578 | @"Iblog", 579 | @"ichiro", 580 | @"Id-search", 581 | @"IdeelaborPlagiaat", 582 | @"IDG Twitter Links Resolver", 583 | @"IDwhois\/", 584 | @"Iframely", 585 | @"igdeSpyder", 586 | @"iGooglePortal", 587 | @"IlTrovatore", 588 | @"Image Fetch", 589 | @"Image Sucker", 590 | @"ImageEngine\/", 591 | @"ImageVisu\/", 592 | @"Imagga", 593 | @"imagineeasy", 594 | @"imgsizer", 595 | @"InAGist", 596 | @"inbound\.li parser", 597 | @"InDesign%20CC", 598 | @"Indy Library", 599 | @"InetURL", 600 | @"infegy", 601 | @"infohelfer", 602 | @"InfoTekies", 603 | @"InfoWizards Reciprocal Link", 604 | @"inpwrd\.com", 605 | @"instabid", 606 | @"Instapaper", 607 | @"Integrity", 608 | @"integromedb", 609 | @"Intelliseek", 610 | @"InterGET", 611 | @"Internet Ninja", 612 | @"InternetSeer", 613 | @"internetVista monitor", 614 | @"internetwache", 615 | @"internet_archive", 616 | @"intraVnews", 617 | @"IODC", 618 | @"IOI", 619 | @"iplabel", 620 | @"ips-agent", 621 | @"IPS\/[0-9]", 622 | @"IPWorks HTTP\/S Component", 623 | @"iqdb\/", 624 | @"Iria", 625 | @"Irokez", 626 | @"isitup\.org", 627 | @"iskanie", 628 | @"isUp\.li", 629 | @"iThemes Sync\/", 630 | @"IZaBEE", 631 | @"iZSearch", 632 | @"JAHHO", 633 | @"janforman", 634 | @"Jaunt\/", 635 | @"Java.*outbrain", 636 | @"javelin\.io", 637 | @"Jbrofuzz", 638 | @"Jersey\/", 639 | @"JetCar", 640 | @"Jigsaw", 641 | @"Jobboerse", 642 | @"JobFeed discovery", 643 | @"Jobg8 URL Monitor", 644 | @"jobo", 645 | @"Jobrapido", 646 | @"Jobsearch1\.5", 647 | @"JoinVision Generic", 648 | @"JolokiaPwn", 649 | @"Joomla", 650 | @"Jorgee", 651 | @"JS-Kit", 652 | @"JungleKeyThumbnail", 653 | @"JustView", 654 | @"Kaspersky Lab CFR link resolver", 655 | @"Kelny\/", 656 | @"Kerrigan\/", 657 | @"KeyCDN", 658 | @"Keyword Density", 659 | @"Keywords Research", 660 | @"khttp\/", 661 | @"KickFire", 662 | @"KimonoLabs\/", 663 | @"Kml-Google", 664 | @"knows\.is", 665 | @"KOCMOHABT", 666 | @"kouio", 667 | @"kube-probe", 668 | @"kubectl", 669 | @"kulturarw3", 670 | @"KumKie", 671 | @"Larbin", 672 | @"Lavf\/", 673 | @"leakix\.net", 674 | @"LeechFTP", 675 | @"LeechGet", 676 | @"letsencrypt", 677 | @"Lftp", 678 | @"LibVLC", 679 | @"LibWeb", 680 | @"Libwhisker", 681 | @"libwww", 682 | @"Licorne", 683 | @"Liferea\/", 684 | @"Lighthouse", 685 | @"Lightspeedsystems", 686 | @"Likse", 687 | @"limber\.io", 688 | @"Link Valet", 689 | @"LinkAlarm\/", 690 | @"LinkAnalyser", 691 | @"linkCheck", 692 | @"linkdex", 693 | @"LinkExaminer", 694 | @"linkfluence", 695 | @"linkpeek", 696 | @"LinkPreview", 697 | @"LinkScan", 698 | @"LinksManager", 699 | @"LinkTiger", 700 | @"LinkWalker", 701 | @"link_thumbnailer", 702 | @"Lipperhey", 703 | @"Litemage_walker", 704 | @"livedoor ScreenShot", 705 | @"LoadImpactRload", 706 | @"localsearch-web", 707 | @"LongURL API", 708 | @"longurl-r-package", 709 | @"looid\.com", 710 | @"looksystems\.net", 711 | @"ltx71", 712 | @"lua-resty-http", 713 | @"Lucee \(CFML Engine\)", 714 | @"Lush Http Client", 715 | @"lwp-request", 716 | @"lwp-trivial", 717 | @"LWP::Simple", 718 | @"lycos", 719 | @"LYT\.SR", 720 | @"L\.webis", 721 | @"mabontland", 722 | @"MacOutlook\/", 723 | @"Mag-Net", 724 | @"MagpieRSS", 725 | @"Mail::STS", 726 | @"MailChimp", 727 | @"Mail\.Ru", 728 | @"Majestic12", 729 | @"makecontact\/", 730 | @"Mandrill", 731 | @"MapperCmd", 732 | @"marketinggrader", 733 | @"MarkMonitor", 734 | @"MarkWatch", 735 | @"Mass Downloader", 736 | @"masscan\/", 737 | @"Mata Hari", 738 | @"mattermost", 739 | @"Mediametric", 740 | @"Mediapartners-Google", 741 | @"mediawords", 742 | @"MegaIndex\.ru", 743 | @"MeltwaterNews", 744 | @"Melvil Rawi", 745 | @"MemGator", 746 | @"Metaspinner", 747 | @"MetaURI", 748 | @"MFC_Tear_Sample", 749 | @"Microsearch", 750 | @"Microsoft Data Access", 751 | @"Microsoft Office", 752 | @"Microsoft Outlook", 753 | @"Microsoft Windows Network Diagnostics", 754 | @"Microsoft-WebDAV-MiniRedir", 755 | @"Microsoft\.Data\.Mashup", 756 | @"MIDown tool", 757 | @"MIIxpc", 758 | @"Mindjet", 759 | @"Miniature\.io", 760 | @"Miniflux", 761 | @"mio_httpc", 762 | @"Miro-HttpClient", 763 | @"Mister PiX", 764 | @"mixdata dot com", 765 | @"mixed-content-scan", 766 | @"mixnode", 767 | @"Mnogosearch", 768 | @"mogimogi", 769 | @"Mojeek", 770 | @"Mojolicious \(Perl\)", 771 | @"monitis", 772 | @"Monitority\/", 773 | @"Monit\/", 774 | @"montastic", 775 | @"MonTools", 776 | @"Moreover", 777 | @"Morfeus Fucking Scanner", 778 | @"Morning Paper", 779 | @"MovableType", 780 | @"mowser", 781 | @"Mrcgiguy", 782 | @"Mr\.4x3 Powered", 783 | @"MS Web Services Client Protocol", 784 | @"MSFrontPage", 785 | @"mShots", 786 | @"MuckRack\/", 787 | @"muhstik-scan", 788 | @"MVAClient", 789 | @"MxToolbox\/", 790 | @"myseosnapshot", 791 | @"nagios", 792 | @"Najdi\.si", 793 | @"Name Intelligence", 794 | @"NameFo\.com", 795 | @"Nameprotect", 796 | @"nationalarchives", 797 | @"Navroad", 798 | @"NearSite", 799 | @"Needle", 800 | @"Nessus", 801 | @"Net Vampire", 802 | @"NetAnts", 803 | @"NETCRAFT", 804 | @"NetLyzer", 805 | @"NetMechanic", 806 | @"NetNewsWire", 807 | @"Netpursual", 808 | @"netresearch", 809 | @"NetShelter ContentScan", 810 | @"Netsparker", 811 | @"NetSystemsResearch", 812 | @"nettle", 813 | @"NetTrack", 814 | @"Netvibes", 815 | @"NetZIP", 816 | @"Neustar WPM", 817 | @"NeutrinoAPI", 818 | @"NewRelicPinger", 819 | @"NewsBlur .*Finder", 820 | @"NewsGator", 821 | @"newsme", 822 | @"newspaper\/", 823 | @"Nexgate Ruby Client", 824 | @"NG-Search", 825 | @"nghttp2", 826 | @"Nibbler", 827 | @"NICErsPRO", 828 | @"NihilScio", 829 | @"Nikto", 830 | @"nineconnections", 831 | @"NLNZ_IAHarvester", 832 | @"Nmap Scripting Engine", 833 | @"node-fetch", 834 | @"node-superagent", 835 | @"node-urllib", 836 | @"Nodemeter", 837 | @"NodePing", 838 | @"node\.io", 839 | @"nominet\.org\.uk", 840 | @"nominet\.uk", 841 | @"Norton-Safeweb", 842 | @"Notifixious", 843 | @"notifyninja", 844 | @"NotionEmbedder", 845 | @"nuhk", 846 | @"nutch", 847 | @"Nuzzel", 848 | @"nWormFeedFinder", 849 | @"nyawc\/", 850 | @"Nymesis", 851 | @"NYU", 852 | @"Observatory\/", 853 | @"Ocelli\/", 854 | @"Octopus", 855 | @"oegp", 856 | @"Offline Explorer", 857 | @"Offline Navigator", 858 | @"OgScrper", 859 | @"okhttp", 860 | @"omgili", 861 | @"OMSC", 862 | @"Online Domain Tools", 863 | @"Open Source RSS", 864 | @"OpenCalaisSemanticProxy", 865 | @"Openfind", 866 | @"OpenLinkProfiler", 867 | @"Openstat\/", 868 | @"OpenVAS", 869 | @"OPPO A33", 870 | @"Optimizer", 871 | @"Orbiter", 872 | @"OrgProbe\/", 873 | @"orion-semantics", 874 | @"Outlook-Express", 875 | @"Outlook-iOS", 876 | @"Owler", 877 | @"Owlin", 878 | @"ownCloud News", 879 | @"ow\.ly", 880 | @"OxfordCloudService", 881 | @"page scorer", 882 | @"Page Valet", 883 | @"page2rss", 884 | @"PageFreezer", 885 | @"PageGrabber", 886 | @"PagePeeker", 887 | @"PageScorer", 888 | @"Pagespeed\/", 889 | @"PageThing", 890 | @"page_verifier", 891 | @"Panopta", 892 | @"panscient", 893 | @"Papa Foto", 894 | @"parsijoo", 895 | @"Pavuk", 896 | @"PayPal IPN", 897 | @"pcBrowser", 898 | @"Pcore-HTTP", 899 | @"PDF24 URL To PDF", 900 | @"Pearltrees", 901 | @"PECL::HTTP", 902 | @"peerindex", 903 | @"Peew", 904 | @"PeoplePal", 905 | @"Perlu -", 906 | @"PhantomJS Screenshoter", 907 | @"PhantomJS\/", 908 | @"Photon\/", 909 | @"php-requests", 910 | @"phpservermon", 911 | @"Pi-Monster", 912 | @"Picscout", 913 | @"Picsearch", 914 | @"PictureFinder", 915 | @"Pimonster", 916 | @"Pingability", 917 | @"PingAdmin\.Ru", 918 | @"Pingdom", 919 | @"Pingoscope", 920 | @"PingSpot", 921 | @"ping\.blo\.gs", 922 | @"pinterest\.com", 923 | @"Pixray", 924 | @"Pizilla", 925 | @"Plagger\/", 926 | @"Pleroma ", 927 | @"Ploetz \+ Zeller", 928 | @"Plukkie", 929 | @"plumanalytics", 930 | @"PocketImageCache", 931 | @"PocketParser", 932 | @"Pockey", 933 | @"PodcastAddict\/", 934 | @"POE-Component-Client-HTTP", 935 | @"Polymail\/", 936 | @"Pompos", 937 | @"Porkbun", 938 | @"Port Monitor", 939 | @"postano", 940 | @"postfix-mta-sts-resolver", 941 | @"PostmanRuntime", 942 | @"postplanner\.com", 943 | @"PostPost", 944 | @"postrank", 945 | @"PowerPoint\/", 946 | @"Prebid", 947 | @"Prerender", 948 | @"Priceonomics Analysis Engine", 949 | @"PrintFriendly", 950 | @"PritTorrent", 951 | @"Prlog", 952 | @"probethenet", 953 | @"Project ?25499", 954 | @"Project-Resonance", 955 | @"prospectb2b", 956 | @"Protopage", 957 | @"ProWebWalker", 958 | @"proximic", 959 | @"PRTG Network Monitor", 960 | @"pshtt, https scanning", 961 | @"PTST ", 962 | @"PTST\/[0-9]+", 963 | @"Pump", 964 | @"Python-httplib2", 965 | @"python-httpx", 966 | @"python-requests", 967 | @"Python-urllib", 968 | @"Qirina Hurdler", 969 | @"QQDownload", 970 | @"QrafterPro", 971 | @"Qseero", 972 | @"Qualidator", 973 | @"QueryN Metasearch", 974 | @"queuedriver", 975 | @"quic-go-HTTP\/", 976 | @"QuiteRSS", 977 | @"Quora Link Preview", 978 | @"Qwantify", 979 | @"Radian6", 980 | @"RadioPublicImageResizer", 981 | @"Railgun\/", 982 | @"RankActive", 983 | @"RankFlex", 984 | @"RankSonicSiteAuditor", 985 | @"RapidLoad\/", 986 | @"Re-re Studio", 987 | @"ReactorNetty", 988 | @"Readability", 989 | @"RealDownload", 990 | @"RealPlayer%20Downloader", 991 | @"RebelMouse", 992 | @"Recorder", 993 | @"RecurPost\/", 994 | @"redback\/", 995 | @"ReederForMac", 996 | @"Reeder\/", 997 | @"ReGet", 998 | @"RepoMonkey", 999 | @"request\.js", 1000 | @"reqwest\/", 1001 | @"ResponseCodeTest", 1002 | @"RestSharp", 1003 | @"Riddler", 1004 | @"Rival IQ", 1005 | @"Robosourcer", 1006 | @"Robozilla", 1007 | @"ROI Hunter", 1008 | @"RPT-HTTPClient", 1009 | @"RSSMix\/", 1010 | @"RSSOwl", 1011 | @"RyowlEngine", 1012 | @"safe-agent-scanner", 1013 | @"SalesIntelligent", 1014 | @"Saleslift", 1015 | @"SAP NetWeaver Application Server", 1016 | @"SauceNAO", 1017 | @"SBIder", 1018 | @"sc-downloader", 1019 | @"scalaj-http", 1020 | @"Scamadviser-Frontend", 1021 | @"ScanAlert", 1022 | @"scan\.lol", 1023 | @"Scoop", 1024 | @"scooter", 1025 | @"ScopeContentAG-HTTP-Client", 1026 | @"ScoutJet", 1027 | @"ScoutURLMonitor", 1028 | @"ScrapeBox Page Scanner", 1029 | @"Scrapy", 1030 | @"Screaming", 1031 | @"ScreenShotService", 1032 | @"Scrubby", 1033 | @"Scrutiny\/", 1034 | @"Search37", 1035 | @"searchenginepromotionhelp", 1036 | @"Searchestate", 1037 | @"SearchExpress", 1038 | @"SearchSight", 1039 | @"SearchWP", 1040 | @"search\.thunderstone", 1041 | @"Seeker", 1042 | @"semanticdiscovery", 1043 | @"semanticjuice", 1044 | @"Semiocast HTTP client", 1045 | @"Semrush", 1046 | @"Sendsay\.Ru", 1047 | @"sentry\/", 1048 | @"SEO Browser", 1049 | @"Seo Servis", 1050 | @"seo-nastroj\.cz", 1051 | @"seo4ajax", 1052 | @"Seobility", 1053 | @"SEOCentro", 1054 | @"SeoCheck", 1055 | @"SEOkicks", 1056 | @"SEOlizer", 1057 | @"Seomoz", 1058 | @"SEOprofiler", 1059 | @"seoscanners", 1060 | @"SEOsearch", 1061 | @"seositecheckup", 1062 | @"SEOstats", 1063 | @"servernfo", 1064 | @"sexsearcher", 1065 | @"Seznam", 1066 | @"Shelob", 1067 | @"Shodan", 1068 | @"Shoppimon", 1069 | @"ShopWiki", 1070 | @"ShortLinkTranslate", 1071 | @"shortURL lengthener", 1072 | @"shrinktheweb", 1073 | @"Sideqik", 1074 | @"Siege", 1075 | @"SimplePie", 1076 | @"SimplyFast", 1077 | @"Siphon", 1078 | @"SISTRIX", 1079 | @"Site Sucker", 1080 | @"Site-Shot\/", 1081 | @"Site24x7", 1082 | @"SiteBar", 1083 | @"Sitebeam", 1084 | @"Sitebulb\/", 1085 | @"SiteCondor", 1086 | @"SiteExplorer", 1087 | @"SiteGuardian", 1088 | @"Siteimprove", 1089 | @"SiteIndexed", 1090 | @"Sitemap(s)? Generator", 1091 | @"SitemapGenerator", 1092 | @"SiteMonitor", 1093 | @"Siteshooter B0t", 1094 | @"SiteSnagger", 1095 | @"SiteSucker", 1096 | @"SiteTruth", 1097 | @"Sitevigil", 1098 | @"sitexy\.com", 1099 | @"SkypeUriPreview", 1100 | @"Slack\/", 1101 | @"sli-systems\.com", 1102 | @"slider\.com", 1103 | @"slurp", 1104 | @"SlySearch", 1105 | @"SmartDownload", 1106 | @"SMRF URL Expander", 1107 | @"SMUrlExpander", 1108 | @"Snake", 1109 | @"Snappy", 1110 | @"SnapSearch", 1111 | @"Snarfer\/", 1112 | @"SniffRSS", 1113 | @"sniptracker", 1114 | @"Snoopy", 1115 | @"SnowHaze Search", 1116 | @"sogou web", 1117 | @"SortSite", 1118 | @"Sottopop", 1119 | @"sovereign\.ai", 1120 | @"SpaceBison", 1121 | @"SpamExperts", 1122 | @"Spammen", 1123 | @"Spanner", 1124 | @"spaziodati", 1125 | @"SPDYCheck", 1126 | @"Specificfeeds", 1127 | @"speedy", 1128 | @"SPEng", 1129 | @"Spinn3r", 1130 | @"spray-can", 1131 | @"Sprinklr ", 1132 | @"spyonweb", 1133 | @"sqlmap", 1134 | @"Sqlworm", 1135 | @"Sqworm", 1136 | @"SSL Labs", 1137 | @"ssl-tools", 1138 | @"StackRambler", 1139 | @"Statastico\/", 1140 | @"Statically-", 1141 | @"StatusCake", 1142 | @"Steeler", 1143 | @"Stratagems Kumo", 1144 | @"Stripe\/", 1145 | @"Stroke\.cz", 1146 | @"StudioFACA", 1147 | @"StumbleUpon", 1148 | @"suchen", 1149 | @"Sucuri", 1150 | @"summify", 1151 | @"SuperHTTP", 1152 | @"Surphace Scout", 1153 | @"Suzuran", 1154 | @"swcd ", 1155 | @"Symfony BrowserKit", 1156 | @"Symfony2 BrowserKit", 1157 | @"Synapse\/", 1158 | @"Syndirella\/", 1159 | @"SynHttpClient-Built", 1160 | @"Sysomos", 1161 | @"sysscan", 1162 | @"Szukacz", 1163 | @"T0PHackTeam", 1164 | @"tAkeOut", 1165 | @"Tarantula\/", 1166 | @"Taringa UGC", 1167 | @"TarmotGezgin", 1168 | @"tchelebi\.io", 1169 | @"techiaith\.cymru", 1170 | @"Teleport", 1171 | @"Telesoft", 1172 | @"Telesphoreo", 1173 | @"Telesphorep", 1174 | @"Tenon\.io", 1175 | @"teoma", 1176 | @"terrainformatica", 1177 | @"Test Certificate Info", 1178 | @"testuri", 1179 | @"Tetrahedron", 1180 | @"TextRazor Downloader", 1181 | @"The Drop Reaper", 1182 | @"The Expert HTML Source Viewer", 1183 | @"The Intraformant", 1184 | @"The Knowledge AI", 1185 | @"theinternetrules", 1186 | @"TheNomad", 1187 | @"Thinklab", 1188 | @"Thumbor", 1189 | @"Thumbshots", 1190 | @"ThumbSniper", 1191 | @"timewe\.net", 1192 | @"TinEye", 1193 | @"Tiny Tiny RSS", 1194 | @"TLSProbe\/", 1195 | @"Toata", 1196 | @"topster", 1197 | @"touche\.com", 1198 | @"Traackr\.com", 1199 | @"tracemyfile", 1200 | @"Trackuity", 1201 | @"TrapitAgent", 1202 | @"Trendiction", 1203 | @"Trendsmap", 1204 | @"trendspottr", 1205 | @"truwoGPS", 1206 | @"TryJsoup", 1207 | @"TulipChain", 1208 | @"Turingos", 1209 | @"Turnitin", 1210 | @"tweetedtimes", 1211 | @"Tweetminster", 1212 | @"Tweezler\/", 1213 | @"twibble", 1214 | @"Twice", 1215 | @"Twikle", 1216 | @"Twingly", 1217 | @"Twisted PageGetter", 1218 | @"Typhoeus", 1219 | @"ubermetrics-technologies", 1220 | @"uclassify", 1221 | @"UdmSearch", 1222 | @"ultimate_sitemap_parser", 1223 | @"unchaos", 1224 | @"unirest-java", 1225 | @"UniversalFeedParser", 1226 | @"unshortenit", 1227 | @"Unshorten\.It", 1228 | @"Untiny", 1229 | @"UnwindFetchor", 1230 | @"updated", 1231 | @"updown\.io daemon", 1232 | @"Upflow", 1233 | @"Uptimia", 1234 | @"URL Verifier", 1235 | @"Urlcheckr", 1236 | @"URLitor", 1237 | @"urlresolver", 1238 | @"Urlstat", 1239 | @"URLTester", 1240 | @"UrlTrends Ranking Updater", 1241 | @"URLy Warning", 1242 | @"URLy\.Warning", 1243 | @"URL\/Emacs", 1244 | @"Vacuum", 1245 | @"Vagabondo", 1246 | @"VB Project", 1247 | @"vBSEO", 1248 | @"VCI", 1249 | @"via ggpht\.com GoogleImageProxy", 1250 | @"Virusdie", 1251 | @"visionutils", 1252 | @"vkShare", 1253 | @"VoidEYE", 1254 | @"Voil", 1255 | @"voltron", 1256 | @"voyager\/", 1257 | @"VSAgent\/", 1258 | @"VSB-TUO\/", 1259 | @"Vulnbusters Meter", 1260 | @"VYU2", 1261 | @"w3af\.org", 1262 | @"W3C-checklink", 1263 | @"W3C-mobileOK", 1264 | @"W3C_Unicorn", 1265 | @"WAC-OFU", 1266 | @"WakeletLinkExpander", 1267 | @"WallpapersHD", 1268 | @"Wallpapers\/[0-9]+", 1269 | @"wangling", 1270 | @"Wappalyzer", 1271 | @"WatchMouse", 1272 | @"WbSrch\/", 1273 | @"WDT\.io", 1274 | @"Web Auto", 1275 | @"Web Collage", 1276 | @"Web Enhancer", 1277 | @"Web Fetch", 1278 | @"Web Fuck", 1279 | @"Web Pix", 1280 | @"Web Sauger", 1281 | @"Web spyder", 1282 | @"Web Sucker", 1283 | @"web-capture\.net", 1284 | @"Web-sniffer", 1285 | @"Webalta", 1286 | @"Webauskunft", 1287 | @"WebAuto", 1288 | @"WebCapture", 1289 | @"WebClient\/", 1290 | @"webcollage", 1291 | @"WebCookies", 1292 | @"WebCopier", 1293 | @"WebCorp", 1294 | @"WebDataStats", 1295 | @"WebDoc", 1296 | @"WebEnhancer", 1297 | @"WebFetch", 1298 | @"WebFuck", 1299 | @"WebGazer", 1300 | @"WebGo IS", 1301 | @"WebImageCollector", 1302 | @"WebImages", 1303 | @"WebIndex", 1304 | @"webkit2png", 1305 | @"WebLeacher", 1306 | @"webmastercoffee", 1307 | @"webmon ", 1308 | @"WebPix", 1309 | @"WebReaper", 1310 | @"WebSauger", 1311 | @"webscreenie", 1312 | @"Webshag", 1313 | @"Webshot", 1314 | @"Website Quester", 1315 | @"websitepulse agent", 1316 | @"WebsiteQuester", 1317 | @"Websnapr", 1318 | @"WebSniffer", 1319 | @"Webster", 1320 | @"WebStripper", 1321 | @"WebSucker", 1322 | @"webtech\/", 1323 | @"WebThumbnail", 1324 | @"Webthumb\/", 1325 | @"WebWhacker", 1326 | @"WebZIP", 1327 | @"WeLikeLinks", 1328 | @"WEPA", 1329 | @"WeSEE", 1330 | @"wf84", 1331 | @"Wfuzz\/", 1332 | @"wget", 1333 | @"WhatCMS", 1334 | @"WhatsApp", 1335 | @"WhatsMyIP", 1336 | @"WhatWeb", 1337 | @"WhereGoes\?", 1338 | @"Whibse", 1339 | @"WhoAPI\/", 1340 | @"WhoRunsCoinHive", 1341 | @"Whynder Magnet", 1342 | @"Windows-RSS-Platform", 1343 | @"WinHttp-Autoproxy-Service", 1344 | @"WinHTTP\/", 1345 | @"WinPodder", 1346 | @"wkhtmlto", 1347 | @"wmtips", 1348 | @"Woko", 1349 | @"Wolfram HTTPClient", 1350 | @"woorankreview", 1351 | @"WordPress\/", 1352 | @"WordupinfoSearch", 1353 | @"Word\/", 1354 | @"worldping-api", 1355 | @"wotbox", 1356 | @"WP Engine Install Performance API", 1357 | @"WP Rocket", 1358 | @"wpif", 1359 | @"wprecon\.com survey", 1360 | @"WPScan", 1361 | @"wscheck", 1362 | @"Wtrace", 1363 | @"WWW-Collector-E", 1364 | @"WWW-Mechanize", 1365 | @"WWW::Document", 1366 | @"WWW::Mechanize", 1367 | @"WWWOFFLE", 1368 | @"www\.monitor\.us", 1369 | @"x09Mozilla", 1370 | @"x22Mozilla", 1371 | @"XaxisSemanticsClassifier", 1372 | @"XenForo\/", 1373 | @"Xenu Link Sleuth", 1374 | @"XING-contenttabreceiver", 1375 | @"xpymep([0-9]?)\.exe", 1376 | @"Y!J-[A-Z][A-Z][A-Z]", 1377 | @"Yaanb", 1378 | @"yacy", 1379 | @"Yahoo Link Preview", 1380 | @"YahooCacheSystem", 1381 | @"YahooMailProxy", 1382 | @"YahooYSMcm", 1383 | @"YandeG", 1384 | @"Yandex(?!Search)", 1385 | @"yanga", 1386 | @"yeti", 1387 | @"Yo-yo", 1388 | @"Yoleo Consumer", 1389 | @"yomins\.com", 1390 | @"yoogliFetchAgent", 1391 | @"YottaaMonitor", 1392 | @"Your-Website-Sucks", 1393 | @"yourls\.org", 1394 | @"YoYs\.net", 1395 | @"YP\.PL", 1396 | @"Zabbix", 1397 | @"Zade", 1398 | @"Zao", 1399 | @"Zauba", 1400 | @"Zemanta Aggregator", 1401 | @"Zend\\\\Http\\\\Client", 1402 | @"Zend_Http_Client", 1403 | @"Zermelo", 1404 | @"Zeus ", 1405 | @"zgrab", 1406 | @"ZnajdzFoto", 1407 | @"ZnHTTP", 1408 | @"Zombie\.js", 1409 | @"Zoom\.Mac", 1410 | @"ZoteroTranslationServer", 1411 | @"ZyBorg", 1412 | @"[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)" 1413 | }; 1414 | } 1415 | } 1416 | } 1417 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect/Fixtures/Exclusions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCrawlerDetect.Fixtures 4 | { 5 | /// 6 | /// A provider of common user-agent regex snippets that may be safely stripped 7 | /// 8 | public class Exclusions : AbstractProvider 9 | { 10 | /// 11 | /// Constructor 12 | /// 13 | public Exclusions() 14 | { 15 | // List of strings to remove from the user agent before running the crawler regex 16 | // Over a large list of user agents, this gives us about a 55% speed increase! 17 | _data = new System.Collections.Generic.List() 18 | { 19 | @"Safari.[\d\.]*", 20 | @"Firefox.[\d\.]*", 21 | @" Chrome.[\d\.]*", 22 | @"Chromium.[\d\.]*", 23 | @"MSIE.[\d\.]", 24 | @"Opera\/[\d\.]*", 25 | @"Mozilla.[\d\.]*", 26 | @"AppleWebKit.[\d\.]*", 27 | @"Trident.[\d\.]*", 28 | @"Windows NT.[\d\.]*", 29 | @"Android [\d\.]*", 30 | @"Macintosh.", 31 | @"Ubuntu", 32 | @"Linux", 33 | @"[ ]Intel", 34 | @"Mac OS X [\d_]*", 35 | @"(like )?Gecko(.[\d\.]*)?", 36 | @"KHTML,", 37 | @"CriOS.[\d\.]*", 38 | @"CPU iPhone OS ([0-9_])* like Mac OS X", 39 | @"CPU OS ([0-9_])* like Mac OS X", 40 | @"iPod", 41 | @"compatible", 42 | @"x86_..", 43 | @"i686", 44 | @"x64", 45 | @"X11", 46 | @"rv:[\d\.]*", 47 | @"Version.[\d\.]*", 48 | @"WOW64", 49 | @"Win64", 50 | @"Dalvik.[\d\.]*", 51 | @" \.NET CLR [\d\.]*", 52 | @"Presto.[\d\.]*", 53 | @"Media Center PC", 54 | @"BlackBerry", 55 | @"Build", 56 | @"Opera Mini\/\d{1,2}\.\d{1,2}\.[\d\.]*\/\d{1,2}\.", 57 | @"Opera", 58 | @" \.NET[\d\.]*", 59 | @"cubot", 60 | @"; M bot", 61 | @"; CRONO", 62 | @"; B bot", 63 | @"; IDbot", 64 | @"; ID bot", 65 | @"; POWER BOT", 66 | @"OCTOPUS-CORE", 67 | 68 | // Remove the following characters ; 69 | @";" 70 | }; 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect/Fixtures/Headers.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace NetCrawlerDetect.Fixtures 4 | { 5 | /// 6 | /// A provider of HTTP headers that may contain a user-agent string 7 | /// 8 | public class Headers : AbstractProvider 9 | { 10 | /// 11 | /// Constructor 12 | /// 13 | public Headers() 14 | { 15 | // All possible HTTP headers that represent the user agent string 16 | _data = new List() 17 | { 18 | // The default User-Agent string 19 | @"user-agent", 20 | 21 | // Header can occur on devices using Opera Mini 22 | @"x-operamini-phone-ua", 23 | 24 | // Vodafone specific header: http://www.seoprinciple.com/mobile-web-community-still-angry-at-vodafone/24/ 25 | @"x-device-user-agent", 26 | @"x-original-user-agent", 27 | @"x-skyfire-phone", 28 | @"x-bolt-phone-ua", 29 | @"device-stock-ua", 30 | @"x-ucbrowser-device-ua", 31 | 32 | // Sometimes, bots (especially Google) use a genuine user agent, but fill this header in with their email address 33 | @"from", 34 | 35 | // Seen in use by Netsparker 36 | @"x-scanner", 37 | }; 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /NetCrawlerDetect/NetCrawlerDetect/NetCrawlerDetect.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.0 5 | 1.2.113 6 | 1.2.113 7 | Graham "Gee" Plumb 8 | https://github.com/gplumb/NetCrawlerDetect/blob/master/LICENSE 9 | Graham "Gee" Plumb 10 | https://github.com/gplumb/NetCrawlerDetect 11 | NetCrawlerDetect 12 | A .net standard port of JayBizzle's CrawlerDetect project (https://github.com/JayBizzle/Crawler-Detect). 13 | NetCrawlerDetect 14 | Synchronized agents and devices to v1.2.113 of CrawlerDetect 15 | A .net standard port of JayBizzle's CrawlerDetect project (https://github.com/JayBizzle/Crawler-Detect). 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NetCrawlerDetect  [![Build Status](https://travis-ci.org/gplumb/NetCrawlerDetect.svg?branch=master)](https://travis-ci.org/gplumb/NetCrawlerDetect) 2 | A .net standard port of JayBizzle's CrawlerDetect project: [https://github.com/JayBizzle/Crawler-Detect](https://github.com/JayBizzle/Crawler-Detect). 3 | 4 | ## About NetCrawlerDetect 5 | 6 | NetCrawlerDetect is a .net standard class for detecting bots/crawlers/spiders via the user agent and/or http "from" header. Currently able to detect 1,000s of bots/spiders/crawlers. 7 | 8 | ### Installation 9 | The easiest way to get started is to add the nuget package `NetCrawlerDetect` (see [here](https://www.nuget.org/packages/NetCrawlerDetect)). 10 | 11 | For those who don't want to use nuget, feel free to clone this repo and copy `CrawlerDetect.cs` and the `Fixtures` folder into your project directly. 12 | 13 | ### Usage 14 | Like its' originator, you can either pass in a collection of web headers (from which the user agent will be extracted) or pass in the user agent string directly. 15 | 16 | The simplest use of this object is as follows: 17 | 18 | ```csharp 19 | // Pass in the user agent directly 20 | var detector = new CrawlerDetect(); 21 | var result = detector.IsCrawler("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"); 22 | 23 | // Do we have a bot/crawler/spider? 24 | if(result == true) 25 | { 26 | // Yes. Fetch the name of the bot (optional) 27 | var bot = detector.Matches[0].Value; 28 | } 29 | ``` 30 | 31 | ### Contributing 32 | If you find a bot/spider/crawler user agent that NetCrawlerDetect fails to detect, please submit a pull request with the regex pattern added to the `_data` List in `Fixtures/Crawlers.cs` and add the failing user agent to `NetCrawlerDetect.Tests/crawlers.txt`. 33 | 34 | Please also consider submitting a pull request to our parent project :) 35 | --------------------------------------------------------------------------------