├── .babelrc ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE │ └── pull_request_template.md ├── .gitignore ├── .mocharc.json ├── .prettierrc.json ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── dist ├── index.d.ts ├── index.d.ts.map ├── index.js ├── index.js.map ├── lib │ ├── crawler.d.ts │ ├── crawler.d.ts.map │ ├── crawler.js │ ├── crawler.js.map │ └── crawler │ │ ├── crawlers.d.ts │ │ ├── crawlers.d.ts.map │ │ ├── crawlers.js │ │ ├── crawlers.js.map │ │ ├── exclusions.d.ts │ │ ├── exclusions.d.ts.map │ │ ├── exclusions.js │ │ ├── exclusions.js.map │ │ ├── headers.d.ts │ │ ├── headers.d.ts.map │ │ ├── headers.js │ │ ├── headers.js.map │ │ ├── provider.d.ts │ │ ├── provider.d.ts.map │ │ ├── provider.js │ │ └── provider.js.map ├── types.d.ts ├── types.d.ts.map ├── types.js └── types.js.map ├── eslint.config.mjs ├── example └── node │ ├── index.html │ ├── package-lock.json │ ├── package.json │ └── server.js ├── package-lock.json ├── package.json ├── src ├── index.ts ├── lib │ ├── crawler.ts │ └── crawler │ │ ├── crawlers.ts │ │ ├── exclusions.ts │ │ ├── headers.ts │ │ └── provider.ts └── types.ts ├── test └── lib │ ├── crawler.spec.ts │ └── database │ ├── crawlers.txt │ └── devices.txt └── tsconfig.json /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": { 7 | // The % refers to the global coverage of users from browserslist 8 | "browsers": ["> 0.25%, not dead"] 9 | }, 10 | "useBuiltIns": "usage", 11 | "corejs": { "version": "3", "proposals": true } 12 | } 13 | ] 14 | ], 15 | "sourceType": "unambiguous", 16 | "plugins": [ 17 | "@babel/plugin-transform-class-properties" 18 | ] 19 | } -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: jefferyhus 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: 'bug' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | 1. Go to '...' 17 | 2. Click on '....' 18 | 3. Scroll down to '....' 19 | 4. See error 20 | 21 | **Expected behavior** 22 | A clear and concise description of what you expected to happen. 23 | 24 | **Screenshots** 25 | If applicable, add screenshots to help explain your problem. 26 | 27 | **Desktop (please complete the following information):** 28 | 29 | - OS: [e.g. iOS] 30 | - Browser [e.g. chrome, safari] 31 | - Version [e.g. 22] 32 | 33 | **Smartphone (please complete the following information):** 34 | 35 | - Device: [e.g. iPhone6] 36 | - OS: [e.g. iOS8.1] 37 | - Browser [e.g. stock browser, safari] 38 | - Version [e.g. 22] 39 | 40 | **Additional context** 41 | Add any other context about the problem here. 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: 'feature' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pull_request_template.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Pull request 3 | about: Improving the existing code 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Description 11 | 12 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. 13 | 14 | ## Summary 15 | 16 | Please delete options that are not relevant. 17 | 18 | - [ ] Bug fix (non-breaking change which fixes an issue) 19 | - [ ] New feature (non-breaking change which adds functionality) 20 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 21 | - [ ] This change requires a documentation update 22 | 23 | ## Notes 24 | 25 | - Run an example command line 26 | 27 | ## Examples 28 | 29 | Writing examples of how to integrate the new code is helpful 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules/ 3 | npm-debug.log 4 | -------------------------------------------------------------------------------- /.mocharc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extension": ["ts"], 3 | "spec": "test/**/*.spec.ts", 4 | "require": "ts-node/register" 5 | } -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 80, 3 | "singleQuote": true, 4 | "trailingComma": "es5" 5 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at jefferytutorials@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Jeffery ThaGintoki 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Crawler Detect 2 | 3 |



4 | crawlerdetect.io 5 |

6 |

7 | 8 | [![DeepScan grade](https://deepscan.io/api/teams/16465/projects/19756/branches/518343/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=16465&pid=19756&bid=518343) 9 | ![Static Badge](https://img.shields.io/badge/TypeScript-3178C6?style=flat&logo=typescript&logoColor=fff) 10 | [![npm version](https://badge.fury.io/js/es6-crawler-detect.svg?icon=si%3Anpm)](https://badge.fury.io/js/es6-crawler-detect) 11 | ![NPM Downloads](https://img.shields.io/npm/dy/es6-crawler-detect) 12 | [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/JefferyHus/es6-crawler-detect/issues) 13 | 14 | ## About 15 | 16 | This Library is an ES6 version of the original PHP class @[CrawlerDetect](https://github.com/JayBizzle/Crawler-Detect), it helps you detect bots/crawlers and spiders only by scanning the user-agent string or from the global `request.headers`. 17 | 18 | ## Installation 19 | 20 | `npm install es6-crawler-detect` 21 | 22 | ## Usage 23 | 24 | ### ECMAScript 6 (ES6) 25 | 26 | ```javascript 27 | 'use strict'; 28 | 29 | const express = require('express') 30 | const { Crawler, middleware } = require('es6-crawler-detect') 31 | 32 | const app = express() 33 | 34 | app.get('your/route', function async (request, response) { 35 | // create a new Crawler instance 36 | var CrawlerDetector = new Crawler(request) 37 | 38 | // check the current visitor's useragent 39 | if ( CrawlerDetector.isCrawler() ) 40 | { 41 | // true if crawler user agent detected 42 | } 43 | 44 | // or check a user agent string 45 | if ( CrawlerDetector.isCrawler('Mozilla/5.0 (compatible; Sosospider/2.0; +http://help.soso.com/webspider.htm)') ) 46 | { 47 | // true if crawler user agent detected 48 | } 49 | 50 | // Output the name of the bot that matched (if any) 51 | response.send(CrawlerDetector.getMatches()) 52 | }) 53 | 54 | /** 55 | * Or by using the middleware 56 | */ 57 | app.use(middleware((request, reponse) => { 58 | // do something here 59 | // e.g. console.log(request.body) 60 | // e.g. return response.status(403).send('Forbidden') 61 | })) 62 | 63 | app.get('/crawler', function async (request, response) { 64 | // or check a user agent string 65 | request.Crawler.isCrawler('Mozilla/5.0 (compatible; Sosospider/2.0; +http://help.soso.com/webspider.htm)') 66 | 67 | // Output the name of the bot that matched (if any) 68 | response.send(request.Crawler.getMatches()) 69 | }) 70 | ``` 71 | 72 | ### TypeScript 73 | 74 | ```typescript 75 | import { Crawler, middleware } from 'es6-crawler-detect' 76 | 77 | const CrawlerDetector = new Crawler() 78 | 79 | // check the current visitor's useragent 80 | if ( CrawlerDetector.isCrawler() ) 81 | { 82 | // true if crawler user agent detected 83 | } 84 | 85 | // or check a user agent string 86 | if ( CrawlerDetector.isCrawler('Mozilla/5.0 (compatible; Sosospider/2.0; +http://help.soso.com/webspider.htm)') ) 87 | { 88 | // true if crawler user agent detected 89 | } 90 | 91 | // Output the name of the bot that matched (if any) 92 | console.log(CrawlerDetector.getMatches()) 93 | 94 | /** 95 | * Or by using the middleware 96 | */ 97 | middleware((request, reponse) => { 98 | // do something here 99 | // e.g. console.log(request.body) 100 | // e.g. return response.status(403).send('Forbidden') 101 | }) 102 | 103 | ``` 104 | 105 | ## Contributing 106 | 107 | If you find a `bot/spider/crawler` user agent that CrawlerDetect fails to detect, please submit a pull request with the regex pattern added to the `data` array in `./crawler/crawlers.ts`. 108 | -------------------------------------------------------------------------------- /dist/index.d.ts: -------------------------------------------------------------------------------- 1 | import { NextFunction, Request, Response } from 'express-serve-static-core'; 2 | import { Crawler } from './lib/crawler'; 3 | export { Crawler }; 4 | export declare const middleware: (cb?: (req: Request, res: Response, next: NextFunction) => void) => (req: Request, res: Response, next: NextFunction) => void; 5 | //# sourceMappingURL=index.d.ts.map -------------------------------------------------------------------------------- /dist/index.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAE5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,OAAO,EAAE,CAAC;AACnB,eAAO,MAAM,UAAU,QAChB,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,KAAK,IAAI,WAEjD,OAAO,OAAO,QAAQ,QAAQ,YAAY,SASxD,CAAC"} -------------------------------------------------------------------------------- /dist/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.middleware = exports.Crawler = void 0; 4 | const crawler_1 = require("./lib/crawler"); 5 | Object.defineProperty(exports, "Crawler", { enumerable: true, get: function () { return crawler_1.Crawler; } }); 6 | const middleware = (cb) => { 7 | return (req, res, next) => { 8 | // If there is a cb, execute it 9 | if (typeof cb === 'function') { 10 | cb.call(this, req, res, next); 11 | } 12 | // Initiate 13 | req.Crawler = new crawler_1.Crawler(req); 14 | next(); 15 | }; 16 | }; 17 | exports.middleware = middleware; 18 | //# sourceMappingURL=index.js.map -------------------------------------------------------------------------------- /dist/index.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAEA,2CAAwC;AAE/B,wFAFA,iBAAO,OAEA;AACT,MAAM,UAAU,GAAG,CACxB,EAA8D,EAC9D,EAAE;IACF,OAAO,CAAC,GAAY,EAAE,GAAa,EAAE,IAAkB,EAAE,EAAE;QACzD,+BAA+B;QAC/B,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;YAC7B,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,CAAC;QACD,WAAW;QACX,GAAG,CAAC,OAAO,GAAG,IAAI,iBAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC,CAAC;AAZW,QAAA,UAAU,cAYrB"} -------------------------------------------------------------------------------- /dist/lib/crawler.d.ts: -------------------------------------------------------------------------------- 1 | import { Request } from 'express-serve-static-core'; 2 | import { IncomingHttpHeaders } from 'http2'; 3 | export declare class Crawler { 4 | private crawlers; 5 | private headers; 6 | private exclusions; 7 | private request; 8 | private compiledRegexList; 9 | private compiledExclusions; 10 | private httpHeaders; 11 | private userAgent; 12 | private matches?; 13 | constructor(request?: Request, headers?: IncomingHttpHeaders, userAgent?: string); 14 | compileRegex(patterns: string[], flags?: string): RegExp; 15 | private setHttpHeaders; 16 | private setUserAgent; 17 | private getUaHttpHeaders; 18 | getMatches(): string | null | object; 19 | isCrawler(userAgent?: string): boolean; 20 | } 21 | //# sourceMappingURL=crawler.d.ts.map -------------------------------------------------------------------------------- /dist/lib/crawler.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAC;AAM5C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAa;IAE/B,OAAO,CAAC,OAAO,CAA2C;IAC1D,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAyB;gBAGvC,OAAO,CAAC,EAAE,OAAO,EACjB,OAAO,CAAC,EAAE,mBAAmB,EAC7B,SAAS,CAAC,EAAE,MAAM;IAmBb,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;IAI/D,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,gBAAgB;IAIjB,UAAU,IAAI,MAAM,GAAG,IAAI,GAAG,MAAM;IAYpC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO;CAsB9C"} -------------------------------------------------------------------------------- /dist/lib/crawler.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Crawler = void 0; 4 | const crawlers_1 = require("./crawler/crawlers"); 5 | const exclusions_1 = require("./crawler/exclusions"); 6 | const headers_1 = require("./crawler/headers"); 7 | class Crawler { 8 | crawlers; 9 | headers; 10 | exclusions; 11 | request; 12 | compiledRegexList; 13 | compiledExclusions; 14 | httpHeaders; 15 | userAgent; 16 | matches; 17 | constructor(request, headers, userAgent) { 18 | this.crawlers = new crawlers_1.Crawlers(); 19 | this.headers = new headers_1.Headers(); 20 | this.exclusions = new exclusions_1.Exclusions(); 21 | this.request = request ?? {}; 22 | // The regex-list must not be used with g-flag! 23 | // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results 24 | this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i'); 25 | // The exclusions should be used with g-flag in order to remove each value. 26 | this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi'); 27 | this.httpHeaders = this.setHttpHeaders(headers); 28 | this.userAgent = this.setUserAgent(userAgent); 29 | } 30 | compileRegex(patterns, flags) { 31 | return new RegExp(patterns.join('|'), flags); 32 | } 33 | setHttpHeaders(headers) { 34 | // Use the Request headers if httpHeaders is not defined 35 | if (!headers || Object.keys(headers).length === 0) { 36 | if (Object.keys(this.request).length) { 37 | if (this.request.headers) { 38 | return this.request.headers; 39 | } 40 | } 41 | return ''; 42 | } 43 | // Save the headers. 44 | return headers; 45 | } 46 | setUserAgent(userAgent) { 47 | if (!userAgent?.length) { 48 | userAgent = ''; 49 | for (const header of this.getUaHttpHeaders()) { 50 | if (typeof this.httpHeaders === 'object' && 51 | !Array.isArray(this.httpHeaders)) { 52 | if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) { 53 | const headerValue = this.httpHeaders[header.toLowerCase()]; 54 | if (typeof headerValue === 'string') { 55 | const separator = userAgent.length > 0 ? ' ' : ''; 56 | userAgent += separator + headerValue; 57 | } 58 | else if (Array.isArray(headerValue)) { 59 | const separator = userAgent.length > 0 ? ' ' : ''; 60 | userAgent += separator + headerValue.join(' '); 61 | } 62 | } 63 | } 64 | } 65 | } 66 | return userAgent; 67 | } 68 | getUaHttpHeaders() { 69 | return this.headers.getAll(); 70 | } 71 | getMatches() { 72 | if (this.matches !== undefined) { 73 | if (this.matches?.length) { 74 | return this.matches[0]; 75 | } 76 | return null; 77 | } 78 | return {}; 79 | } 80 | isCrawler(userAgent) { 81 | if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) { 82 | return false; 83 | } 84 | let agent = userAgent ?? this.userAgent; 85 | // test on compiled regx 86 | agent = agent.replace(this.compiledExclusions, ''); 87 | if (agent.trim().length === 0) { 88 | return false; 89 | } 90 | const matches = this.compiledRegexList.exec(agent); 91 | if (matches) { 92 | this.matches = matches; 93 | } 94 | return matches !== null && matches.length > 0; 95 | } 96 | } 97 | exports.Crawler = Crawler; 98 | //# sourceMappingURL=crawler.js.map -------------------------------------------------------------------------------- /dist/lib/crawler.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":";;;AAGA,iDAA8C;AAC9C,qDAAkD;AAClD,+CAA4C;AAE5C,MAAa,OAAO;IACV,QAAQ,CAAW;IACnB,OAAO,CAAU;IACjB,UAAU,CAAa;IAEvB,OAAO,CAA2C;IAClD,iBAAiB,CAAS;IAC1B,kBAAkB,CAAS;IAC3B,WAAW,CAA0C;IACrD,SAAS,CAAS;IAClB,OAAO,CAA0B;IAEzC,YACE,OAAiB,EACjB,OAA6B,EAC7B,SAAkB;QAElB,IAAI,CAAC,QAAQ,GAAG,IAAI,mBAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,EAAE,CAAC;QAEnC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAE7B,+CAA+C;QAC/C,yGAAyG;QACzG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAExE,2EAA2E;QAC3E,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;QAE5E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAEM,YAAY,CAAC,QAAkB,EAAE,KAAc;QACpD,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAEO,cAAc,CACpB,OAAiD;QAEjD,wDAAwD;QACxD,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;gBACrC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,oBAAoB;QACpB,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,YAAY,CAAC,SAAkB;QACrC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YACvB,SAAS,GAAG,EAAE,CAAC;YACf,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAC7C,IACE,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ;oBACpC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAChC,CAAC;oBACD,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;wBAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;wBAE3D,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;4BACpC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC;wBACvC,CAAC;6BAAM,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;4BACtC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjD,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,gBAAgB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IAC/B,CAAC;IAEM,UAAU;QACf,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,SAAS,CAAC,SAAkB;QACjC,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACtD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,KAAK,GAAG,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QAExC,wBAAwB;QACxB,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAEnD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACzB,CAAC;QAED,OAAO,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,CAAC;CACF;AAxHD,0BAwHC"} -------------------------------------------------------------------------------- /dist/lib/crawler/crawlers.d.ts: -------------------------------------------------------------------------------- 1 | import { Provider } from './provider'; 2 | export declare class Crawlers implements Provider { 3 | getAll(): string[]; 4 | } 5 | //# sourceMappingURL=crawlers.d.ts.map -------------------------------------------------------------------------------- /dist/lib/crawler/crawlers.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"crawlers.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/crawlers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,QAAS,YAAW,QAAQ;IAChC,MAAM,IAAI,MAAM,EAAE;CAi3C1B"} -------------------------------------------------------------------------------- /dist/lib/crawler/crawlers.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Crawlers = void 0; 4 | class Crawlers { 5 | getAll() { 6 | return [ 7 | ' YLT', 8 | '^Aether', 9 | '^Amazon Simple Notification Service Agent$', 10 | '^Amazon-Route53-Health-Check-Service', 11 | '^b0t$', 12 | '^bluefish ', 13 | '^Calypso v\\/', 14 | '^COMODO DCV', 15 | '^Corax', 16 | '^DangDang', 17 | '^DavClnt', 18 | '^DHSH', 19 | '^docker\\/[0-9]', 20 | '^Expanse', 21 | '^FDM ', 22 | '^git\\/', 23 | '^Goose\\/', 24 | '^Grabber', 25 | '^Gradle\\/', 26 | '^HTTPClient\\/', 27 | '^HTTPing', 28 | '^Java\\/', 29 | '^Jeode\\/', 30 | '^Jetty\\/', 31 | '^Mail\\/', 32 | '^Mget', 33 | '^Microsoft URL Control', 34 | '^Mikrotik\\/', 35 | '^Netlab360', 36 | '^NG\\/[0-9\\.]', 37 | '^NING\\/', 38 | '^npm\\/', 39 | '^Nuclei', 40 | '^PHP-AYMAPI\\/', 41 | '^PHP\\/', 42 | '^pip\\/', 43 | '^pnpm\\/', 44 | '^RMA\\/', 45 | '^Ruby|Ruby\\/[0-9]', 46 | '^Swurl ', 47 | '^TLS tester ', 48 | '^twine\\/', 49 | '^ureq', 50 | '^VSE\\/[0-9]', 51 | '^WordPress\\.com', 52 | '^XRL\\/[0-9]', 53 | '^ZmEu', 54 | '008\\/', 55 | '13TABS', 56 | '192\\.comAgent', 57 | '2GDPR\\/', 58 | '2ip\\.ru', 59 | '404enemy', 60 | '7Siters', 61 | '80legs', 62 | 'a3logics\\.in', 63 | 'A6-Indexer', 64 | 'Abonti', 65 | 'Aboundex', 66 | 'aboutthedomain', 67 | 'Accoona-AI-Agent', 68 | 'acebookexternalhit\\/', 69 | 'acoon', 70 | 'acrylicapps\\.com\\/pulp', 71 | 'Acunetix', 72 | 'AdAuth\\/', 73 | 'adbeat', 74 | 'AddThis', 75 | 'ADmantX', 76 | 'AdminLabs', 77 | 'adressendeutschland', 78 | 'adreview\\/', 79 | 'adscanner', 80 | 'adstxt-worker', 81 | 'Adstxtaggregator', 82 | 'adstxt\\.com', 83 | 'Adyen HttpClient', 84 | 'AffiliateLabz\\/', 85 | 'affilimate-puppeteer', 86 | 'agentslug', 87 | 'AHC', 88 | 'aihit', 89 | 'aiohttp\\/', 90 | 'Airmail', 91 | 'akka-http\\/', 92 | 'akula\\/', 93 | 'alertra', 94 | 'alexa site audit', 95 | 'Alibaba\\.Security\\.Heimdall', 96 | 'Alligator', 97 | 'allloadin', 98 | 'AllSubmitter', 99 | 'alyze\\.info', 100 | 'amagit', 101 | 'Anarchie', 102 | 'AndroidDownloadManager', 103 | 'Anemone', 104 | 'AngleSharp', 105 | 'annotate_google', 106 | 'Anthill', 107 | 'Anturis Agent', 108 | 'Ant\\.com', 109 | 'AnyEvent-HTTP\\/', 110 | 'Apache Ant\\/', 111 | 'Apache Droid', 112 | 'Apache OpenOffice', 113 | 'Apache-HttpAsyncClient', 114 | 'Apache-HttpClient', 115 | 'ApacheBench', 116 | 'Apexoo', 117 | 'apimon\\.de', 118 | 'APIs-Google', 119 | 'AportWorm\\/', 120 | 'AppBeat\\/', 121 | 'AppEngine-Google', 122 | 'AppleSyndication', 123 | 'Aprc\\/[0-9]', 124 | 'Arachmo', 125 | 'arachnode', 126 | 'Arachnophilia', 127 | 'aria2', 128 | 'Arukereso', 129 | 'asafaweb', 130 | 'Asana\\/', 131 | 'Ask Jeeves', 132 | 'AskQuickly', 133 | 'ASPSeek', 134 | 'Asterias', 135 | 'Astute', 136 | 'asynchttp', 137 | 'Attach', 138 | 'attohttpc', 139 | 'autocite', 140 | 'AutomaticWPTester', 141 | 'Autonomy', 142 | 'awin\\.com', 143 | 'AWS Security Scanner', 144 | 'axios\\/', 145 | 'a\\.pr-cy\\.ru', 146 | 'B-l-i-t-z-B-O-T', 147 | 'Backlink-Ceck', 148 | 'backlink-check', 149 | 'BacklinkHttpStatus', 150 | 'BackStreet', 151 | 'BackupLand', 152 | 'BackWeb', 153 | 'Bad-Neighborhood', 154 | 'Badass', 155 | 'baidu\\.com', 156 | 'Bandit', 157 | 'basicstate', 158 | 'BatchFTP', 159 | 'Battleztar Bazinga', 160 | 'baypup\\/', 161 | 'BazQux', 162 | 'BBBike', 163 | 'BCKLINKS', 164 | 'BDFetch', 165 | 'BegunAdvertising', 166 | 'Bewica-security-scan', 167 | 'Bidtellect', 168 | 'BigBozz', 169 | 'Bigfoot', 170 | 'biglotron', 171 | 'BingLocalSearch', 172 | 'BingPreview', 173 | 'binlar', 174 | 'biNu image cacher', 175 | 'Bitacle', 176 | 'Bitrix link preview', 177 | 'biz_Directory', 178 | 'BKCTwitterUnshortener\\/', 179 | 'Black Hole', 180 | 'Blackboard Safeassign', 181 | 'BlackWidow', 182 | 'BlockNote\\.Net', 183 | 'BlogBridge', 184 | 'Bloglines', 185 | 'Bloglovin', 186 | 'BlogPulseLive', 187 | 'BlogSearch', 188 | 'Blogtrottr', 189 | 'BlowFish', 190 | 'boitho\\.com-dc', 191 | 'Boost\\.Beast', 192 | 'BPImageWalker', 193 | 'Braintree-Webhooks', 194 | 'Branch Metrics API', 195 | 'Branch-Passthrough', 196 | 'Brandprotect', 197 | 'BrandVerity', 198 | 'Brandwatch', 199 | 'Brodie\\/', 200 | 'Browsershots', 201 | 'BUbiNG', 202 | 'Buck\\/', 203 | 'Buddy', 204 | 'BuiltWith', 205 | 'Bullseye', 206 | 'BunnySlippers', 207 | 'Burf Search', 208 | 'Butterfly\\/', 209 | 'BuzzSumo', 210 | 'CAAM\\/[0-9]', 211 | 'CakePHP', 212 | 'Calculon', 213 | 'Canary%20Mail', 214 | 'CaretNail', 215 | 'catexplorador', 216 | 'CC Metadata Scaper', 217 | 'Cegbfeieh', 218 | 'censys', 219 | 'centuryb.o.t9[at]gmail.com', 220 | 'Cerberian Drtrs', 221 | 'CERT\\.at-Statistics-Survey', 222 | 'cf-facebook', 223 | 'cg-eye', 224 | 'changedetection', 225 | 'ChangesMeter', 226 | 'Charlotte', 227 | 'CheckHost', 228 | 'checkprivacy', 229 | 'CherryPicker', 230 | 'ChinaClaw', 231 | 'Chirp\\/', 232 | 'chkme\\.com', 233 | 'Chlooe', 234 | 'Chromaxa', 235 | 'CirrusExplorer', 236 | 'CISPA Vulnerability Notification', 237 | 'CISPA Web Analyser', 238 | 'Citoid', 239 | 'CJNetworkQuality', 240 | 'Clarsentia', 241 | 'clips\\.ua\\.ac\\.be', 242 | 'Cloud mapping', 243 | 'CloudEndure', 244 | 'CloudFlare-AlwaysOnline', 245 | 'Cloudflare-Healthchecks', 246 | 'Cloudinary', 247 | 'cmcm\\.com', 248 | 'coccoc', 249 | 'cognitiveseo', 250 | 'ColdFusion', 251 | 'colly -', 252 | 'CommaFeed', 253 | 'Commons-HttpClient', 254 | 'commonscan', 255 | 'contactbigdatafr', 256 | 'contentkingapp', 257 | 'Contextual Code Sites Explorer', 258 | 'convera', 259 | 'CookieReports', 260 | 'copyright sheriff', 261 | 'CopyRightCheck', 262 | 'Copyscape', 263 | 'cortex\\/', 264 | 'Cosmos4j\\.feedback', 265 | 'Covario-IDS', 266 | 'Craw\\/', 267 | 'Crescent', 268 | 'Criteo', 269 | 'Crowsnest', 270 | 'CSHttp', 271 | 'CSSCheck', 272 | 'Cula\\/', 273 | 'curb', 274 | 'Curious George', 275 | 'curl', 276 | 'cuwhois\\/', 277 | 'cybo\\.com', 278 | 'DAP\\/NetHTTP', 279 | 'DareBoost', 280 | 'DatabaseDriverMysqli', 281 | 'DataCha0s', 282 | 'Datafeedwatch', 283 | 'Datanyze', 284 | 'DataparkSearch', 285 | 'dataprovider', 286 | 'DataXu', 287 | 'Daum(oa)?[ \\/][0-9]', 288 | 'dBpoweramp', 289 | 'ddline', 290 | 'deeris', 291 | 'delve\\.ai', 292 | 'Demon', 293 | 'DeuSu', 294 | 'developers\\.google\\.com\\/\\+\\/web\\/snippet\\/', 295 | 'Devil', 296 | 'Digg', 297 | 'Digincore', 298 | 'DigitalPebble', 299 | 'Dirbuster', 300 | 'Discourse Forum Onebox', 301 | 'Dispatch\\/', 302 | 'Disqus\\/', 303 | 'DittoSpyder', 304 | 'dlvr', 305 | 'DMBrowser', 306 | 'DNSPod-reporting', 307 | 'docoloc', 308 | 'Dolphin http client', 309 | 'DomainAppender', 310 | 'DomainLabz', 311 | 'Domains Project\\/', 312 | 'Donuts Content Explorer', 313 | 'dotMailer content retrieval', 314 | 'dotSemantic', 315 | 'downforeveryoneorjustme', 316 | 'Download Wonder', 317 | 'downnotifier', 318 | 'DowntimeDetector', 319 | 'Drip', 320 | 'drupact', 321 | 'Drupal \\(\\+http:\\/\\/drupal\\.org\\/\\)', 322 | 'DTS Agent', 323 | 'dubaiindex', 324 | 'DuplexWeb-Google', 325 | 'DynatraceSynthetic', 326 | 'EARTHCOM', 327 | 'Easy-Thumb', 328 | 'EasyDL', 329 | 'Ebingbong', 330 | 'ec2linkfinder', 331 | 'eCairn-Grabber', 332 | 'eCatch', 333 | 'ECCP', 334 | 'eContext\\/', 335 | 'Ecxi', 336 | 'EirGrabber', 337 | 'ElectricMonk', 338 | 'elefent', 339 | 'EMail Exractor', 340 | 'EMail Wolf', 341 | 'EmailWolf', 342 | 'Embarcadero', 343 | 'Embed PHP Library', 344 | 'Embedly', 345 | 'endo\\/', 346 | 'europarchive\\.org', 347 | 'evc-batch', 348 | 'EventMachine HttpClient', 349 | 'Everwall Link Expander', 350 | 'Evidon', 351 | 'Evrinid', 352 | 'ExactSearch', 353 | 'ExaleadCloudview', 354 | 'Excel\\/', 355 | 'exif', 356 | 'ExoRank', 357 | 'Exploratodo', 358 | 'Express WebPictures', 359 | 'Extreme Picture Finder', 360 | 'EyeNetIE', 361 | 'ezooms', 362 | 'facebookexternalhit', 363 | 'facebookexternalua', 364 | 'facebookplatform', 365 | 'fairshare', 366 | 'Faraday v', 367 | 'fasthttp', 368 | 'Faveeo', 369 | 'Favicon downloader', 370 | 'faviconarchive', 371 | 'faviconkit', 372 | 'FavOrg', 373 | 'Feed Wrangler', 374 | 'Feedable\\/', 375 | 'Feedbin', 376 | 'FeedBooster', 377 | 'FeedBucket', 378 | 'FeedBunch\\/', 379 | 'FeedBurner', 380 | 'feeder', 381 | 'Feedly', 382 | 'FeedshowOnline', 383 | 'Feedshow\\/', 384 | 'Feedspot', 385 | 'FeedViewer\\/', 386 | 'Feedwind\\/', 387 | 'FeedZcollector', 388 | 'feeltiptop', 389 | 'Fetch API', 390 | 'Fetch\\/[0-9]', 391 | 'Fever\\/[0-9]', 392 | 'FHscan', 393 | 'Fiery%20Feeds', 394 | 'Filestack', 395 | 'Fimap', 396 | 'findlink', 397 | 'findthatfile', 398 | 'FlashGet', 399 | 'FlipboardBrowserProxy', 400 | 'FlipboardProxy', 401 | 'FlipboardRSS', 402 | 'Flock\\/', 403 | 'Florienzh\\/', 404 | 'fluffy', 405 | 'Flunky', 406 | 'flynxapp', 407 | 'forensiq', 408 | 'FoundSeoTool', 409 | 'free thumbnails', 410 | 'Freeuploader', 411 | 'FreshRSS', 412 | 'Funnelback', 413 | 'Fuzz Faster U Fool', 414 | 'G-i-g-a-b-o-t', 415 | 'g00g1e\\.net', 416 | 'ganarvisitas', 417 | 'gdnplus\\.com', 418 | 'geek-tools', 419 | 'Genieo', 420 | 'GentleSource', 421 | 'GetCode', 422 | 'Getintent', 423 | 'GetLinkInfo', 424 | 'getprismatic', 425 | 'GetRight', 426 | 'getroot', 427 | 'GetURLInfo\\/', 428 | 'GetWeb', 429 | 'Geziyor', 430 | 'Ghost Inspector', 431 | 'GigablastOpenSource', 432 | 'GIS-LABS', 433 | 'github-camo', 434 | 'GitHub-Hookshot', 435 | 'github\\.com', 436 | 'Go http package', 437 | 'Go [\\d\\.]* package http', 438 | 'Go!Zilla', 439 | 'Go-Ahead-Got-It', 440 | 'Go-http-client', 441 | 'go-mtasts\\/', 442 | 'gobyus', 443 | 'Gofeed', 444 | 'gofetch', 445 | 'Goldfire Server', 446 | 'GomezAgent', 447 | 'gooblog', 448 | 'Goodzer\\/', 449 | 'Google AppsViewer', 450 | 'Google Desktop', 451 | 'Google favicon', 452 | 'Google Keyword Suggestion', 453 | 'Google Keyword Tool', 454 | 'Google Page Speed Insights', 455 | 'Google PP Default', 456 | 'Google Search Console', 457 | 'Google Web Preview', 458 | 'Google-Ads-Creatives-Assistant', 459 | 'Google-Ads-Overview', 460 | 'Google-Adwords', 461 | 'Google-Apps-Script', 462 | 'Google-Calendar-Importer', 463 | 'Google-HotelAdsVerifier', 464 | 'Google-HTTP-Java-Client', 465 | 'Google-Podcast', 466 | 'Google-Publisher-Plugin', 467 | 'Google-Read-Aloud', 468 | 'Google-SearchByImage', 469 | 'Google-Site-Verification', 470 | 'Google-SMTP-STS', 471 | 'Google-speakr', 472 | 'Google-Structured-Data-Testing-Tool', 473 | 'Google-Transparency-Report', 474 | 'google-xrawler', 475 | 'Google-Youtube-Links', 476 | 'GoogleDocs', 477 | 'GoogleHC\\/', 478 | 'GoogleProber', 479 | 'GoogleProducer', 480 | 'GoogleSites', 481 | 'Gookey', 482 | 'GoSpotCheck', 483 | 'gosquared-thumbnailer', 484 | 'Gotit', 485 | 'GoZilla', 486 | 'grabify', 487 | 'GrabNet', 488 | 'Grafula', 489 | 'Grammarly', 490 | 'GrapeFX', 491 | 'GreatNews', 492 | 'Gregarius', 493 | 'GRequests', 494 | 'grokkit', 495 | 'grouphigh', 496 | 'grub-client', 497 | 'gSOAP\\/', 498 | 'GT::WWW', 499 | 'GTmetrix', 500 | 'GuzzleHttp', 501 | 'gvfs\\/', 502 | 'HAA(A)?RTLAND http client', 503 | 'Haansoft', 504 | 'hackney\\/', 505 | 'Hadi Agent', 506 | 'HappyApps-WebCheck', 507 | 'Hardenize', 508 | 'Hatena', 509 | 'Havij', 510 | 'HaxerMen', 511 | 'HeadlessChrome', 512 | 'HEADMasterSEO', 513 | 'HeartRails_Capture', 514 | 'help@dataminr\\.com', 515 | 'heritrix', 516 | 'Hexometer', 517 | 'historious', 518 | 'hkedcity', 519 | 'hledejLevne\\.cz', 520 | 'Hloader', 521 | 'HMView', 522 | 'Holmes', 523 | 'HonesoSearchEngine', 524 | 'HootSuite Image proxy', 525 | 'Hootsuite-WebFeed', 526 | 'hosterstats', 527 | 'HostTracker', 528 | 'ht:\\/\\/check', 529 | 'htdig', 530 | 'HTMLparser', 531 | 'htmlyse', 532 | 'HTTP Banner Detection', 533 | 'http-get', 534 | 'HTTP-Header-Abfrage', 535 | 'http-kit', 536 | 'http-request\\/', 537 | 'HTTP-Tiny', 538 | 'HTTP::Lite', 539 | 'http:\\/\\/www.neomo.de\\/', 540 | 'HttpComponents', 541 | 'httphr', 542 | 'HTTPie', 543 | 'HTTPMon', 544 | 'httpRequest', 545 | 'httpscheck', 546 | 'httpssites_power', 547 | 'httpunit', 548 | 'HttpUrlConnection', 549 | 'http\\.rb\\/', 550 | 'HTTP_Compression_Test', 551 | 'http_get', 552 | 'http_request2', 553 | 'http_requester', 554 | 'httrack', 555 | 'huaweisymantec', 556 | 'HubSpot ', 557 | 'HubSpot-Link-Resolver', 558 | 'Humanlinks', 559 | 'i2kconnect\\/', 560 | 'Iblog', 561 | 'ichiro', 562 | 'Id-search', 563 | 'IdeelaborPlagiaat', 564 | 'IDG Twitter Links Resolver', 565 | 'IDwhois\\/', 566 | 'Iframely', 567 | 'igdeSpyder', 568 | 'iGooglePortal', 569 | 'IlTrovatore', 570 | 'Image Fetch', 571 | 'Image Sucker', 572 | 'ImageEngine\\/', 573 | 'ImageVisu\\/', 574 | 'Imagga', 575 | 'imagineeasy', 576 | 'imgsizer', 577 | 'InAGist', 578 | 'inbound\\.li parser', 579 | 'InDesign%20CC', 580 | 'Indy Library', 581 | 'InetURL', 582 | 'infegy', 583 | 'infohelfer', 584 | 'InfoTekies', 585 | 'InfoWizards Reciprocal Link', 586 | 'inpwrd\\.com', 587 | 'instabid', 588 | 'Instapaper', 589 | 'Integrity', 590 | 'integromedb', 591 | 'Intelliseek', 592 | 'InterGET', 593 | 'Internet Ninja', 594 | 'InternetSeer', 595 | 'internetVista monitor', 596 | 'internetwache', 597 | 'internet_archive', 598 | 'intraVnews', 599 | 'IODC', 600 | 'IOI', 601 | 'iplabel', 602 | 'ips-agent', 603 | 'IPS\\/[0-9]', 604 | 'IPWorks HTTP\\/S Component', 605 | 'iqdb\\/', 606 | 'Iria', 607 | 'Irokez', 608 | 'isitup\\.org', 609 | 'iskanie', 610 | 'isUp\\.li', 611 | 'iThemes Sync\\/', 612 | 'IZaBEE', 613 | 'iZSearch', 614 | 'JAHHO', 615 | 'janforman', 616 | 'Jaunt\\/', 617 | 'Java.*outbrain', 618 | 'javelin\\.io', 619 | 'Jbrofuzz', 620 | 'Jersey\\/', 621 | 'JetCar', 622 | 'Jigsaw', 623 | 'Jobboerse', 624 | 'JobFeed discovery', 625 | 'Jobg8 URL Monitor', 626 | 'jobo', 627 | 'Jobrapido', 628 | 'Jobsearch1\\.5', 629 | 'JoinVision Generic', 630 | 'JolokiaPwn', 631 | 'Joomla', 632 | 'Jorgee', 633 | 'JS-Kit', 634 | 'JungleKeyThumbnail', 635 | 'JustView', 636 | 'Kaspersky Lab CFR link resolver', 637 | 'Kelny\\/', 638 | 'Kerrigan\\/', 639 | 'KeyCDN', 640 | 'Keyword Density', 641 | 'Keywords Research', 642 | 'khttp\\/', 643 | 'KickFire', 644 | 'KimonoLabs\\/', 645 | 'Kml-Google', 646 | 'knows\\.is', 647 | 'KOCMOHABT', 648 | 'kouio', 649 | 'kube-probe', 650 | 'kubectl', 651 | 'kulturarw3', 652 | 'KumKie', 653 | 'Larbin', 654 | 'Lavf\\/', 655 | 'leakix\\.net', 656 | 'LeechFTP', 657 | 'LeechGet', 658 | 'letsencrypt', 659 | 'Lftp', 660 | 'LibVLC', 661 | 'LibWeb', 662 | 'Libwhisker', 663 | 'libwww', 664 | 'Licorne', 665 | 'Liferea\\/', 666 | 'Lighthouse', 667 | 'Lightspeedsystems', 668 | 'Likse', 669 | 'limber\\.io', 670 | 'Link Valet', 671 | 'LinkAlarm\\/', 672 | 'LinkAnalyser', 673 | 'linkCheck', 674 | 'linkdex', 675 | 'LinkExaminer', 676 | 'linkfluence', 677 | 'linkpeek', 678 | 'LinkPreview', 679 | 'LinkScan', 680 | 'LinksManager', 681 | 'LinkTiger', 682 | 'LinkWalker', 683 | 'link_thumbnailer', 684 | 'Lipperhey', 685 | 'Litemage_walker', 686 | 'livedoor ScreenShot', 687 | 'LoadImpactRload', 688 | 'localsearch-web', 689 | 'LongURL API', 690 | 'longurl-r-package', 691 | 'looid\\.com', 692 | 'looksystems\\.net', 693 | 'ltx71', 694 | 'lua-resty-http', 695 | 'Lucee \\(CFML Engine\\)', 696 | 'Lush Http Client', 697 | 'lwp-request', 698 | 'lwp-trivial', 699 | 'LWP::Simple', 700 | 'lycos', 701 | 'LYT\\.SR', 702 | 'L\\.webis', 703 | 'mabontland', 704 | 'MacOutlook\\/', 705 | 'Mag-Net', 706 | 'MagpieRSS', 707 | 'Mail::STS', 708 | 'MailChimp', 709 | 'Mail\\.Ru', 710 | 'Majestic12', 711 | 'makecontact\\/', 712 | 'Mandrill', 713 | 'MapperCmd', 714 | 'marketinggrader', 715 | 'MarkMonitor', 716 | 'MarkWatch', 717 | 'Mass Downloader', 718 | 'masscan\\/', 719 | 'Mata Hari', 720 | 'mattermost', 721 | 'Mediametric', 722 | 'Mediapartners-Google', 723 | 'mediawords', 724 | 'MegaIndex\\.ru', 725 | 'MeltwaterNews', 726 | 'Melvil Rawi', 727 | 'MemGator', 728 | 'Metaspinner', 729 | 'MetaURI', 730 | 'MFC_Tear_Sample', 731 | 'Microsearch', 732 | 'Microsoft Data Access', 733 | 'Microsoft Office', 734 | 'Microsoft Outlook', 735 | 'Microsoft Windows Network Diagnostics', 736 | 'Microsoft-WebDAV-MiniRedir', 737 | 'Microsoft\\.Data\\.Mashup', 738 | 'MIDown tool', 739 | 'MIIxpc', 740 | 'Mindjet', 741 | 'Miniature\\.io', 742 | 'Miniflux', 743 | 'mio_httpc', 744 | 'Miro-HttpClient', 745 | 'Mister PiX', 746 | 'mixdata dot com', 747 | 'mixed-content-scan', 748 | 'mixnode', 749 | 'Mnogosearch', 750 | 'mogimogi', 751 | 'Mojeek', 752 | 'Mojolicious \\(Perl\\)', 753 | 'monitis', 754 | 'Monitority\\/', 755 | 'Monit\\/', 756 | 'montastic', 757 | 'MonTools', 758 | 'Moreover', 759 | 'Morfeus Fucking Scanner', 760 | 'Morning Paper', 761 | 'MovableType', 762 | 'mowser', 763 | 'Mrcgiguy', 764 | 'Mr\\.4x3 Powered', 765 | 'MS Web Services Client Protocol', 766 | 'MSFrontPage', 767 | 'mShots', 768 | 'MuckRack\\/', 769 | 'muhstik-scan', 770 | 'MVAClient', 771 | 'MxToolbox\\/', 772 | 'myseosnapshot', 773 | 'nagios', 774 | 'Najdi\\.si', 775 | 'Name Intelligence', 776 | 'NameFo\\.com', 777 | 'Nameprotect', 778 | 'nationalarchives', 779 | 'Navroad', 780 | 'NearSite', 781 | 'Needle', 782 | 'Nessus', 783 | 'Net Vampire', 784 | 'NetAnts', 785 | 'NETCRAFT', 786 | 'NetLyzer', 787 | 'NetMechanic', 788 | 'NetNewsWire', 789 | 'Netpursual', 790 | 'netresearch', 791 | 'NetShelter ContentScan', 792 | 'Netsparker', 793 | 'NetSystemsResearch', 794 | 'nettle', 795 | 'NetTrack', 796 | 'Netvibes', 797 | 'NetZIP', 798 | 'Neustar WPM', 799 | 'NeutrinoAPI', 800 | 'NewRelicPinger', 801 | 'NewsBlur .*Finder', 802 | 'NewsGator', 803 | 'newsme', 804 | 'newspaper\\/', 805 | 'Nexgate Ruby Client', 806 | 'NG-Search', 807 | 'nghttp2', 808 | 'Nibbler', 809 | 'NICErsPRO', 810 | 'NihilScio', 811 | 'Nikto', 812 | 'nineconnections', 813 | 'NLNZ_IAHarvester', 814 | 'Nmap Scripting Engine', 815 | 'node-fetch', 816 | 'node-superagent', 817 | 'node-urllib', 818 | 'Nodemeter', 819 | 'NodePing', 820 | 'node\\.io', 821 | 'nominet\\.org\\.uk', 822 | 'nominet\\.uk', 823 | 'Norton-Safeweb', 824 | 'Notifixious', 825 | 'notifyninja', 826 | 'NotionEmbedder', 827 | 'nuhk', 828 | 'nutch', 829 | 'Nuzzel', 830 | 'nWormFeedFinder', 831 | 'nyawc\\/', 832 | 'Nymesis', 833 | 'NYU', 834 | 'Observatory\\/', 835 | 'Ocelli\\/', 836 | 'Octopus', 837 | 'oegp', 838 | 'Offline Explorer', 839 | 'Offline Navigator', 840 | 'OgScrper', 841 | 'okhttp', 842 | 'omgili', 843 | 'OMSC', 844 | 'Online Domain Tools', 845 | 'Open Source RSS', 846 | 'OpenCalaisSemanticProxy', 847 | 'Openfind', 848 | 'OpenLinkProfiler', 849 | 'Openstat\\/', 850 | 'OpenVAS', 851 | 'OPPO A33', 852 | 'Optimizer', 853 | 'Orbiter', 854 | 'OrgProbe\\/', 855 | 'orion-semantics', 856 | 'Outlook-Express', 857 | 'Outlook-iOS', 858 | 'Owler', 859 | 'Owlin', 860 | 'ownCloud News', 861 | 'ow\\.ly', 862 | 'OxfordCloudService', 863 | 'page scorer', 864 | 'Page Valet', 865 | 'page2rss', 866 | 'PageFreezer', 867 | 'PageGrabber', 868 | 'PagePeeker', 869 | 'PageScorer', 870 | 'Pagespeed\\/', 871 | 'PageThing', 872 | 'page_verifier', 873 | 'Panopta', 874 | 'panscient', 875 | 'Papa Foto', 876 | 'parsijoo', 877 | 'Pavuk', 878 | 'PayPal IPN', 879 | 'pcBrowser', 880 | 'Pcore-HTTP', 881 | 'PDF24 URL To PDF', 882 | 'Pearltrees', 883 | 'PECL::HTTP', 884 | 'peerindex', 885 | 'Peew', 886 | 'PeoplePal', 887 | 'Perlu -', 888 | 'PhantomJS Screenshoter', 889 | 'PhantomJS\\/', 890 | 'Photon\\/', 891 | 'php-requests', 892 | 'phpservermon', 893 | 'Pi-Monster', 894 | 'Picscout', 895 | 'Picsearch', 896 | 'PictureFinder', 897 | 'Pimonster', 898 | 'Pingability', 899 | 'PingAdmin\\.Ru', 900 | 'Pingdom', 901 | 'Pingoscope', 902 | 'PingSpot', 903 | 'ping\\.blo\\.gs', 904 | 'pinterest\\.com', 905 | 'Pixray', 906 | 'Pizilla', 907 | 'Plagger\\/', 908 | 'Pleroma ', 909 | 'Ploetz \\+ Zeller', 910 | 'Plukkie', 911 | 'plumanalytics', 912 | 'PocketImageCache', 913 | 'PocketParser', 914 | 'Pockey', 915 | 'PodcastAddict\\/', 916 | 'POE-Component-Client-HTTP', 917 | 'Polymail\\/', 918 | 'Pompos', 919 | 'Porkbun', 920 | 'Port Monitor', 921 | 'postano', 922 | 'postfix-mta-sts-resolver', 923 | 'PostmanRuntime', 924 | 'postplanner\\.com', 925 | 'PostPost', 926 | 'postrank', 927 | 'PowerPoint\\/', 928 | 'Prebid', 929 | 'Prerender', 930 | 'Priceonomics Analysis Engine', 931 | 'PrintFriendly', 932 | 'PritTorrent', 933 | 'Prlog', 934 | 'probethenet', 935 | 'Project ?25499', 936 | 'Project-Resonance', 937 | 'prospectb2b', 938 | 'Protopage', 939 | 'ProWebWalker', 940 | 'proximic', 941 | 'PRTG Network Monitor', 942 | 'pshtt, https scanning', 943 | 'PTST ', 944 | 'PTST\\/[0-9]+', 945 | 'Pump', 946 | 'Python-httplib2', 947 | 'python-httpx', 948 | 'python-requests', 949 | 'Python-urllib', 950 | 'Qirina Hurdler', 951 | 'QQDownload', 952 | 'QrafterPro', 953 | 'Qseero', 954 | 'Qualidator', 955 | 'QueryN Metasearch', 956 | 'queuedriver', 957 | 'quic-go-HTTP\\/', 958 | 'QuiteRSS', 959 | 'Quora Link Preview', 960 | 'Qwantify', 961 | 'Radian6', 962 | 'RadioPublicImageResizer', 963 | 'Railgun\\/', 964 | 'RankActive', 965 | 'RankFlex', 966 | 'RankSonicSiteAuditor', 967 | 'RapidLoad\\/', 968 | 'Re-re Studio', 969 | 'ReactorNetty', 970 | 'Readability', 971 | 'RealDownload', 972 | 'RealPlayer%20Downloader', 973 | 'RebelMouse', 974 | 'Recorder', 975 | 'RecurPost\\/', 976 | 'redback\\/', 977 | 'ReederForMac', 978 | 'Reeder\\/', 979 | 'ReGet', 980 | 'RepoMonkey', 981 | 'request\\.js', 982 | 'reqwest\\/', 983 | 'ResponseCodeTest', 984 | 'RestSharp', 985 | 'Riddler', 986 | 'Rival IQ', 987 | 'Robosourcer', 988 | 'Robozilla', 989 | 'ROI Hunter', 990 | 'RPT-HTTPClient', 991 | 'RSSMix\\/', 992 | 'RSSOwl', 993 | 'RyowlEngine', 994 | 'safe-agent-scanner', 995 | 'SalesIntelligent', 996 | 'Saleslift', 997 | 'SAP NetWeaver Application Server', 998 | 'SauceNAO', 999 | 'SBIder', 1000 | 'sc-downloader', 1001 | 'scalaj-http', 1002 | 'Scamadviser-Frontend', 1003 | 'ScanAlert', 1004 | 'scan\\.lol', 1005 | 'Scoop', 1006 | 'scooter', 1007 | 'ScopeContentAG-HTTP-Client', 1008 | 'ScoutJet', 1009 | 'ScoutURLMonitor', 1010 | 'ScrapeBox Page Scanner', 1011 | 'Scrapy', 1012 | 'Screaming', 1013 | 'ScreenShotService', 1014 | 'Scrubby', 1015 | 'Scrutiny\\/', 1016 | 'Search37', 1017 | 'searchenginepromotionhelp', 1018 | 'Searchestate', 1019 | 'SearchExpress', 1020 | 'SearchSight', 1021 | 'SearchWP', 1022 | 'search\\.thunderstone', 1023 | 'Seeker', 1024 | 'semanticdiscovery', 1025 | 'semanticjuice', 1026 | 'Semiocast HTTP client', 1027 | 'Semrush', 1028 | 'Sendsay\\.Ru', 1029 | 'sentry\\/', 1030 | 'SEO Browser', 1031 | 'Seo Servis', 1032 | 'seo-nastroj\\.cz', 1033 | 'seo4ajax', 1034 | 'Seobility', 1035 | 'SEOCentro', 1036 | 'SeoCheck', 1037 | 'SEOkicks', 1038 | 'SEOlizer', 1039 | 'Seomoz', 1040 | 'SEOprofiler', 1041 | 'seoscanners', 1042 | 'SEOsearch', 1043 | 'seositecheckup', 1044 | 'SEOstats', 1045 | 'servernfo', 1046 | 'sexsearcher', 1047 | 'Seznam', 1048 | 'Shelob', 1049 | 'Shodan', 1050 | 'Shoppimon', 1051 | 'ShopWiki', 1052 | 'ShortLinkTranslate', 1053 | 'shortURL lengthener', 1054 | 'shrinktheweb', 1055 | 'Sideqik', 1056 | 'Siege', 1057 | 'SimplePie', 1058 | 'SimplyFast', 1059 | 'Siphon', 1060 | 'SISTRIX', 1061 | 'Site Sucker', 1062 | 'Site-Shot\\/', 1063 | 'Site24x7', 1064 | 'SiteBar', 1065 | 'Sitebeam', 1066 | 'Sitebulb\\/', 1067 | 'SiteCondor', 1068 | 'SiteExplorer', 1069 | 'SiteGuardian', 1070 | 'Siteimprove', 1071 | 'SiteIndexed', 1072 | 'Sitemap(s)? Generator', 1073 | 'SitemapGenerator', 1074 | 'SiteMonitor', 1075 | 'Siteshooter B0t', 1076 | 'SiteSnagger', 1077 | 'SiteSucker', 1078 | 'SiteTruth', 1079 | 'Sitevigil', 1080 | 'sitexy\\.com', 1081 | 'SkypeUriPreview', 1082 | 'Slack\\/', 1083 | 'sli-systems\\.com', 1084 | 'slider\\.com', 1085 | 'slurp', 1086 | 'SlySearch', 1087 | 'SmartDownload', 1088 | 'SMRF URL Expander', 1089 | 'SMUrlExpander', 1090 | 'Snake', 1091 | 'Snappy', 1092 | 'SnapSearch', 1093 | 'Snarfer\\/', 1094 | 'SniffRSS', 1095 | 'sniptracker', 1096 | 'Snoopy', 1097 | 'SnowHaze Search', 1098 | 'sogou web', 1099 | 'SortSite', 1100 | 'Sottopop', 1101 | 'sovereign\\.ai', 1102 | 'SpaceBison', 1103 | 'SpamExperts', 1104 | 'Spammen', 1105 | 'Spanner', 1106 | 'spaziodati', 1107 | 'SPDYCheck', 1108 | 'Specificfeeds', 1109 | 'speedy', 1110 | 'SPEng', 1111 | 'Spinn3r', 1112 | 'spray-can', 1113 | 'Sprinklr ', 1114 | 'spyonweb', 1115 | 'sqlmap', 1116 | 'Sqlworm', 1117 | 'Sqworm', 1118 | 'SSL Labs', 1119 | 'ssl-tools', 1120 | 'StackRambler', 1121 | 'Statastico\\/', 1122 | 'Statically-', 1123 | 'StatusCake', 1124 | 'Steeler', 1125 | 'Stratagems Kumo', 1126 | 'Stripe\\/', 1127 | 'Stroke\\.cz', 1128 | 'StudioFACA', 1129 | 'StumbleUpon', 1130 | 'suchen', 1131 | 'Sucuri', 1132 | 'summify', 1133 | 'SuperHTTP', 1134 | 'Surphace Scout', 1135 | 'Suzuran', 1136 | 'swcd ', 1137 | 'Symfony BrowserKit', 1138 | 'Symfony2 BrowserKit', 1139 | 'Synapse\\/', 1140 | 'Syndirella\\/', 1141 | 'SynHttpClient-Built', 1142 | 'Sysomos', 1143 | 'sysscan', 1144 | 'Szukacz', 1145 | 'T0PHackTeam', 1146 | 'tAkeOut', 1147 | 'Tarantula\\/', 1148 | 'Taringa UGC', 1149 | 'TarmotGezgin', 1150 | 'tchelebi\\.io', 1151 | 'techiaith\\.cymru', 1152 | 'TelegramBot', 1153 | 'Teleport', 1154 | 'Telesoft', 1155 | 'Telesphoreo', 1156 | 'Telesphorep', 1157 | 'Tenon\\.io', 1158 | 'teoma', 1159 | 'terrainformatica', 1160 | 'Test Certificate Info', 1161 | 'testuri', 1162 | 'Tetrahedron', 1163 | 'TextRazor Downloader', 1164 | 'The Drop Reaper', 1165 | 'The Expert HTML Source Viewer', 1166 | 'The Intraformant', 1167 | 'The Knowledge AI', 1168 | 'theinternetrules', 1169 | 'TheNomad', 1170 | 'Thinklab', 1171 | 'Thumbor', 1172 | 'Thumbshots', 1173 | 'ThumbSniper', 1174 | 'timewe\\.net', 1175 | 'TinEye', 1176 | 'Tiny Tiny RSS', 1177 | 'TLSProbe\\/', 1178 | 'Toata', 1179 | 'topster', 1180 | 'touche\\.com', 1181 | 'Traackr\\.com', 1182 | 'tracemyfile', 1183 | 'Trackuity', 1184 | 'TrapitAgent', 1185 | 'Trendiction', 1186 | 'Trendsmap', 1187 | 'trendspottr', 1188 | 'truwoGPS', 1189 | 'TryJsoup', 1190 | 'TulipChain', 1191 | 'Turingos', 1192 | 'Turnitin', 1193 | 'tweetedtimes', 1194 | 'Tweetminster', 1195 | 'Tweezler\\/', 1196 | 'twibble', 1197 | 'Twice', 1198 | 'Twikle', 1199 | 'Twingly', 1200 | 'Twisted PageGetter', 1201 | 'Typhoeus', 1202 | 'ubermetrics-technologies', 1203 | 'uclassify', 1204 | 'UdmSearch', 1205 | 'ultimate_sitemap_parser', 1206 | 'unchaos', 1207 | 'unirest-java', 1208 | 'UniversalFeedParser', 1209 | 'unshortenit', 1210 | 'Unshorten\\.It', 1211 | 'Untiny', 1212 | 'UnwindFetchor', 1213 | 'updated', 1214 | 'updown\\.io daemon', 1215 | 'Upflow', 1216 | 'Uptimia', 1217 | 'URL Verifier', 1218 | 'Urlcheckr', 1219 | 'URLitor', 1220 | 'urlresolver', 1221 | 'Urlstat', 1222 | 'URLTester', 1223 | 'UrlTrends Ranking Updater', 1224 | 'URLy Warning', 1225 | 'URLy\\.Warning', 1226 | 'URL\\/Emacs', 1227 | 'Vacuum', 1228 | 'Vagabondo', 1229 | 'VB Project', 1230 | 'vBSEO', 1231 | 'VCI', 1232 | 'via ggpht\\.com GoogleImageProxy', 1233 | 'Virusdie', 1234 | 'visionutils', 1235 | 'vkShare', 1236 | 'VoidEYE', 1237 | 'Voil', 1238 | 'voltron', 1239 | 'voyager\\/', 1240 | 'VSAgent\\/', 1241 | 'VSB-TUO\\/', 1242 | 'Vulnbusters Meter', 1243 | 'VYU2', 1244 | 'w3af\\.org', 1245 | 'W3C-checklink', 1246 | 'W3C-mobileOK', 1247 | 'W3C_Unicorn', 1248 | 'WAC-OFU', 1249 | 'WakeletLinkExpander', 1250 | 'WallpapersHD', 1251 | 'Wallpapers\\/[0-9]+', 1252 | 'wangling', 1253 | 'Wappalyzer', 1254 | 'WatchMouse', 1255 | 'WbSrch\\/', 1256 | 'WDT\\.io', 1257 | 'Web Auto', 1258 | 'Web Collage', 1259 | 'Web Enhancer', 1260 | 'Web Fetch', 1261 | 'Web Fuck', 1262 | 'Web Pix', 1263 | 'Web Sauger', 1264 | 'Web spyder', 1265 | 'Web Sucker', 1266 | 'web-capture\\.net', 1267 | 'Web-sniffer', 1268 | 'Webalta', 1269 | 'Webauskunft', 1270 | 'WebAuto', 1271 | 'WebCapture', 1272 | 'WebClient\\/', 1273 | 'webcollage', 1274 | 'WebCookies', 1275 | 'WebCopier', 1276 | 'WebCorp', 1277 | 'WebDataStats', 1278 | 'WebDoc', 1279 | 'WebEnhancer', 1280 | 'WebFetch', 1281 | 'WebFuck', 1282 | 'WebGazer', 1283 | 'WebGo IS', 1284 | 'WebImageCollector', 1285 | 'WebImages', 1286 | 'WebIndex', 1287 | 'webkit2png', 1288 | 'WebLeacher', 1289 | 'webmastercoffee', 1290 | 'webmon ', 1291 | 'WebPix', 1292 | 'WebReaper', 1293 | 'WebSauger', 1294 | 'webscreenie', 1295 | 'Webshag', 1296 | 'Webshot', 1297 | 'Website Quester', 1298 | 'websitepulse agent', 1299 | 'WebsiteQuester', 1300 | 'Websnapr', 1301 | 'WebSniffer', 1302 | 'Webster', 1303 | 'WebStripper', 1304 | 'WebSucker', 1305 | 'webtech\\/', 1306 | 'WebThumbnail', 1307 | 'Webthumb\\/', 1308 | 'WebWhacker', 1309 | 'WebZIP', 1310 | 'WeLikeLinks', 1311 | 'WEPA', 1312 | 'WeSEE', 1313 | 'wf84', 1314 | 'Wfuzz\\/', 1315 | 'wget', 1316 | 'WhatCMS', 1317 | 'WhatsApp', 1318 | 'WhatsMyIP', 1319 | 'WhatWeb', 1320 | 'WhereGoes\\?', 1321 | 'Whibse', 1322 | 'WhoAPI\\/', 1323 | 'WhoRunsCoinHive', 1324 | 'Whynder Magnet', 1325 | 'Windows-RSS-Platform', 1326 | 'WinHttp-Autoproxy-Service', 1327 | 'WinHTTP\\/', 1328 | 'WinPodder', 1329 | 'wkhtmlto', 1330 | 'wmtips', 1331 | 'Woko', 1332 | 'Wolfram HTTPClient', 1333 | 'woorankreview', 1334 | 'WordPress\\/', 1335 | 'WordupinfoSearch', 1336 | 'Word\\/', 1337 | 'worldping-api', 1338 | 'wotbox', 1339 | 'WP Engine Install Performance API', 1340 | 'WP Rocket', 1341 | 'wpif', 1342 | 'wprecon\\.com survey', 1343 | 'WPScan', 1344 | 'wscheck', 1345 | 'Wtrace', 1346 | 'WWW-Collector-E', 1347 | 'WWW-Mechanize', 1348 | 'WWW::Document', 1349 | 'WWW::Mechanize', 1350 | 'WWWOFFLE', 1351 | 'www\\.monitor\\.us', 1352 | 'x09Mozilla', 1353 | 'x22Mozilla', 1354 | 'XaxisSemanticsClassifier', 1355 | 'XenForo\\/', 1356 | 'Xenu Link Sleuth', 1357 | 'XING-contenttabreceiver', 1358 | 'xpymep([0-9]?)\\.exe', 1359 | 'Y!J-[A-Z][A-Z][A-Z]', 1360 | 'Yaanb', 1361 | 'yacy', 1362 | 'Yahoo Link Preview', 1363 | 'YahooCacheSystem', 1364 | 'YahooMailProxy', 1365 | 'YahooYSMcm', 1366 | 'YandeG', 1367 | 'Yandex(?!Search)', 1368 | 'yanga', 1369 | 'yeti', 1370 | 'Yo-yo', 1371 | 'Yoleo Consumer', 1372 | 'yomins\\.com', 1373 | 'yoogliFetchAgent', 1374 | 'YottaaMonitor', 1375 | 'Your-Website-Sucks', 1376 | 'yourls\\.org', 1377 | 'YoYs\\.net', 1378 | 'YP\\.PL', 1379 | 'Zabbix', 1380 | 'Zade', 1381 | 'Zao', 1382 | 'Zauba', 1383 | 'Zemanta Aggregator', 1384 | 'Zend\\\\Http\\\\Client', 1385 | 'Zend_Http_Client', 1386 | 'Zermelo', 1387 | 'Zeus ', 1388 | 'zgrab', 1389 | 'ZnajdzFoto', 1390 | 'ZnHTTP', 1391 | 'Zombie\\.js', 1392 | 'Zoom\\.Mac', 1393 | 'ZoteroTranslationServer', 1394 | 'ZyBorg', 1395 | '[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)', 1396 | ]; 1397 | } 1398 | } 1399 | exports.Crawlers = Crawlers; 1400 | //# sourceMappingURL=crawlers.js.map -------------------------------------------------------------------------------- /dist/lib/crawler/crawlers.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"crawlers.js","sourceRoot":"","sources":["../../../src/lib/crawler/crawlers.ts"],"names":[],"mappings":";;;AAEA,MAAa,QAAQ;IACZ,MAAM;QACX,OAAO;YACL,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,sCAAsC;YACtC,OAAO;YACP,YAAY;YACZ,eAAe;YACf,aAAa;YACb,QAAQ;YACR,WAAW;YACX,UAAU;YACV,OAAO;YACP,iBAAiB;YACjB,UAAU;YACV,OAAO;YACP,SAAS;YACT,WAAW;YACX,UAAU;YACV,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,wBAAwB;YACxB,cAAc;YACd,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,SAAS;YACT,SAAS;YACT,gBAAgB;YAChB,SAAS;YACT,SAAS;YACT,UAAU;YACV,SAAS;YACT,oBAAoB;YACpB,SAAS;YACT,cAAc;YACd,WAAW;YACX,OAAO;YACP,cAAc;YACd,kBAAkB;YAClB,cAAc;YACd,OAAO;YACP,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,UAAU;YACV,SAAS;YACT,QAAQ;YACR,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kBAAkB;YAClB,uBAAuB;YACvB,OAAO;YACP,0BAA0B;YAC1B,UAAU;YACV,WAAW;YACX,QAAQ;YACR,SAAS;YACT,SAAS;YACT,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,WAAW;YACX,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,kBAAkB;YAClB,kBAAkB;YAClB,sBAAsB;YACtB,WAAW;YACX,KAAK;YACL,OAAO;YACP,YAAY;YACZ,SAAS;YACT,cAAc;YACd,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,+BAA+B;YAC/B,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,QAAQ;YACR,UAAU;YACV,wBAAwB;YACxB,SAAS;YACT,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,eAAe;YACf,WAAW;YACX,kBAAkB;YAClB,eAAe;YACf,cAAc;YACd,mBAAmB;YACnB,wBAAwB;YACxB,mBAAmB;YACnB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,aAAa;YACb,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,kBAAkB;YAClB,cAAc;YACd,SAAS;YACT,WAAW;YACX,eAAe;YACf,OAAO;YACP,WAAW;YACX,UAAU;YACV,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,WAAW;YACX,UAAU;YACV,mBAAmB;YACnB,UAAU;YACV,YAAY;YACZ,sBAAsB;YACtB,UAAU;YACV,gBAAgB;YAChB,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,kBAAkB;YAClB,QAAQ;YACR,aAAa;YACb,QAAQ;YACR,YAAY;YACZ,UAAU;YACV,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,sBAAsB;YACtB,YAAY;YACZ,SAAS;YACT,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,QAAQ;YACR,mBAAmB;YACnB,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,0BAA0B;YAC1B,YAAY;YACZ,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,WAAW;YACX,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,oBAAoB;YACpB,oBAAoB;YACpB,oBAAoB;YACpB,cAAc;YACd,aAAa;YACb,YAAY;YACZ,WAAW;YACX,cAAc;YACd,QAAQ;YACR,SAAS;YACT,OAAO;YACP,WAAW;YACX,UAAU;YACV,eAAe;YACf,aAAa;YACb,cAAc;YACd,UAAU;YACV,cAAc;YACd,SAAS;YACT,UAAU;YACV,eAAe;YACf,WAAW;YACX,eAAe;YACf,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,4BAA4B;YAC5B,iBAAiB;YACjB,6BAA6B;YAC7B,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,cAAc;YACd,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,WAAW;YACX,UAAU;YACV,aAAa;YACb,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kCAAkC;YAClC,oBAAoB;YACpB,QAAQ;YACR,kBAAkB;YAClB,YAAY;YACZ,sBAAsB;YACtB,eAAe;YACf,aAAa;YACb,yBAAyB;YACzB,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,YAAY;YACZ,SAAS;YACT,WAAW;YACX,oBAAoB;YACpB,YAAY;YACZ,kBAAkB;YAClB,gBAAgB;YAChB,gCAAgC;YAChC,SAAS;YACT,eAAe;YACf,mBAAmB;YACnB,gBAAgB;YAChB,WAAW;YACX,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,UAAU;YACV,SAAS;YACT,MAAM;YACN,gBAAgB;YAChB,MAAM;YACN,YAAY;YACZ,YAAY;YACZ,eAAe;YACf,WAAW;YACX,sBAAsB;YACtB,WAAW;YACX,eAAe;YACf,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,sBAAsB;YACtB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,OAAO;YACP,OAAO;YACP,oDAAoD;YACpD,OAAO;YACP,MAAM;YACN,WAAW;YACX,eAAe;YACf,WAAW;YACX,wBAAwB;YACxB,aAAa;YACb,WAAW;YACX,aAAa;YACb,MAAM;YACN,WAAW;YACX,kBAAkB;YAClB,SAAS;YACT,qBAAqB;YACrB,gBAAgB;YAChB,YAAY;YACZ,oBAAoB;YACpB,yBAAyB;YACzB,6BAA6B;YAC7B,aAAa;YACb,yBAAyB;YACzB,iBAAiB;YACjB,cAAc;YACd,kBAAkB;YAClB,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,oBAAoB;YACpB,UAAU;YACV,YAAY;YACZ,QAAQ;YACR,WAAW;YACX,eAAe;YACf,gBAAgB;YAChB,QAAQ;YACR,MAAM;YACN,aAAa;YACb,MAAM;YACN,YAAY;YACZ,cAAc;YACd,SAAS;YACT,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,aAAa;YACb,mBAAmB;YACnB,SAAS;YACT,SAAS;YACT,oBAAoB;YACpB,WAAW;YACX,yBAAyB;YACzB,wBAAwB;YACxB,QAAQ;YACR,SAAS;YACT,aAAa;YACb,kBAAkB;YAClB,UAAU;YACV,MAAM;YACN,SAAS;YACT,aAAa;YACb,qBAAqB;YACrB,wBAAwB;YACxB,UAAU;YACV,QAAQ;YACR,qBAAqB;YACrB,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,oBAAoB;YACpB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,eAAe;YACf,aAAa;YACb,SAAS;YACT,aAAa;YACb,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,aAAa;YACb,UAAU;YACV,eAAe;YACf,aAAa;YACb,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,eAAe;YACf,eAAe;YACf,QAAQ;YACR,eAAe;YACf,WAAW;YACX,OAAO;YACP,UAAU;YACV,cAAc;YACd,UAAU;YACV,uBAAuB;YACvB,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,UAAU;YACV,cAAc;YACd,iBAAiB;YACjB,cAAc;YACd,UAAU;YACV,YAAY;YACZ,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,cAAc;YACd,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,eAAe;YACf,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,qBAAqB;YACrB,UAAU;YACV,aAAa;YACb,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,2BAA2B;YAC3B,UAAU;YACV,iBAAiB;YACjB,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,mBAAmB;YACnB,gBAAgB;YAChB,gBAAgB;YAChB,2BAA2B;YAC3B,qBAAqB;YACrB,4BAA4B;YAC5B,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,gCAAgC;YAChC,qBAAqB;YACrB,gBAAgB;YAChB,oBAAoB;YACpB,0BAA0B;YAC1B,yBAAyB;YACzB,yBAAyB;YACzB,gBAAgB;YAChB,yBAAyB;YACzB,mBAAmB;YACnB,sBAAsB;YACtB,0BAA0B;YAC1B,iBAAiB;YACjB,eAAe;YACf,qCAAqC;YACrC,4BAA4B;YAC5B,gBAAgB;YAChB,sBAAsB;YACtB,YAAY;YACZ,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,uBAAuB;YACvB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,WAAW;YACX,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,SAAS;YACT,WAAW;YACX,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,YAAY;YACZ,SAAS;YACT,2BAA2B;YAC3B,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,OAAO;YACP,UAAU;YACV,gBAAgB;YAChB,eAAe;YACf,oBAAoB;YACpB,qBAAqB;YACrB,UAAU;YACV,WAAW;YACX,YAAY;YACZ,UAAU;YACV,kBAAkB;YAClB,SAAS;YACT,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,uBAAuB;YACvB,mBAAmB;YACnB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,OAAO;YACP,YAAY;YACZ,SAAS;YACT,uBAAuB;YACvB,UAAU;YACV,qBAAqB;YACrB,UAAU;YACV,iBAAiB;YACjB,WAAW;YACX,YAAY;YACZ,4BAA4B;YAC5B,gBAAgB;YAChB,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,uBAAuB;YACvB,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,uBAAuB;YACvB,YAAY;YACZ,eAAe;YACf,OAAO;YACP,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,4BAA4B;YAC5B,YAAY;YACZ,UAAU;YACV,YAAY;YACZ,eAAe;YACf,aAAa;YACb,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,cAAc;YACd,SAAS;YACT,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,6BAA6B;YAC7B,cAAc;YACd,UAAU;YACV,YAAY;YACZ,WAAW;YACX,aAAa;YACb,aAAa;YACb,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,uBAAuB;YACvB,eAAe;YACf,kBAAkB;YAClB,YAAY;YACZ,MAAM;YACN,KAAK;YACL,SAAS;YACT,WAAW;YACX,aAAa;YACb,4BAA4B;YAC5B,SAAS;YACT,MAAM;YACN,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,QAAQ;YACR,UAAU;YACV,OAAO;YACP,WAAW;YACX,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,mBAAmB;YACnB,MAAM;YACN,WAAW;YACX,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,UAAU;YACV,iCAAiC;YACjC,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,OAAO;YACP,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,cAAc;YACd,UAAU;YACV,UAAU;YACV,aAAa;YACb,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,OAAO;YACP,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,WAAW;YACX,SAAS;YACT,cAAc;YACd,aAAa;YACb,UAAU;YACV,aAAa;YACb,UAAU;YACV,cAAc;YACd,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,iBAAiB;YACjB,qBAAqB;YACrB,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,mBAAmB;YACnB,aAAa;YACb,mBAAmB;YACnB,OAAO;YACP,gBAAgB;YAChB,yBAAyB;YACzB,kBAAkB;YAClB,aAAa;YACb,aAAa;YACb,aAAa;YACb,OAAO;YACP,UAAU;YACV,WAAW;YACX,YAAY;YACZ,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,aAAa;YACb,sBAAsB;YACtB,YAAY;YACZ,gBAAgB;YAChB,eAAe;YACf,aAAa;YACb,UAAU;YACV,aAAa;YACb,SAAS;YACT,iBAAiB;YACjB,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,mBAAmB;YACnB,uCAAuC;YACvC,4BAA4B;YAC5B,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,iBAAiB;YACjB,oBAAoB;YACpB,SAAS;YACT,aAAa;YACb,UAAU;YACV,QAAQ;YACR,wBAAwB;YACxB,SAAS;YACT,eAAe;YACf,UAAU;YACV,WAAW;YACX,UAAU;YACV,UAAU;YACV,yBAAyB;YACzB,eAAe;YACf,aAAa;YACb,QAAQ;YACR,UAAU;YACV,kBAAkB;YAClB,iCAAiC;YACjC,aAAa;YACb,QAAQ;YACR,aAAa;YACb,cAAc;YACd,WAAW;YACX,cAAc;YACd,eAAe;YACf,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,cAAc;YACd,aAAa;YACb,kBAAkB;YAClB,SAAS;YACT,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,aAAa;YACb,SAAS;YACT,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,aAAa;YACb,wBAAwB;YACxB,YAAY;YACZ,oBAAoB;YACpB,QAAQ;YACR,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,WAAW;YACX,QAAQ;YACR,cAAc;YACd,qBAAqB;YACrB,WAAW;YACX,SAAS;YACT,SAAS;YACT,WAAW;YACX,WAAW;YACX,OAAO;YACP,iBAAiB;YACjB,kBAAkB;YAClB,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,UAAU;YACV,WAAW;YACX,oBAAoB;YACpB,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,MAAM;YACN,OAAO;YACP,QAAQ;YACR,iBAAiB;YACjB,UAAU;YACV,SAAS;YACT,KAAK;YACL,gBAAgB;YAChB,WAAW;YACX,SAAS;YACT,MAAM;YACN,kBAAkB;YAClB,mBAAmB;YACnB,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,MAAM;YACN,qBAAqB;YACrB,iBAAiB;YACjB,yBAAyB;YACzB,UAAU;YACV,kBAAkB;YAClB,aAAa;YACb,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,aAAa;YACb,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,OAAO;YACP,OAAO;YACP,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,aAAa;YACb,YAAY;YACZ,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,YAAY;YACZ,cAAc;YACd,WAAW;YACX,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,MAAM;YACN,WAAW;YACX,SAAS;YACT,wBAAwB;YACxB,cAAc;YACd,WAAW;YACX,cAAc;YACd,cAAc;YACd,YAAY;YACZ,UAAU;YACV,WAAW;YACX,eAAe;YACf,WAAW;YACX,aAAa;YACb,gBAAgB;YAChB,SAAS;YACT,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,iBAAiB;YACjB,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,UAAU;YACV,mBAAmB;YACnB,SAAS;YACT,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,QAAQ;YACR,kBAAkB;YAClB,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,cAAc;YACd,SAAS;YACT,0BAA0B;YAC1B,gBAAgB;YAChB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,QAAQ;YACR,WAAW;YACX,8BAA8B;YAC9B,eAAe;YACf,aAAa;YACb,OAAO;YACP,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,aAAa;YACb,WAAW;YACX,cAAc;YACd,UAAU;YACV,sBAAsB;YACtB,uBAAuB;YACvB,OAAO;YACP,eAAe;YACf,MAAM;YACN,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,iBAAiB;YACjB,UAAU;YACV,oBAAoB;YACpB,UAAU;YACV,SAAS;YACT,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,sBAAsB;YACtB,cAAc;YACd,cAAc;YACd,cAAc;YACd,aAAa;YACb,cAAc;YACd,yBAAyB;YACzB,YAAY;YACZ,UAAU;YACV,cAAc;YACd,YAAY;YACZ,cAAc;YACd,WAAW;YACX,OAAO;YACP,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,SAAS;YACT,UAAU;YACV,aAAa;YACb,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,WAAW;YACX,QAAQ;YACR,aAAa;YACb,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,kCAAkC;YAClC,UAAU;YACV,QAAQ;YACR,eAAe;YACf,aAAa;YACb,sBAAsB;YACtB,WAAW;YACX,YAAY;YACZ,OAAO;YACP,SAAS;YACT,4BAA4B;YAC5B,UAAU;YACV,iBAAiB;YACjB,wBAAwB;YACxB,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,SAAS;YACT,aAAa;YACb,UAAU;YACV,2BAA2B;YAC3B,cAAc;YACd,eAAe;YACf,aAAa;YACb,UAAU;YACV,uBAAuB;YACvB,QAAQ;YACR,mBAAmB;YACnB,eAAe;YACf,uBAAuB;YACvB,SAAS;YACT,cAAc;YACd,WAAW;YACX,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,WAAW;YACX,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,UAAU;YACV,oBAAoB;YACpB,qBAAqB;YACrB,cAAc;YACd,SAAS;YACT,OAAO;YACP,WAAW;YACX,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,UAAU;YACV,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,aAAa;YACb,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,aAAa;YACb,iBAAiB;YACjB,aAAa;YACb,YAAY;YACZ,WAAW;YACX,WAAW;YACX,cAAc;YACd,iBAAiB;YACjB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,OAAO;YACP,WAAW;YACX,eAAe;YACf,mBAAmB;YACnB,eAAe;YACf,OAAO;YACP,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,UAAU;YACV,UAAU;YACV,gBAAgB;YAChB,YAAY;YACZ,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,WAAW;YACX,eAAe;YACf,QAAQ;YACR,OAAO;YACP,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,UAAU;YACV,WAAW;YACX,cAAc;YACd,eAAe;YACf,aAAa;YACb,YAAY;YACZ,SAAS;YACT,iBAAiB;YACjB,WAAW;YACX,aAAa;YACb,YAAY;YACZ,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,WAAW;YACX,gBAAgB;YAChB,SAAS;YACT,OAAO;YACP,oBAAoB;YACpB,qBAAqB;YACrB,YAAY;YACZ,eAAe;YACf,qBAAqB;YACrB,SAAS;YACT,SAAS;YACT,SAAS;YACT,aAAa;YACb,SAAS;YACT,cAAc;YACd,aAAa;YACb,cAAc;YACd,eAAe;YACf,mBAAmB;YACnB,aAAa;YACb,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,SAAS;YACT,aAAa;YACb,sBAAsB;YACtB,iBAAiB;YACjB,+BAA+B;YAC/B,kBAAkB;YAClB,kBAAkB;YAClB,kBAAkB;YAClB,UAAU;YACV,UAAU;YACV,SAAS;YACT,YAAY;YACZ,aAAa;YACb,cAAc;YACd,QAAQ;YACR,eAAe;YACf,aAAa;YACb,OAAO;YACP,SAAS;YACT,cAAc;YACd,eAAe;YACf,aAAa;YACb,WAAW;YACX,aAAa;YACb,aAAa;YACb,WAAW;YACX,aAAa;YACb,UAAU;YACV,UAAU;YACV,YAAY;YACZ,UAAU;YACV,UAAU;YACV,cAAc;YACd,cAAc;YACd,aAAa;YACb,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;YACT,oBAAoB;YACpB,UAAU;YACV,0BAA0B;YAC1B,WAAW;YACX,WAAW;YACX,yBAAyB;YACzB,SAAS;YACT,cAAc;YACd,qBAAqB;YACrB,aAAa;YACb,gBAAgB;YAChB,QAAQ;YACR,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,QAAQ;YACR,SAAS;YACT,cAAc;YACd,WAAW;YACX,SAAS;YACT,aAAa;YACb,SAAS;YACT,WAAW;YACX,2BAA2B;YAC3B,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,WAAW;YACX,YAAY;YACZ,OAAO;YACP,KAAK;YACL,kCAAkC;YAClC,UAAU;YACV,aAAa;YACb,SAAS;YACT,SAAS;YACT,MAAM;YACN,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,MAAM;YACN,YAAY;YACZ,eAAe;YACf,cAAc;YACd,aAAa;YACb,SAAS;YACT,qBAAqB;YACrB,cAAc;YACd,qBAAqB;YACrB,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,UAAU;YACV,UAAU;YACV,aAAa;YACb,cAAc;YACd,WAAW;YACX,UAAU;YACV,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,SAAS;YACT,aAAa;YACb,SAAS;YACT,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,UAAU;YACV,mBAAmB;YACnB,WAAW;YACX,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,QAAQ;YACR,WAAW;YACX,WAAW;YACX,aAAa;YACb,SAAS;YACT,SAAS;YACT,iBAAiB;YACjB,oBAAoB;YACpB,gBAAgB;YAChB,UAAU;YACV,YAAY;YACZ,SAAS;YACT,aAAa;YACb,WAAW;YACX,YAAY;YACZ,cAAc;YACd,aAAa;YACb,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,MAAM;YACN,OAAO;YACP,MAAM;YACN,UAAU;YACV,MAAM;YACN,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,WAAW;YACX,iBAAiB;YACjB,gBAAgB;YAChB,sBAAsB;YACtB,2BAA2B;YAC3B,YAAY;YACZ,WAAW;YACX,UAAU;YACV,QAAQ;YACR,MAAM;YACN,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,kBAAkB;YAClB,SAAS;YACT,eAAe;YACf,QAAQ;YACR,mCAAmC;YACnC,WAAW;YACX,MAAM;YACN,sBAAsB;YACtB,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,gBAAgB;YAChB,UAAU;YACV,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,0BAA0B;YAC1B,YAAY;YACZ,kBAAkB;YAClB,yBAAyB;YACzB,sBAAsB;YACtB,qBAAqB;YACrB,OAAO;YACP,MAAM;YACN,oBAAoB;YACpB,kBAAkB;YAClB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,kBAAkB;YAClB,OAAO;YACP,MAAM;YACN,OAAO;YACP,gBAAgB;YAChB,cAAc;YACd,kBAAkB;YAClB,eAAe;YACf,oBAAoB;YACpB,cAAc;YACd,YAAY;YACZ,SAAS;YACT,QAAQ;YACR,MAAM;YACN,KAAK;YACL,OAAO;YACP,oBAAoB;YACpB,wBAAwB;YACxB,kBAAkB;YAClB,SAAS;YACT,OAAO;YACP,OAAO;YACP,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,YAAY;YACZ,yBAAyB;YACzB,QAAQ;YACR,wIAAwI;SACzI,CAAC;IACJ,CAAC;CACF;AAl3CD,4BAk3CC"} -------------------------------------------------------------------------------- /dist/lib/crawler/exclusions.d.ts: -------------------------------------------------------------------------------- 1 | import { Provider } from './provider'; 2 | export declare class Exclusions implements Provider { 3 | getAll(): string[]; 4 | } 5 | //# sourceMappingURL=exclusions.d.ts.map -------------------------------------------------------------------------------- /dist/lib/crawler/exclusions.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"exclusions.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,UAAW,YAAW,QAAQ;IACzC,MAAM,IAAI,MAAM,EAAE;CAoDnB"} -------------------------------------------------------------------------------- /dist/lib/crawler/exclusions.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Exclusions = void 0; 4 | class Exclusions { 5 | getAll() { 6 | return [ 7 | 'Safari.[\\d\\.]*', 8 | 'Firefox.[\\d\\.]*', 9 | ' Chrome.[\\d\\.]*', 10 | 'Chromium.[\\d\\.]*', 11 | 'MSIE.[\\d\\.]', 12 | 'Opera\\/[\\d\\.]*', 13 | 'Mozilla.[\\d\\.]*', 14 | 'AppleWebKit.[\\d\\.]*', 15 | 'Trident.[\\d\\.]*', 16 | 'Windows NT.[\\d\\.]*', 17 | 'Android [\\d\\.]*', 18 | 'Macintosh.', 19 | 'Ubuntu', 20 | 'Linux', 21 | '[ ]Intel', 22 | 'Mac OS X [\\d_]*', 23 | '(like )?Gecko(.[\\d\\.]*)?', 24 | 'KHTML,', 25 | 'CriOS.[\\d\\.]*', 26 | 'CPU iPhone OS ([0-9_])* like Mac OS X', 27 | 'CPU OS ([0-9_])* like Mac OS X', 28 | 'iPod', 29 | 'compatible', 30 | 'x86_..', 31 | 'i686', 32 | 'x64', 33 | 'X11', 34 | 'rv:[\\d\\.]*', 35 | 'Version.[\\d\\.]*', 36 | 'WOW64', 37 | 'Win64', 38 | 'Dalvik.[\\d\\.]*', 39 | ' \\.NET CLR [\\d\\.]*', 40 | 'Presto.[\\d\\.]*', 41 | 'Media Center PC', 42 | 'BlackBerry', 43 | 'Build', 44 | 'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.', 45 | 'Opera', 46 | ' \\.NET[\\d\\.]*', 47 | 'cubot', 48 | '; M bot', 49 | '; CRONO', 50 | '; B bot', 51 | '; IDbot', 52 | '; ID bot', 53 | '; POWER BOT', 54 | 'OCTOPUS-CORE', 55 | ]; 56 | } 57 | } 58 | exports.Exclusions = Exclusions; 59 | //# sourceMappingURL=exclusions.js.map -------------------------------------------------------------------------------- /dist/lib/crawler/exclusions.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"exclusions.js","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":";;;AAEA,MAAa,UAAU;IACrB,MAAM;QACJ,OAAO;YACL,kBAAkB;YAClB,mBAAmB;YACnB,mBAAmB;YACnB,oBAAoB;YACpB,eAAe;YACf,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;YACnB,sBAAsB;YACtB,mBAAmB;YACnB,YAAY;YACZ,QAAQ;YACR,OAAO;YACP,UAAU;YACV,kBAAkB;YAClB,4BAA4B;YAC5B,QAAQ;YACR,iBAAiB;YACjB,uCAAuC;YACvC,gCAAgC;YAChC,MAAM;YACN,YAAY;YACZ,QAAQ;YACR,MAAM;YACN,KAAK;YACL,KAAK;YACL,cAAc;YACd,mBAAmB;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,kBAAkB;YAClB,iBAAiB;YACjB,YAAY;YACZ,OAAO;YACP,4DAA4D;YAC5D,OAAO;YACP,kBAAkB;YAClB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,UAAU;YACV,aAAa;YACb,cAAc;SACf,CAAC;IACJ,CAAC;CACF;AArDD,gCAqDC"} -------------------------------------------------------------------------------- /dist/lib/crawler/headers.d.ts: -------------------------------------------------------------------------------- 1 | import { Provider } from './provider'; 2 | export declare class Headers implements Provider { 3 | getAll(): string[]; 4 | } 5 | //# sourceMappingURL=headers.d.ts.map -------------------------------------------------------------------------------- /dist/lib/crawler/headers.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,OAAQ,YAAW,QAAQ;IACtC,MAAM,IAAI,MAAM,EAAE;CAcnB"} -------------------------------------------------------------------------------- /dist/lib/crawler/headers.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | exports.Headers = void 0; 4 | class Headers { 5 | getAll() { 6 | return [ 7 | 'USER-AGENT', 8 | 'X-OPERAMINI-PHONE-UA', 9 | 'X-DEVICE-USER-AGENT', 10 | 'X-ORIGINAL-USER-AGENT', 11 | 'X-SKYFIRE-PHONE', 12 | 'X-BOLT-PHONE-UA', 13 | 'DEVICE-STOCK-UA', 14 | 'X-UCBROWSER-DEVICE-UA', 15 | 'FROM', 16 | 'X-SCANNER', 17 | ]; 18 | } 19 | } 20 | exports.Headers = Headers; 21 | //# sourceMappingURL=headers.js.map -------------------------------------------------------------------------------- /dist/lib/crawler/headers.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":";;;AAEA,MAAa,OAAO;IAClB,MAAM;QACJ,OAAO;YACL,YAAY;YACZ,sBAAsB;YACtB,qBAAqB;YACrB,uBAAuB;YACvB,iBAAiB;YACjB,iBAAiB;YACjB,iBAAiB;YACjB,uBAAuB;YACvB,MAAM;YACN,WAAW;SACZ,CAAC;IACJ,CAAC;CACF;AAfD,0BAeC"} -------------------------------------------------------------------------------- /dist/lib/crawler/provider.d.ts: -------------------------------------------------------------------------------- 1 | export interface Provider { 2 | getAll(): string[]; 3 | } 4 | //# sourceMappingURL=provider.d.ts.map -------------------------------------------------------------------------------- /dist/lib/crawler/provider.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,MAAM,IAAI,MAAM,EAAE,CAAC;CACpB"} -------------------------------------------------------------------------------- /dist/lib/crawler/provider.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | //# sourceMappingURL=provider.js.map -------------------------------------------------------------------------------- /dist/lib/crawler/provider.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":""} -------------------------------------------------------------------------------- /dist/types.d.ts: -------------------------------------------------------------------------------- 1 | import { NextFunction, Request, Response } from 'express-serve-static-core'; 2 | import { Crawler } from './lib/crawler'; 3 | export type Middleware = (request: Request, response: Response, next: NextFunction) => Promise; 4 | declare module 'express-serve-static-core' { 5 | interface Request { 6 | Crawler: Crawler; 7 | } 8 | } 9 | //# sourceMappingURL=types.d.ts.map -------------------------------------------------------------------------------- /dist/types.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAE5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,MAAM,MAAM,UAAU,GAAG,CACvB,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,YAAY,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB,OAAO,QAAQ,2BAA2B,CAAC;IACzC,UAAU,OAAO;QACf,OAAO,EAAE,OAAO,CAAC;KAClB;CACF"} -------------------------------------------------------------------------------- /dist/types.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | //# sourceMappingURL=types.js.map -------------------------------------------------------------------------------- /dist/types.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""} -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import typescriptEslint from "@typescript-eslint/eslint-plugin"; 2 | import simpleImportSort from "eslint-plugin-simple-import-sort"; 3 | import tsParser from "@typescript-eslint/parser"; 4 | 5 | export default [{ 6 | ignores: [ 7 | "!**/.*", 8 | "**/coverage/", 9 | "**/junit/", 10 | "**/dist/", 11 | "**/mobiles/", 12 | "**/node_modules/*", 13 | ], 14 | 15 | files: ['**/*.ts', '**/*.tsx'], 16 | 17 | plugins: { 18 | "@typescript-eslint": typescriptEslint, 19 | "simple-import-sort": simpleImportSort, 20 | }, 21 | 22 | languageOptions: { 23 | parser: tsParser, 24 | parserOptions: { 25 | project: "./tsconfig.json", 26 | }, 27 | }, 28 | 29 | rules: { 30 | "@typescript-eslint/no-restricted-types": "error", 31 | "@typescript-eslint/no-explicit-any": "warn", 32 | 33 | "max-len": ["warn", { 34 | code: 80, 35 | tabWidth: 2, 36 | comments: 80, 37 | ignoreComments: true, 38 | ignoreTrailingComments: true, 39 | ignoreUrls: true, 40 | ignoreStrings: true, 41 | ignoreTemplateLiterals: true, 42 | ignoreRegExpLiterals: true, 43 | }], 44 | 45 | "@typescript-eslint/no-unused-vars": ["warn"], 46 | 47 | "simple-import-sort/imports": "error", 48 | "simple-import-sort/exports": "error", 49 | }, 50 | }]; -------------------------------------------------------------------------------- /example/node/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ES6-Crawler-Detect 8 | 9 | 10 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /example/node/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-example", 3 | "version": "1.0.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "node-example", 9 | "version": "1.0.0", 10 | "license": "MIT", 11 | "dependencies": { 12 | "es6-crawler-detect": "^4.0.0", 13 | "express": "^4.21.2" 14 | } 15 | }, 16 | "node_modules/accepts": { 17 | "version": "1.3.8", 18 | "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", 19 | "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", 20 | "dependencies": { 21 | "mime-types": "~2.1.34", 22 | "negotiator": "0.6.3" 23 | }, 24 | "engines": { 25 | "node": ">= 0.6" 26 | } 27 | }, 28 | "node_modules/array-flatten": { 29 | "version": "1.1.1", 30 | "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", 31 | "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI= sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" 32 | }, 33 | "node_modules/body-parser": { 34 | "version": "1.20.3", 35 | "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", 36 | "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", 37 | "dependencies": { 38 | "bytes": "3.1.2", 39 | "content-type": "~1.0.5", 40 | "debug": "2.6.9", 41 | "depd": "2.0.0", 42 | "destroy": "1.2.0", 43 | "http-errors": "2.0.0", 44 | "iconv-lite": "0.4.24", 45 | "on-finished": "2.4.1", 46 | "qs": "6.13.0", 47 | "raw-body": "2.5.2", 48 | "type-is": "~1.6.18", 49 | "unpipe": "1.0.0" 50 | }, 51 | "engines": { 52 | "node": ">= 0.8", 53 | "npm": "1.2.8000 || >= 1.4.16" 54 | } 55 | }, 56 | "node_modules/bytes": { 57 | "version": "3.1.2", 58 | "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", 59 | "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", 60 | "engines": { 61 | "node": ">= 0.8" 62 | } 63 | }, 64 | "node_modules/call-bind": { 65 | "version": "1.0.7", 66 | "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", 67 | "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", 68 | "dependencies": { 69 | "es-define-property": "^1.0.0", 70 | "es-errors": "^1.3.0", 71 | "function-bind": "^1.1.2", 72 | "get-intrinsic": "^1.2.4", 73 | "set-function-length": "^1.2.1" 74 | }, 75 | "engines": { 76 | "node": ">= 0.4" 77 | }, 78 | "funding": { 79 | "url": "https://github.com/sponsors/ljharb" 80 | } 81 | }, 82 | "node_modules/content-disposition": { 83 | "version": "0.5.4", 84 | "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", 85 | "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", 86 | "dependencies": { 87 | "safe-buffer": "5.2.1" 88 | }, 89 | "engines": { 90 | "node": ">= 0.6" 91 | } 92 | }, 93 | "node_modules/content-type": { 94 | "version": "1.0.5", 95 | "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", 96 | "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", 97 | "engines": { 98 | "node": ">= 0.6" 99 | } 100 | }, 101 | "node_modules/cookie": { 102 | "version": "0.7.1", 103 | "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", 104 | "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", 105 | "engines": { 106 | "node": ">= 0.6" 107 | } 108 | }, 109 | "node_modules/cookie-signature": { 110 | "version": "1.0.6", 111 | "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", 112 | "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw= sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" 113 | }, 114 | "node_modules/debug": { 115 | "version": "2.6.9", 116 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", 117 | "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", 118 | "dependencies": { 119 | "ms": "2.0.0" 120 | } 121 | }, 122 | "node_modules/define-data-property": { 123 | "version": "1.1.4", 124 | "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", 125 | "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", 126 | "dependencies": { 127 | "es-define-property": "^1.0.0", 128 | "es-errors": "^1.3.0", 129 | "gopd": "^1.0.1" 130 | }, 131 | "engines": { 132 | "node": ">= 0.4" 133 | }, 134 | "funding": { 135 | "url": "https://github.com/sponsors/ljharb" 136 | } 137 | }, 138 | "node_modules/depd": { 139 | "version": "2.0.0", 140 | "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", 141 | "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", 142 | "engines": { 143 | "node": ">= 0.8" 144 | } 145 | }, 146 | "node_modules/destroy": { 147 | "version": "1.2.0", 148 | "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", 149 | "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", 150 | "engines": { 151 | "node": ">= 0.8", 152 | "npm": "1.2.8000 || >= 1.4.16" 153 | } 154 | }, 155 | "node_modules/ee-first": { 156 | "version": "1.1.1", 157 | "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", 158 | "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" 159 | }, 160 | "node_modules/encodeurl": { 161 | "version": "2.0.0", 162 | "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", 163 | "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", 164 | "engines": { 165 | "node": ">= 0.8" 166 | } 167 | }, 168 | "node_modules/es-define-property": { 169 | "version": "1.0.0", 170 | "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", 171 | "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", 172 | "dependencies": { 173 | "get-intrinsic": "^1.2.4" 174 | }, 175 | "engines": { 176 | "node": ">= 0.4" 177 | } 178 | }, 179 | "node_modules/es-errors": { 180 | "version": "1.3.0", 181 | "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", 182 | "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", 183 | "engines": { 184 | "node": ">= 0.4" 185 | } 186 | }, 187 | "node_modules/es6-crawler-detect": { 188 | "version": "4.0.0", 189 | "resolved": "https://registry.npmjs.org/es6-crawler-detect/-/es6-crawler-detect-4.0.0.tgz", 190 | "integrity": "sha512-i7Vw+cer4DoGzzc8tHUWMe5RVyJatAldQo86glFyqoOvmeVm0bosXovQYkLqo7Kgr1gOOtMg5vO2rocp/0rPmA==", 191 | "license": "MIT" 192 | }, 193 | "node_modules/escape-html": { 194 | "version": "1.0.3", 195 | "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", 196 | "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" 197 | }, 198 | "node_modules/etag": { 199 | "version": "1.8.1", 200 | "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", 201 | "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", 202 | "engines": { 203 | "node": ">= 0.6" 204 | } 205 | }, 206 | "node_modules/express": { 207 | "version": "4.21.2", 208 | "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", 209 | "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", 210 | "dependencies": { 211 | "accepts": "~1.3.8", 212 | "array-flatten": "1.1.1", 213 | "body-parser": "1.20.3", 214 | "content-disposition": "0.5.4", 215 | "content-type": "~1.0.4", 216 | "cookie": "0.7.1", 217 | "cookie-signature": "1.0.6", 218 | "debug": "2.6.9", 219 | "depd": "2.0.0", 220 | "encodeurl": "~2.0.0", 221 | "escape-html": "~1.0.3", 222 | "etag": "~1.8.1", 223 | "finalhandler": "1.3.1", 224 | "fresh": "0.5.2", 225 | "http-errors": "2.0.0", 226 | "merge-descriptors": "1.0.3", 227 | "methods": "~1.1.2", 228 | "on-finished": "2.4.1", 229 | "parseurl": "~1.3.3", 230 | "path-to-regexp": "0.1.12", 231 | "proxy-addr": "~2.0.7", 232 | "qs": "6.13.0", 233 | "range-parser": "~1.2.1", 234 | "safe-buffer": "5.2.1", 235 | "send": "0.19.0", 236 | "serve-static": "1.16.2", 237 | "setprototypeof": "1.2.0", 238 | "statuses": "2.0.1", 239 | "type-is": "~1.6.18", 240 | "utils-merge": "1.0.1", 241 | "vary": "~1.1.2" 242 | }, 243 | "engines": { 244 | "node": ">= 0.10.0" 245 | }, 246 | "funding": { 247 | "type": "opencollective", 248 | "url": "https://opencollective.com/express" 249 | } 250 | }, 251 | "node_modules/finalhandler": { 252 | "version": "1.3.1", 253 | "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", 254 | "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", 255 | "dependencies": { 256 | "debug": "2.6.9", 257 | "encodeurl": "~2.0.0", 258 | "escape-html": "~1.0.3", 259 | "on-finished": "2.4.1", 260 | "parseurl": "~1.3.3", 261 | "statuses": "2.0.1", 262 | "unpipe": "~1.0.0" 263 | }, 264 | "engines": { 265 | "node": ">= 0.8" 266 | } 267 | }, 268 | "node_modules/forwarded": { 269 | "version": "0.2.0", 270 | "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", 271 | "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", 272 | "engines": { 273 | "node": ">= 0.6" 274 | } 275 | }, 276 | "node_modules/fresh": { 277 | "version": "0.5.2", 278 | "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", 279 | "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", 280 | "engines": { 281 | "node": ">= 0.6" 282 | } 283 | }, 284 | "node_modules/function-bind": { 285 | "version": "1.1.2", 286 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", 287 | "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", 288 | "funding": { 289 | "url": "https://github.com/sponsors/ljharb" 290 | } 291 | }, 292 | "node_modules/get-intrinsic": { 293 | "version": "1.2.4", 294 | "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", 295 | "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", 296 | "dependencies": { 297 | "es-errors": "^1.3.0", 298 | "function-bind": "^1.1.2", 299 | "has-proto": "^1.0.1", 300 | "has-symbols": "^1.0.3", 301 | "hasown": "^2.0.0" 302 | }, 303 | "engines": { 304 | "node": ">= 0.4" 305 | }, 306 | "funding": { 307 | "url": "https://github.com/sponsors/ljharb" 308 | } 309 | }, 310 | "node_modules/gopd": { 311 | "version": "1.0.1", 312 | "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", 313 | "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", 314 | "dependencies": { 315 | "get-intrinsic": "^1.1.3" 316 | }, 317 | "funding": { 318 | "url": "https://github.com/sponsors/ljharb" 319 | } 320 | }, 321 | "node_modules/has-property-descriptors": { 322 | "version": "1.0.2", 323 | "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", 324 | "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", 325 | "dependencies": { 326 | "es-define-property": "^1.0.0" 327 | }, 328 | "funding": { 329 | "url": "https://github.com/sponsors/ljharb" 330 | } 331 | }, 332 | "node_modules/has-proto": { 333 | "version": "1.0.3", 334 | "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", 335 | "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", 336 | "engines": { 337 | "node": ">= 0.4" 338 | }, 339 | "funding": { 340 | "url": "https://github.com/sponsors/ljharb" 341 | } 342 | }, 343 | "node_modules/has-symbols": { 344 | "version": "1.0.3", 345 | "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", 346 | "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", 347 | "engines": { 348 | "node": ">= 0.4" 349 | }, 350 | "funding": { 351 | "url": "https://github.com/sponsors/ljharb" 352 | } 353 | }, 354 | "node_modules/hasown": { 355 | "version": "2.0.2", 356 | "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", 357 | "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", 358 | "dependencies": { 359 | "function-bind": "^1.1.2" 360 | }, 361 | "engines": { 362 | "node": ">= 0.4" 363 | } 364 | }, 365 | "node_modules/http-errors": { 366 | "version": "2.0.0", 367 | "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", 368 | "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", 369 | "dependencies": { 370 | "depd": "2.0.0", 371 | "inherits": "2.0.4", 372 | "setprototypeof": "1.2.0", 373 | "statuses": "2.0.1", 374 | "toidentifier": "1.0.1" 375 | }, 376 | "engines": { 377 | "node": ">= 0.8" 378 | } 379 | }, 380 | "node_modules/iconv-lite": { 381 | "version": "0.4.24", 382 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", 383 | "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", 384 | "dependencies": { 385 | "safer-buffer": ">= 2.1.2 < 3" 386 | }, 387 | "engines": { 388 | "node": ">=0.10.0" 389 | } 390 | }, 391 | "node_modules/inherits": { 392 | "version": "2.0.4", 393 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 394 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" 395 | }, 396 | "node_modules/ipaddr.js": { 397 | "version": "1.9.1", 398 | "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", 399 | "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", 400 | "engines": { 401 | "node": ">= 0.10" 402 | } 403 | }, 404 | "node_modules/media-typer": { 405 | "version": "0.3.0", 406 | "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", 407 | "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", 408 | "engines": { 409 | "node": ">= 0.6" 410 | } 411 | }, 412 | "node_modules/merge-descriptors": { 413 | "version": "1.0.3", 414 | "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", 415 | "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", 416 | "funding": { 417 | "url": "https://github.com/sponsors/sindresorhus" 418 | } 419 | }, 420 | "node_modules/methods": { 421 | "version": "1.1.2", 422 | "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", 423 | "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", 424 | "engines": { 425 | "node": ">= 0.6" 426 | } 427 | }, 428 | "node_modules/mime": { 429 | "version": "1.6.0", 430 | "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", 431 | "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", 432 | "bin": { 433 | "mime": "cli.js" 434 | }, 435 | "engines": { 436 | "node": ">=4" 437 | } 438 | }, 439 | "node_modules/mime-db": { 440 | "version": "1.52.0", 441 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 442 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 443 | "engines": { 444 | "node": ">= 0.6" 445 | } 446 | }, 447 | "node_modules/mime-types": { 448 | "version": "2.1.35", 449 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 450 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 451 | "dependencies": { 452 | "mime-db": "1.52.0" 453 | }, 454 | "engines": { 455 | "node": ">= 0.6" 456 | } 457 | }, 458 | "node_modules/ms": { 459 | "version": "2.0.0", 460 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 461 | "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" 462 | }, 463 | "node_modules/negotiator": { 464 | "version": "0.6.3", 465 | "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", 466 | "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", 467 | "engines": { 468 | "node": ">= 0.6" 469 | } 470 | }, 471 | "node_modules/object-inspect": { 472 | "version": "1.13.2", 473 | "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", 474 | "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", 475 | "engines": { 476 | "node": ">= 0.4" 477 | }, 478 | "funding": { 479 | "url": "https://github.com/sponsors/ljharb" 480 | } 481 | }, 482 | "node_modules/on-finished": { 483 | "version": "2.4.1", 484 | "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", 485 | "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", 486 | "dependencies": { 487 | "ee-first": "1.1.1" 488 | }, 489 | "engines": { 490 | "node": ">= 0.8" 491 | } 492 | }, 493 | "node_modules/parseurl": { 494 | "version": "1.3.3", 495 | "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", 496 | "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", 497 | "engines": { 498 | "node": ">= 0.8" 499 | } 500 | }, 501 | "node_modules/path-to-regexp": { 502 | "version": "0.1.12", 503 | "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", 504 | "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==" 505 | }, 506 | "node_modules/proxy-addr": { 507 | "version": "2.0.7", 508 | "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", 509 | "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", 510 | "dependencies": { 511 | "forwarded": "0.2.0", 512 | "ipaddr.js": "1.9.1" 513 | }, 514 | "engines": { 515 | "node": ">= 0.10" 516 | } 517 | }, 518 | "node_modules/qs": { 519 | "version": "6.13.0", 520 | "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", 521 | "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", 522 | "dependencies": { 523 | "side-channel": "^1.0.6" 524 | }, 525 | "engines": { 526 | "node": ">=0.6" 527 | }, 528 | "funding": { 529 | "url": "https://github.com/sponsors/ljharb" 530 | } 531 | }, 532 | "node_modules/range-parser": { 533 | "version": "1.2.1", 534 | "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", 535 | "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", 536 | "engines": { 537 | "node": ">= 0.6" 538 | } 539 | }, 540 | "node_modules/raw-body": { 541 | "version": "2.5.2", 542 | "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", 543 | "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", 544 | "dependencies": { 545 | "bytes": "3.1.2", 546 | "http-errors": "2.0.0", 547 | "iconv-lite": "0.4.24", 548 | "unpipe": "1.0.0" 549 | }, 550 | "engines": { 551 | "node": ">= 0.8" 552 | } 553 | }, 554 | "node_modules/safe-buffer": { 555 | "version": "5.2.1", 556 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", 557 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", 558 | "funding": [ 559 | { 560 | "type": "github", 561 | "url": "https://github.com/sponsors/feross" 562 | }, 563 | { 564 | "type": "patreon", 565 | "url": "https://www.patreon.com/feross" 566 | }, 567 | { 568 | "type": "consulting", 569 | "url": "https://feross.org/support" 570 | } 571 | ] 572 | }, 573 | "node_modules/safer-buffer": { 574 | "version": "2.1.2", 575 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", 576 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" 577 | }, 578 | "node_modules/send": { 579 | "version": "0.19.0", 580 | "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", 581 | "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", 582 | "dependencies": { 583 | "debug": "2.6.9", 584 | "depd": "2.0.0", 585 | "destroy": "1.2.0", 586 | "encodeurl": "~1.0.2", 587 | "escape-html": "~1.0.3", 588 | "etag": "~1.8.1", 589 | "fresh": "0.5.2", 590 | "http-errors": "2.0.0", 591 | "mime": "1.6.0", 592 | "ms": "2.1.3", 593 | "on-finished": "2.4.1", 594 | "range-parser": "~1.2.1", 595 | "statuses": "2.0.1" 596 | }, 597 | "engines": { 598 | "node": ">= 0.8.0" 599 | } 600 | }, 601 | "node_modules/send/node_modules/encodeurl": { 602 | "version": "1.0.2", 603 | "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", 604 | "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", 605 | "engines": { 606 | "node": ">= 0.8" 607 | } 608 | }, 609 | "node_modules/send/node_modules/ms": { 610 | "version": "2.1.3", 611 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 612 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 613 | }, 614 | "node_modules/serve-static": { 615 | "version": "1.16.2", 616 | "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", 617 | "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", 618 | "dependencies": { 619 | "encodeurl": "~2.0.0", 620 | "escape-html": "~1.0.3", 621 | "parseurl": "~1.3.3", 622 | "send": "0.19.0" 623 | }, 624 | "engines": { 625 | "node": ">= 0.8.0" 626 | } 627 | }, 628 | "node_modules/set-function-length": { 629 | "version": "1.2.2", 630 | "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", 631 | "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", 632 | "dependencies": { 633 | "define-data-property": "^1.1.4", 634 | "es-errors": "^1.3.0", 635 | "function-bind": "^1.1.2", 636 | "get-intrinsic": "^1.2.4", 637 | "gopd": "^1.0.1", 638 | "has-property-descriptors": "^1.0.2" 639 | }, 640 | "engines": { 641 | "node": ">= 0.4" 642 | } 643 | }, 644 | "node_modules/setprototypeof": { 645 | "version": "1.2.0", 646 | "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", 647 | "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" 648 | }, 649 | "node_modules/side-channel": { 650 | "version": "1.0.6", 651 | "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", 652 | "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", 653 | "dependencies": { 654 | "call-bind": "^1.0.7", 655 | "es-errors": "^1.3.0", 656 | "get-intrinsic": "^1.2.4", 657 | "object-inspect": "^1.13.1" 658 | }, 659 | "engines": { 660 | "node": ">= 0.4" 661 | }, 662 | "funding": { 663 | "url": "https://github.com/sponsors/ljharb" 664 | } 665 | }, 666 | "node_modules/statuses": { 667 | "version": "2.0.1", 668 | "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", 669 | "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", 670 | "engines": { 671 | "node": ">= 0.8" 672 | } 673 | }, 674 | "node_modules/toidentifier": { 675 | "version": "1.0.1", 676 | "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", 677 | "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", 678 | "engines": { 679 | "node": ">=0.6" 680 | } 681 | }, 682 | "node_modules/type-is": { 683 | "version": "1.6.18", 684 | "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", 685 | "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", 686 | "dependencies": { 687 | "media-typer": "0.3.0", 688 | "mime-types": "~2.1.24" 689 | }, 690 | "engines": { 691 | "node": ">= 0.6" 692 | } 693 | }, 694 | "node_modules/unpipe": { 695 | "version": "1.0.0", 696 | "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", 697 | "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", 698 | "engines": { 699 | "node": ">= 0.8" 700 | } 701 | }, 702 | "node_modules/utils-merge": { 703 | "version": "1.0.1", 704 | "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", 705 | "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", 706 | "engines": { 707 | "node": ">= 0.4.0" 708 | } 709 | }, 710 | "node_modules/vary": { 711 | "version": "1.1.2", 712 | "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", 713 | "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", 714 | "engines": { 715 | "node": ">= 0.8" 716 | } 717 | } 718 | } 719 | } 720 | -------------------------------------------------------------------------------- /example/node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-example", 3 | "version": "1.0.0", 4 | "description": "Using the crawler with express servers", 5 | "main": "server.js", 6 | "author": "Jeffery Hussin (http://github.com/jefferyhus)", 7 | "license": "MIT", 8 | "dependencies": { 9 | "es6-crawler-detect": "^4.0.0", 10 | "express": "^4.21.2" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /example/node/server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const path = require('path'); 4 | const express = require('express'); 5 | const { middleware, Crawler } = require('es6-crawler-detect'); 6 | 7 | const app = express(); 8 | const port = 3000; 9 | 10 | app.use( 11 | middleware(() => { 12 | console.log('Testing the callback\n'); 13 | }) 14 | ); 15 | 16 | app.use('/dist', express.static(path.join(__dirname + '/dist'))); 17 | 18 | app.get('/', function (req, res) { 19 | res.sendFile(path.join(__dirname + '/index.html')); 20 | }); 21 | 22 | app.get('/crawler', function async(request, response) { 23 | // or check a user agent string 24 | request.Crawler.isCrawler( 25 | 'TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)' 26 | ); 27 | 28 | // Output the name of the bot that matched (if any) 29 | response.send(request.Crawler.getMatches()); 30 | }); 31 | 32 | app.get('/curl', function async(request, response) { 33 | const CrawlerDetector = new Crawler(request); 34 | CrawlerDetector.isCrawler(); // true 35 | response.send(CrawlerDetector.getMatches()); 36 | }); 37 | 38 | app.listen(port, () => console.log(`Example app listening on port ${port}!`)); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "es6-crawler-detect", 3 | "version": "4.0.2", 4 | "description": "This is an ES6 adaptation of the original PHP library CrawlerDetect, this library will help you detect bots/crawlers/spiders vie the useragent.", 5 | "main": "dist/index.js", 6 | "type": "commonjs", 7 | "files": [ 8 | "dist" 9 | ], 10 | "sideEffects": false, 11 | "directories": { 12 | "lib": "lib" 13 | }, 14 | "scripts": { 15 | "build": "tsc", 16 | "watch": "tsc -w", 17 | "test": "mocha", 18 | "lint": "eslint --fix ./src ./test", 19 | "npm:minor": "npm version minor", 20 | "npm:major": "npm version major", 21 | "npm:patch": "npm version patch", 22 | "npm:publish": "npm publish" 23 | }, 24 | "repository": { 25 | "type": "git", 26 | "url": "git+https://github.com/JefferyHus/es6-crawler-detect.git" 27 | }, 28 | "keywords": [ 29 | "es6-javascript", 30 | "crawler", 31 | "bots", 32 | "spider", 33 | "detection" 34 | ], 35 | "author": "Jeffery Hussin (http://github.com/jefferyhus)", 36 | "license": "MIT", 37 | "bugs": { 38 | "url": "https://github.com/JefferyHus/es6-crawler-detect/issues" 39 | }, 40 | "homepage": "https://github.com/JefferyHus/es6-crawler-detect#readme", 41 | "devDependencies": { 42 | "@babel/cli": "^7.25.6", 43 | "@babel/core": "^7.25.2", 44 | "@babel/eslint-parser": "^7.25.1", 45 | "@babel/polyfill": "^7.12.1", 46 | "@babel/preset-env": "^7.25.4", 47 | "@types/chai": "^4.3.19", 48 | "@types/express-serve-static-core": "^4.19.5", 49 | "@types/mocha": "^10.0.8", 50 | "@types/node": "^22.5.5", 51 | "@typescript-eslint/eslint-plugin": "^8.6.0", 52 | "babel-loader": "^9.2.1", 53 | "chai": "^5.1.1", 54 | "core-js": "^3.38.1", 55 | "cross-env": "^7.0.3", 56 | "eslint": "^9.11.0", 57 | "eslint-config-prettier": "^9.1.0", 58 | "eslint-plugin-prettier": "^5.2.1", 59 | "eslint-plugin-simple-import-sort": "^12.1.1", 60 | "mocha": "^10.7.3", 61 | "prettier": "^3.3.3", 62 | "style-loader": "^4.0.0", 63 | "terser-webpack-plugin": "^5.3.10", 64 | "ts-node": "^10.9.2", 65 | "typescript": "^5.6.2", 66 | "webpack": "^5.94.0", 67 | "webpack-cli": "^5.1.4", 68 | "webpack-merge": "^6.0.1" 69 | }, 70 | "types": "dist/index.d.ts" 71 | } 72 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { NextFunction, Request, Response } from 'express-serve-static-core'; 2 | 3 | import { Crawler } from './lib/crawler'; 4 | 5 | export { Crawler }; 6 | export const middleware = ( 7 | cb?: (req: Request, res: Response, next: NextFunction) => void 8 | ) => { 9 | return (req: Request, res: Response, next: NextFunction) => { 10 | // If there is a cb, execute it 11 | if (typeof cb === 'function') { 12 | cb.call(this, req, res, next); 13 | } 14 | // Initiate 15 | req.Crawler = new Crawler(req); 16 | next(); 17 | }; 18 | }; 19 | -------------------------------------------------------------------------------- /src/lib/crawler.ts: -------------------------------------------------------------------------------- 1 | import { Request } from 'express-serve-static-core'; 2 | import { IncomingHttpHeaders } from 'http2'; 3 | 4 | import { Crawlers } from './crawler/crawlers'; 5 | import { Exclusions } from './crawler/exclusions'; 6 | import { Headers } from './crawler/headers'; 7 | 8 | export class Crawler { 9 | private crawlers: Crawlers; 10 | private headers: Headers; 11 | private exclusions: Exclusions; 12 | 13 | private request: Request | NodeJS.Dict; 14 | private compiledRegexList: RegExp; 15 | private compiledExclusions: RegExp; 16 | private httpHeaders: string | string[] | IncomingHttpHeaders; 17 | private userAgent: string; 18 | private matches?: RegExpExecArray | null; 19 | 20 | constructor( 21 | request?: Request, 22 | headers?: IncomingHttpHeaders, 23 | userAgent?: string 24 | ) { 25 | this.crawlers = new Crawlers(); 26 | this.headers = new Headers(); 27 | this.exclusions = new Exclusions(); 28 | 29 | this.request = request ?? {}; 30 | 31 | // The regex-list must not be used with g-flag! 32 | // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results 33 | this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i'); 34 | 35 | // The exclusions should be used with g-flag in order to remove each value. 36 | this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi'); 37 | 38 | this.httpHeaders = this.setHttpHeaders(headers); 39 | this.userAgent = this.setUserAgent(userAgent); 40 | } 41 | 42 | public compileRegex(patterns: string[], flags?: string): RegExp { 43 | return new RegExp(patterns.join('|'), flags); 44 | } 45 | 46 | private setHttpHeaders( 47 | headers?: string | string[] | IncomingHttpHeaders 48 | ): string | string[] | IncomingHttpHeaders { 49 | // Use the Request headers if httpHeaders is not defined 50 | if (!headers || Object.keys(headers).length === 0) { 51 | if (Object.keys(this.request).length) { 52 | if (this.request.headers) { 53 | return this.request.headers; 54 | } 55 | } 56 | 57 | return ''; 58 | } 59 | 60 | // Save the headers. 61 | return headers; 62 | } 63 | 64 | private setUserAgent(userAgent?: string): string { 65 | if (!userAgent?.length) { 66 | userAgent = ''; 67 | for (const header of this.getUaHttpHeaders()) { 68 | if ( 69 | typeof this.httpHeaders === 'object' && 70 | !Array.isArray(this.httpHeaders) 71 | ) { 72 | if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) { 73 | const headerValue = this.httpHeaders[header.toLowerCase()]; 74 | 75 | if (typeof headerValue === 'string') { 76 | const separator = userAgent.length > 0 ? ' ' : ''; 77 | userAgent += separator + headerValue; 78 | } else if (Array.isArray(headerValue)) { 79 | const separator = userAgent.length > 0 ? ' ' : ''; 80 | userAgent += separator + headerValue.join(' '); 81 | } 82 | } 83 | } 84 | } 85 | } 86 | 87 | return userAgent; 88 | } 89 | 90 | private getUaHttpHeaders() { 91 | return this.headers.getAll(); 92 | } 93 | 94 | public getMatches(): string | null | object { 95 | if (this.matches !== undefined) { 96 | if (this.matches?.length) { 97 | return this.matches[0]; 98 | } 99 | 100 | return null; 101 | } 102 | 103 | return {}; 104 | } 105 | 106 | public isCrawler(userAgent?: string): boolean { 107 | if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) { 108 | return false; 109 | } 110 | 111 | let agent = userAgent ?? this.userAgent; 112 | 113 | // test on compiled regx 114 | agent = agent.replace(this.compiledExclusions, ''); 115 | 116 | if (agent.trim().length === 0) { 117 | return false; 118 | } 119 | 120 | const matches = this.compiledRegexList.exec(agent); 121 | 122 | if (matches) { 123 | this.matches = matches; 124 | } 125 | 126 | return matches !== null && matches.length > 0; 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/lib/crawler/crawlers.ts: -------------------------------------------------------------------------------- 1 | import { Provider } from './provider'; 2 | 3 | export class Crawlers implements Provider { 4 | public getAll(): string[] { 5 | return [ 6 | ' YLT', 7 | '^Aether', 8 | '^Amazon Simple Notification Service Agent$', 9 | '^Amazon-Route53-Health-Check-Service', 10 | '^b0t$', 11 | '^bluefish ', 12 | '^Calypso v\\/', 13 | '^COMODO DCV', 14 | '^Corax', 15 | '^DangDang', 16 | '^DavClnt', 17 | '^DHSH', 18 | '^docker\\/[0-9]', 19 | '^Expanse', 20 | '^FDM ', 21 | '^git\\/', 22 | '^Goose\\/', 23 | '^Grabber', 24 | '^Gradle\\/', 25 | '^HTTPClient\\/', 26 | '^HTTPing', 27 | '^Java\\/', 28 | '^Jeode\\/', 29 | '^Jetty\\/', 30 | '^Mail\\/', 31 | '^Mget', 32 | '^Microsoft URL Control', 33 | '^Mikrotik\\/', 34 | '^Netlab360', 35 | '^NG\\/[0-9\\.]', 36 | '^NING\\/', 37 | '^npm\\/', 38 | '^Nuclei', 39 | '^PHP-AYMAPI\\/', 40 | '^PHP\\/', 41 | '^pip\\/', 42 | '^pnpm\\/', 43 | '^RMA\\/', 44 | '^Ruby|Ruby\\/[0-9]', 45 | '^Swurl ', 46 | '^TLS tester ', 47 | '^twine\\/', 48 | '^ureq', 49 | '^VSE\\/[0-9]', 50 | '^WordPress\\.com', 51 | '^XRL\\/[0-9]', 52 | '^ZmEu', 53 | '008\\/', 54 | '13TABS', 55 | '192\\.comAgent', 56 | '2GDPR\\/', 57 | '2ip\\.ru', 58 | '404enemy', 59 | '7Siters', 60 | '80legs', 61 | 'a3logics\\.in', 62 | 'A6-Indexer', 63 | 'Abonti', 64 | 'Aboundex', 65 | 'aboutthedomain', 66 | 'Accoona-AI-Agent', 67 | 'acebookexternalhit\\/', 68 | 'acoon', 69 | 'acrylicapps\\.com\\/pulp', 70 | 'Acunetix', 71 | 'AdAuth\\/', 72 | 'adbeat', 73 | 'AddThis', 74 | 'ADmantX', 75 | 'AdminLabs', 76 | 'adressendeutschland', 77 | 'adreview\\/', 78 | 'adscanner', 79 | 'adstxt-worker', 80 | 'Adstxtaggregator', 81 | 'adstxt\\.com', 82 | 'Adyen HttpClient', 83 | 'AffiliateLabz\\/', 84 | 'affilimate-puppeteer', 85 | 'agentslug', 86 | 'AHC', 87 | 'aihit', 88 | 'aiohttp\\/', 89 | 'Airmail', 90 | 'akka-http\\/', 91 | 'akula\\/', 92 | 'alertra', 93 | 'alexa site audit', 94 | 'Alibaba\\.Security\\.Heimdall', 95 | 'Alligator', 96 | 'allloadin', 97 | 'AllSubmitter', 98 | 'alyze\\.info', 99 | 'amagit', 100 | 'Anarchie', 101 | 'AndroidDownloadManager', 102 | 'Anemone', 103 | 'AngleSharp', 104 | 'annotate_google', 105 | 'Anthill', 106 | 'Anturis Agent', 107 | 'Ant\\.com', 108 | 'AnyEvent-HTTP\\/', 109 | 'Apache Ant\\/', 110 | 'Apache Droid', 111 | 'Apache OpenOffice', 112 | 'Apache-HttpAsyncClient', 113 | 'Apache-HttpClient', 114 | 'ApacheBench', 115 | 'Apexoo', 116 | 'apimon\\.de', 117 | 'APIs-Google', 118 | 'AportWorm\\/', 119 | 'AppBeat\\/', 120 | 'AppEngine-Google', 121 | 'AppleSyndication', 122 | 'Aprc\\/[0-9]', 123 | 'Arachmo', 124 | 'arachnode', 125 | 'Arachnophilia', 126 | 'aria2', 127 | 'Arukereso', 128 | 'asafaweb', 129 | 'Asana\\/', 130 | 'Ask Jeeves', 131 | 'AskQuickly', 132 | 'ASPSeek', 133 | 'Asterias', 134 | 'Astute', 135 | 'asynchttp', 136 | 'Attach', 137 | 'attohttpc', 138 | 'autocite', 139 | 'AutomaticWPTester', 140 | 'Autonomy', 141 | 'awin\\.com', 142 | 'AWS Security Scanner', 143 | 'axios\\/', 144 | 'a\\.pr-cy\\.ru', 145 | 'B-l-i-t-z-B-O-T', 146 | 'Backlink-Ceck', 147 | 'backlink-check', 148 | 'BacklinkHttpStatus', 149 | 'BackStreet', 150 | 'BackupLand', 151 | 'BackWeb', 152 | 'Bad-Neighborhood', 153 | 'Badass', 154 | 'baidu\\.com', 155 | 'Bandit', 156 | 'basicstate', 157 | 'BatchFTP', 158 | 'Battleztar Bazinga', 159 | 'baypup\\/', 160 | 'BazQux', 161 | 'BBBike', 162 | 'BCKLINKS', 163 | 'BDFetch', 164 | 'BegunAdvertising', 165 | 'Bewica-security-scan', 166 | 'Bidtellect', 167 | 'BigBozz', 168 | 'Bigfoot', 169 | 'biglotron', 170 | 'BingLocalSearch', 171 | 'BingPreview', 172 | 'binlar', 173 | 'biNu image cacher', 174 | 'Bitacle', 175 | 'Bitrix link preview', 176 | 'biz_Directory', 177 | 'BKCTwitterUnshortener\\/', 178 | 'Black Hole', 179 | 'Blackboard Safeassign', 180 | 'BlackWidow', 181 | 'BlockNote\\.Net', 182 | 'BlogBridge', 183 | 'Bloglines', 184 | 'Bloglovin', 185 | 'BlogPulseLive', 186 | 'BlogSearch', 187 | 'Blogtrottr', 188 | 'BlowFish', 189 | 'boitho\\.com-dc', 190 | 'Boost\\.Beast', 191 | 'BPImageWalker', 192 | 'Braintree-Webhooks', 193 | 'Branch Metrics API', 194 | 'Branch-Passthrough', 195 | 'Brandprotect', 196 | 'BrandVerity', 197 | 'Brandwatch', 198 | 'Brodie\\/', 199 | 'Browsershots', 200 | 'BUbiNG', 201 | 'Buck\\/', 202 | 'Buddy', 203 | 'BuiltWith', 204 | 'Bullseye', 205 | 'BunnySlippers', 206 | 'Burf Search', 207 | 'Butterfly\\/', 208 | 'BuzzSumo', 209 | 'CAAM\\/[0-9]', 210 | 'CakePHP', 211 | 'Calculon', 212 | 'Canary%20Mail', 213 | 'CaretNail', 214 | 'catexplorador', 215 | 'CC Metadata Scaper', 216 | 'Cegbfeieh', 217 | 'censys', 218 | 'centuryb.o.t9[at]gmail.com', 219 | 'Cerberian Drtrs', 220 | 'CERT\\.at-Statistics-Survey', 221 | 'cf-facebook', 222 | 'cg-eye', 223 | 'changedetection', 224 | 'ChangesMeter', 225 | 'Charlotte', 226 | 'CheckHost', 227 | 'checkprivacy', 228 | 'CherryPicker', 229 | 'ChinaClaw', 230 | 'Chirp\\/', 231 | 'chkme\\.com', 232 | 'Chlooe', 233 | 'Chromaxa', 234 | 'CirrusExplorer', 235 | 'CISPA Vulnerability Notification', 236 | 'CISPA Web Analyser', 237 | 'Citoid', 238 | 'CJNetworkQuality', 239 | 'Clarsentia', 240 | 'clips\\.ua\\.ac\\.be', 241 | 'Cloud mapping', 242 | 'CloudEndure', 243 | 'CloudFlare-AlwaysOnline', 244 | 'Cloudflare-Healthchecks', 245 | 'Cloudinary', 246 | 'cmcm\\.com', 247 | 'coccoc', 248 | 'cognitiveseo', 249 | 'ColdFusion', 250 | 'colly -', 251 | 'CommaFeed', 252 | 'Commons-HttpClient', 253 | 'commonscan', 254 | 'contactbigdatafr', 255 | 'contentkingapp', 256 | 'Contextual Code Sites Explorer', 257 | 'convera', 258 | 'CookieReports', 259 | 'copyright sheriff', 260 | 'CopyRightCheck', 261 | 'Copyscape', 262 | 'cortex\\/', 263 | 'Cosmos4j\\.feedback', 264 | 'Covario-IDS', 265 | 'Craw\\/', 266 | 'Crescent', 267 | 'Criteo', 268 | 'Crowsnest', 269 | 'CSHttp', 270 | 'CSSCheck', 271 | 'Cula\\/', 272 | 'curb', 273 | 'Curious George', 274 | 'curl', 275 | 'cuwhois\\/', 276 | 'cybo\\.com', 277 | 'DAP\\/NetHTTP', 278 | 'DareBoost', 279 | 'DatabaseDriverMysqli', 280 | 'DataCha0s', 281 | 'Datafeedwatch', 282 | 'Datanyze', 283 | 'DataparkSearch', 284 | 'dataprovider', 285 | 'DataXu', 286 | 'Daum(oa)?[ \\/][0-9]', 287 | 'dBpoweramp', 288 | 'ddline', 289 | 'deeris', 290 | 'delve\\.ai', 291 | 'Demon', 292 | 'DeuSu', 293 | 'developers\\.google\\.com\\/\\+\\/web\\/snippet\\/', 294 | 'Devil', 295 | 'Digg', 296 | 'Digincore', 297 | 'DigitalPebble', 298 | 'Dirbuster', 299 | 'Discourse Forum Onebox', 300 | 'Dispatch\\/', 301 | 'Disqus\\/', 302 | 'DittoSpyder', 303 | 'dlvr', 304 | 'DMBrowser', 305 | 'DNSPod-reporting', 306 | 'docoloc', 307 | 'Dolphin http client', 308 | 'DomainAppender', 309 | 'DomainLabz', 310 | 'Domains Project\\/', 311 | 'Donuts Content Explorer', 312 | 'dotMailer content retrieval', 313 | 'dotSemantic', 314 | 'downforeveryoneorjustme', 315 | 'Download Wonder', 316 | 'downnotifier', 317 | 'DowntimeDetector', 318 | 'Drip', 319 | 'drupact', 320 | 'Drupal \\(\\+http:\\/\\/drupal\\.org\\/\\)', 321 | 'DTS Agent', 322 | 'dubaiindex', 323 | 'DuplexWeb-Google', 324 | 'DynatraceSynthetic', 325 | 'EARTHCOM', 326 | 'Easy-Thumb', 327 | 'EasyDL', 328 | 'Ebingbong', 329 | 'ec2linkfinder', 330 | 'eCairn-Grabber', 331 | 'eCatch', 332 | 'ECCP', 333 | 'eContext\\/', 334 | 'Ecxi', 335 | 'EirGrabber', 336 | 'ElectricMonk', 337 | 'elefent', 338 | 'EMail Exractor', 339 | 'EMail Wolf', 340 | 'EmailWolf', 341 | 'Embarcadero', 342 | 'Embed PHP Library', 343 | 'Embedly', 344 | 'endo\\/', 345 | 'europarchive\\.org', 346 | 'evc-batch', 347 | 'EventMachine HttpClient', 348 | 'Everwall Link Expander', 349 | 'Evidon', 350 | 'Evrinid', 351 | 'ExactSearch', 352 | 'ExaleadCloudview', 353 | 'Excel\\/', 354 | 'exif', 355 | 'ExoRank', 356 | 'Exploratodo', 357 | 'Express WebPictures', 358 | 'Extreme Picture Finder', 359 | 'EyeNetIE', 360 | 'ezooms', 361 | 'facebookexternalhit', 362 | 'facebookexternalua', 363 | 'facebookplatform', 364 | 'fairshare', 365 | 'Faraday v', 366 | 'fasthttp', 367 | 'Faveeo', 368 | 'Favicon downloader', 369 | 'faviconarchive', 370 | 'faviconkit', 371 | 'FavOrg', 372 | 'Feed Wrangler', 373 | 'Feedable\\/', 374 | 'Feedbin', 375 | 'FeedBooster', 376 | 'FeedBucket', 377 | 'FeedBunch\\/', 378 | 'FeedBurner', 379 | 'feeder', 380 | 'Feedly', 381 | 'FeedshowOnline', 382 | 'Feedshow\\/', 383 | 'Feedspot', 384 | 'FeedViewer\\/', 385 | 'Feedwind\\/', 386 | 'FeedZcollector', 387 | 'feeltiptop', 388 | 'Fetch API', 389 | 'Fetch\\/[0-9]', 390 | 'Fever\\/[0-9]', 391 | 'FHscan', 392 | 'Fiery%20Feeds', 393 | 'Filestack', 394 | 'Fimap', 395 | 'findlink', 396 | 'findthatfile', 397 | 'FlashGet', 398 | 'FlipboardBrowserProxy', 399 | 'FlipboardProxy', 400 | 'FlipboardRSS', 401 | 'Flock\\/', 402 | 'Florienzh\\/', 403 | 'fluffy', 404 | 'Flunky', 405 | 'flynxapp', 406 | 'forensiq', 407 | 'FoundSeoTool', 408 | 'free thumbnails', 409 | 'Freeuploader', 410 | 'FreshRSS', 411 | 'Funnelback', 412 | 'Fuzz Faster U Fool', 413 | 'G-i-g-a-b-o-t', 414 | 'g00g1e\\.net', 415 | 'ganarvisitas', 416 | 'gdnplus\\.com', 417 | 'geek-tools', 418 | 'Genieo', 419 | 'GentleSource', 420 | 'GetCode', 421 | 'Getintent', 422 | 'GetLinkInfo', 423 | 'getprismatic', 424 | 'GetRight', 425 | 'getroot', 426 | 'GetURLInfo\\/', 427 | 'GetWeb', 428 | 'Geziyor', 429 | 'Ghost Inspector', 430 | 'GigablastOpenSource', 431 | 'GIS-LABS', 432 | 'github-camo', 433 | 'GitHub-Hookshot', 434 | 'github\\.com', 435 | 'Go http package', 436 | 'Go [\\d\\.]* package http', 437 | 'Go!Zilla', 438 | 'Go-Ahead-Got-It', 439 | 'Go-http-client', 440 | 'go-mtasts\\/', 441 | 'gobyus', 442 | 'Gofeed', 443 | 'gofetch', 444 | 'Goldfire Server', 445 | 'GomezAgent', 446 | 'gooblog', 447 | 'Goodzer\\/', 448 | 'Google AppsViewer', 449 | 'Google Desktop', 450 | 'Google favicon', 451 | 'Google Keyword Suggestion', 452 | 'Google Keyword Tool', 453 | 'Google Page Speed Insights', 454 | 'Google PP Default', 455 | 'Google Search Console', 456 | 'Google Web Preview', 457 | 'Google-Ads-Creatives-Assistant', 458 | 'Google-Ads-Overview', 459 | 'Google-Adwords', 460 | 'Google-Apps-Script', 461 | 'Google-Calendar-Importer', 462 | 'Google-HotelAdsVerifier', 463 | 'Google-HTTP-Java-Client', 464 | 'Google-Podcast', 465 | 'Google-Publisher-Plugin', 466 | 'Google-Read-Aloud', 467 | 'Google-SearchByImage', 468 | 'Google-Site-Verification', 469 | 'Google-SMTP-STS', 470 | 'Google-speakr', 471 | 'Google-Structured-Data-Testing-Tool', 472 | 'Google-Transparency-Report', 473 | 'google-xrawler', 474 | 'Google-Youtube-Links', 475 | 'GoogleDocs', 476 | 'GoogleHC\\/', 477 | 'GoogleProber', 478 | 'GoogleProducer', 479 | 'GoogleSites', 480 | 'Gookey', 481 | 'GoSpotCheck', 482 | 'gosquared-thumbnailer', 483 | 'Gotit', 484 | 'GoZilla', 485 | 'grabify', 486 | 'GrabNet', 487 | 'Grafula', 488 | 'Grammarly', 489 | 'GrapeFX', 490 | 'GreatNews', 491 | 'Gregarius', 492 | 'GRequests', 493 | 'grokkit', 494 | 'grouphigh', 495 | 'grub-client', 496 | 'gSOAP\\/', 497 | 'GT::WWW', 498 | 'GTmetrix', 499 | 'GuzzleHttp', 500 | 'gvfs\\/', 501 | 'HAA(A)?RTLAND http client', 502 | 'Haansoft', 503 | 'hackney\\/', 504 | 'Hadi Agent', 505 | 'HappyApps-WebCheck', 506 | 'Hardenize', 507 | 'Hatena', 508 | 'Havij', 509 | 'HaxerMen', 510 | 'HeadlessChrome', 511 | 'HEADMasterSEO', 512 | 'HeartRails_Capture', 513 | 'help@dataminr\\.com', 514 | 'heritrix', 515 | 'Hexometer', 516 | 'historious', 517 | 'hkedcity', 518 | 'hledejLevne\\.cz', 519 | 'Hloader', 520 | 'HMView', 521 | 'Holmes', 522 | 'HonesoSearchEngine', 523 | 'HootSuite Image proxy', 524 | 'Hootsuite-WebFeed', 525 | 'hosterstats', 526 | 'HostTracker', 527 | 'ht:\\/\\/check', 528 | 'htdig', 529 | 'HTMLparser', 530 | 'htmlyse', 531 | 'HTTP Banner Detection', 532 | 'http-get', 533 | 'HTTP-Header-Abfrage', 534 | 'http-kit', 535 | 'http-request\\/', 536 | 'HTTP-Tiny', 537 | 'HTTP::Lite', 538 | 'http:\\/\\/www.neomo.de\\/', 539 | 'HttpComponents', 540 | 'httphr', 541 | 'HTTPie', 542 | 'HTTPMon', 543 | 'httpRequest', 544 | 'httpscheck', 545 | 'httpssites_power', 546 | 'httpunit', 547 | 'HttpUrlConnection', 548 | 'http\\.rb\\/', 549 | 'HTTP_Compression_Test', 550 | 'http_get', 551 | 'http_request2', 552 | 'http_requester', 553 | 'httrack', 554 | 'huaweisymantec', 555 | 'HubSpot ', 556 | 'HubSpot-Link-Resolver', 557 | 'Humanlinks', 558 | 'i2kconnect\\/', 559 | 'Iblog', 560 | 'ichiro', 561 | 'Id-search', 562 | 'IdeelaborPlagiaat', 563 | 'IDG Twitter Links Resolver', 564 | 'IDwhois\\/', 565 | 'Iframely', 566 | 'igdeSpyder', 567 | 'iGooglePortal', 568 | 'IlTrovatore', 569 | 'Image Fetch', 570 | 'Image Sucker', 571 | 'ImageEngine\\/', 572 | 'ImageVisu\\/', 573 | 'Imagga', 574 | 'imagineeasy', 575 | 'imgsizer', 576 | 'InAGist', 577 | 'inbound\\.li parser', 578 | 'InDesign%20CC', 579 | 'Indy Library', 580 | 'InetURL', 581 | 'infegy', 582 | 'infohelfer', 583 | 'InfoTekies', 584 | 'InfoWizards Reciprocal Link', 585 | 'inpwrd\\.com', 586 | 'instabid', 587 | 'Instapaper', 588 | 'Integrity', 589 | 'integromedb', 590 | 'Intelliseek', 591 | 'InterGET', 592 | 'Internet Ninja', 593 | 'InternetSeer', 594 | 'internetVista monitor', 595 | 'internetwache', 596 | 'internet_archive', 597 | 'intraVnews', 598 | 'IODC', 599 | 'IOI', 600 | 'iplabel', 601 | 'ips-agent', 602 | 'IPS\\/[0-9]', 603 | 'IPWorks HTTP\\/S Component', 604 | 'iqdb\\/', 605 | 'Iria', 606 | 'Irokez', 607 | 'isitup\\.org', 608 | 'iskanie', 609 | 'isUp\\.li', 610 | 'iThemes Sync\\/', 611 | 'IZaBEE', 612 | 'iZSearch', 613 | 'JAHHO', 614 | 'janforman', 615 | 'Jaunt\\/', 616 | 'Java.*outbrain', 617 | 'javelin\\.io', 618 | 'Jbrofuzz', 619 | 'Jersey\\/', 620 | 'JetCar', 621 | 'Jigsaw', 622 | 'Jobboerse', 623 | 'JobFeed discovery', 624 | 'Jobg8 URL Monitor', 625 | 'jobo', 626 | 'Jobrapido', 627 | 'Jobsearch1\\.5', 628 | 'JoinVision Generic', 629 | 'JolokiaPwn', 630 | 'Joomla', 631 | 'Jorgee', 632 | 'JS-Kit', 633 | 'JungleKeyThumbnail', 634 | 'JustView', 635 | 'Kaspersky Lab CFR link resolver', 636 | 'Kelny\\/', 637 | 'Kerrigan\\/', 638 | 'KeyCDN', 639 | 'Keyword Density', 640 | 'Keywords Research', 641 | 'khttp\\/', 642 | 'KickFire', 643 | 'KimonoLabs\\/', 644 | 'Kml-Google', 645 | 'knows\\.is', 646 | 'KOCMOHABT', 647 | 'kouio', 648 | 'kube-probe', 649 | 'kubectl', 650 | 'kulturarw3', 651 | 'KumKie', 652 | 'Larbin', 653 | 'Lavf\\/', 654 | 'leakix\\.net', 655 | 'LeechFTP', 656 | 'LeechGet', 657 | 'letsencrypt', 658 | 'Lftp', 659 | 'LibVLC', 660 | 'LibWeb', 661 | 'Libwhisker', 662 | 'libwww', 663 | 'Licorne', 664 | 'Liferea\\/', 665 | 'Lighthouse', 666 | 'Lightspeedsystems', 667 | 'Likse', 668 | 'limber\\.io', 669 | 'Link Valet', 670 | 'LinkAlarm\\/', 671 | 'LinkAnalyser', 672 | 'linkCheck', 673 | 'linkdex', 674 | 'LinkExaminer', 675 | 'linkfluence', 676 | 'linkpeek', 677 | 'LinkPreview', 678 | 'LinkScan', 679 | 'LinksManager', 680 | 'LinkTiger', 681 | 'LinkWalker', 682 | 'link_thumbnailer', 683 | 'Lipperhey', 684 | 'Litemage_walker', 685 | 'livedoor ScreenShot', 686 | 'LoadImpactRload', 687 | 'localsearch-web', 688 | 'LongURL API', 689 | 'longurl-r-package', 690 | 'looid\\.com', 691 | 'looksystems\\.net', 692 | 'ltx71', 693 | 'lua-resty-http', 694 | 'Lucee \\(CFML Engine\\)', 695 | 'Lush Http Client', 696 | 'lwp-request', 697 | 'lwp-trivial', 698 | 'LWP::Simple', 699 | 'lycos', 700 | 'LYT\\.SR', 701 | 'L\\.webis', 702 | 'mabontland', 703 | 'MacOutlook\\/', 704 | 'Mag-Net', 705 | 'MagpieRSS', 706 | 'Mail::STS', 707 | 'MailChimp', 708 | 'Mail\\.Ru', 709 | 'Majestic12', 710 | 'makecontact\\/', 711 | 'Mandrill', 712 | 'MapperCmd', 713 | 'marketinggrader', 714 | 'MarkMonitor', 715 | 'MarkWatch', 716 | 'Mass Downloader', 717 | 'masscan\\/', 718 | 'Mata Hari', 719 | 'mattermost', 720 | 'Mediametric', 721 | 'Mediapartners-Google', 722 | 'mediawords', 723 | 'MegaIndex\\.ru', 724 | 'MeltwaterNews', 725 | 'Melvil Rawi', 726 | 'MemGator', 727 | 'Metaspinner', 728 | 'MetaURI', 729 | 'MFC_Tear_Sample', 730 | 'Microsearch', 731 | 'Microsoft Data Access', 732 | 'Microsoft Office', 733 | 'Microsoft Outlook', 734 | 'Microsoft Windows Network Diagnostics', 735 | 'Microsoft-WebDAV-MiniRedir', 736 | 'Microsoft\\.Data\\.Mashup', 737 | 'MIDown tool', 738 | 'MIIxpc', 739 | 'Mindjet', 740 | 'Miniature\\.io', 741 | 'Miniflux', 742 | 'mio_httpc', 743 | 'Miro-HttpClient', 744 | 'Mister PiX', 745 | 'mixdata dot com', 746 | 'mixed-content-scan', 747 | 'mixnode', 748 | 'Mnogosearch', 749 | 'mogimogi', 750 | 'Mojeek', 751 | 'Mojolicious \\(Perl\\)', 752 | 'monitis', 753 | 'Monitority\\/', 754 | 'Monit\\/', 755 | 'montastic', 756 | 'MonTools', 757 | 'Moreover', 758 | 'Morfeus Fucking Scanner', 759 | 'Morning Paper', 760 | 'MovableType', 761 | 'mowser', 762 | 'Mrcgiguy', 763 | 'Mr\\.4x3 Powered', 764 | 'MS Web Services Client Protocol', 765 | 'MSFrontPage', 766 | 'mShots', 767 | 'MuckRack\\/', 768 | 'muhstik-scan', 769 | 'MVAClient', 770 | 'MxToolbox\\/', 771 | 'myseosnapshot', 772 | 'nagios', 773 | 'Najdi\\.si', 774 | 'Name Intelligence', 775 | 'NameFo\\.com', 776 | 'Nameprotect', 777 | 'nationalarchives', 778 | 'Navroad', 779 | 'NearSite', 780 | 'Needle', 781 | 'Nessus', 782 | 'Net Vampire', 783 | 'NetAnts', 784 | 'NETCRAFT', 785 | 'NetLyzer', 786 | 'NetMechanic', 787 | 'NetNewsWire', 788 | 'Netpursual', 789 | 'netresearch', 790 | 'NetShelter ContentScan', 791 | 'Netsparker', 792 | 'NetSystemsResearch', 793 | 'nettle', 794 | 'NetTrack', 795 | 'Netvibes', 796 | 'NetZIP', 797 | 'Neustar WPM', 798 | 'NeutrinoAPI', 799 | 'NewRelicPinger', 800 | 'NewsBlur .*Finder', 801 | 'NewsGator', 802 | 'newsme', 803 | 'newspaper\\/', 804 | 'Nexgate Ruby Client', 805 | 'NG-Search', 806 | 'nghttp2', 807 | 'Nibbler', 808 | 'NICErsPRO', 809 | 'NihilScio', 810 | 'Nikto', 811 | 'nineconnections', 812 | 'NLNZ_IAHarvester', 813 | 'Nmap Scripting Engine', 814 | 'node-fetch', 815 | 'node-superagent', 816 | 'node-urllib', 817 | 'Nodemeter', 818 | 'NodePing', 819 | 'node\\.io', 820 | 'nominet\\.org\\.uk', 821 | 'nominet\\.uk', 822 | 'Norton-Safeweb', 823 | 'Notifixious', 824 | 'notifyninja', 825 | 'NotionEmbedder', 826 | 'nuhk', 827 | 'nutch', 828 | 'Nuzzel', 829 | 'nWormFeedFinder', 830 | 'nyawc\\/', 831 | 'Nymesis', 832 | 'NYU', 833 | 'Observatory\\/', 834 | 'Ocelli\\/', 835 | 'Octopus', 836 | 'oegp', 837 | 'Offline Explorer', 838 | 'Offline Navigator', 839 | 'OgScrper', 840 | 'okhttp', 841 | 'omgili', 842 | 'OMSC', 843 | 'Online Domain Tools', 844 | 'Open Source RSS', 845 | 'OpenCalaisSemanticProxy', 846 | 'Openfind', 847 | 'OpenLinkProfiler', 848 | 'Openstat\\/', 849 | 'OpenVAS', 850 | 'OPPO A33', 851 | 'Optimizer', 852 | 'Orbiter', 853 | 'OrgProbe\\/', 854 | 'orion-semantics', 855 | 'Outlook-Express', 856 | 'Outlook-iOS', 857 | 'Owler', 858 | 'Owlin', 859 | 'ownCloud News', 860 | 'ow\\.ly', 861 | 'OxfordCloudService', 862 | 'page scorer', 863 | 'Page Valet', 864 | 'page2rss', 865 | 'PageFreezer', 866 | 'PageGrabber', 867 | 'PagePeeker', 868 | 'PageScorer', 869 | 'Pagespeed\\/', 870 | 'PageThing', 871 | 'page_verifier', 872 | 'Panopta', 873 | 'panscient', 874 | 'Papa Foto', 875 | 'parsijoo', 876 | 'Pavuk', 877 | 'PayPal IPN', 878 | 'pcBrowser', 879 | 'Pcore-HTTP', 880 | 'PDF24 URL To PDF', 881 | 'Pearltrees', 882 | 'PECL::HTTP', 883 | 'peerindex', 884 | 'Peew', 885 | 'PeoplePal', 886 | 'Perlu -', 887 | 'PhantomJS Screenshoter', 888 | 'PhantomJS\\/', 889 | 'Photon\\/', 890 | 'php-requests', 891 | 'phpservermon', 892 | 'Pi-Monster', 893 | 'Picscout', 894 | 'Picsearch', 895 | 'PictureFinder', 896 | 'Pimonster', 897 | 'Pingability', 898 | 'PingAdmin\\.Ru', 899 | 'Pingdom', 900 | 'Pingoscope', 901 | 'PingSpot', 902 | 'ping\\.blo\\.gs', 903 | 'pinterest\\.com', 904 | 'Pixray', 905 | 'Pizilla', 906 | 'Plagger\\/', 907 | 'Pleroma ', 908 | 'Ploetz \\+ Zeller', 909 | 'Plukkie', 910 | 'plumanalytics', 911 | 'PocketImageCache', 912 | 'PocketParser', 913 | 'Pockey', 914 | 'PodcastAddict\\/', 915 | 'POE-Component-Client-HTTP', 916 | 'Polymail\\/', 917 | 'Pompos', 918 | 'Porkbun', 919 | 'Port Monitor', 920 | 'postano', 921 | 'postfix-mta-sts-resolver', 922 | 'PostmanRuntime', 923 | 'postplanner\\.com', 924 | 'PostPost', 925 | 'postrank', 926 | 'PowerPoint\\/', 927 | 'Prebid', 928 | 'Prerender', 929 | 'Priceonomics Analysis Engine', 930 | 'PrintFriendly', 931 | 'PritTorrent', 932 | 'Prlog', 933 | 'probethenet', 934 | 'Project ?25499', 935 | 'Project-Resonance', 936 | 'prospectb2b', 937 | 'Protopage', 938 | 'ProWebWalker', 939 | 'proximic', 940 | 'PRTG Network Monitor', 941 | 'pshtt, https scanning', 942 | 'PTST ', 943 | 'PTST\\/[0-9]+', 944 | 'Pump', 945 | 'Python-httplib2', 946 | 'python-httpx', 947 | 'python-requests', 948 | 'Python-urllib', 949 | 'Qirina Hurdler', 950 | 'QQDownload', 951 | 'QrafterPro', 952 | 'Qseero', 953 | 'Qualidator', 954 | 'QueryN Metasearch', 955 | 'queuedriver', 956 | 'quic-go-HTTP\\/', 957 | 'QuiteRSS', 958 | 'Quora Link Preview', 959 | 'Qwantify', 960 | 'Radian6', 961 | 'RadioPublicImageResizer', 962 | 'Railgun\\/', 963 | 'RankActive', 964 | 'RankFlex', 965 | 'RankSonicSiteAuditor', 966 | 'RapidLoad\\/', 967 | 'Re-re Studio', 968 | 'ReactorNetty', 969 | 'Readability', 970 | 'RealDownload', 971 | 'RealPlayer%20Downloader', 972 | 'RebelMouse', 973 | 'Recorder', 974 | 'RecurPost\\/', 975 | 'redback\\/', 976 | 'ReederForMac', 977 | 'Reeder\\/', 978 | 'ReGet', 979 | 'RepoMonkey', 980 | 'request\\.js', 981 | 'reqwest\\/', 982 | 'ResponseCodeTest', 983 | 'RestSharp', 984 | 'Riddler', 985 | 'Rival IQ', 986 | 'Robosourcer', 987 | 'Robozilla', 988 | 'ROI Hunter', 989 | 'RPT-HTTPClient', 990 | 'RSSMix\\/', 991 | 'RSSOwl', 992 | 'RyowlEngine', 993 | 'safe-agent-scanner', 994 | 'SalesIntelligent', 995 | 'Saleslift', 996 | 'SAP NetWeaver Application Server', 997 | 'SauceNAO', 998 | 'SBIder', 999 | 'sc-downloader', 1000 | 'scalaj-http', 1001 | 'Scamadviser-Frontend', 1002 | 'ScanAlert', 1003 | 'scan\\.lol', 1004 | 'Scoop', 1005 | 'scooter', 1006 | 'ScopeContentAG-HTTP-Client', 1007 | 'ScoutJet', 1008 | 'ScoutURLMonitor', 1009 | 'ScrapeBox Page Scanner', 1010 | 'Scrapy', 1011 | 'Screaming', 1012 | 'ScreenShotService', 1013 | 'Scrubby', 1014 | 'Scrutiny\\/', 1015 | 'Search37', 1016 | 'searchenginepromotionhelp', 1017 | 'Searchestate', 1018 | 'SearchExpress', 1019 | 'SearchSight', 1020 | 'SearchWP', 1021 | 'search\\.thunderstone', 1022 | 'Seeker', 1023 | 'semanticdiscovery', 1024 | 'semanticjuice', 1025 | 'Semiocast HTTP client', 1026 | 'Semrush', 1027 | 'Sendsay\\.Ru', 1028 | 'sentry\\/', 1029 | 'SEO Browser', 1030 | 'Seo Servis', 1031 | 'seo-nastroj\\.cz', 1032 | 'seo4ajax', 1033 | 'Seobility', 1034 | 'SEOCentro', 1035 | 'SeoCheck', 1036 | 'SEOkicks', 1037 | 'SEOlizer', 1038 | 'Seomoz', 1039 | 'SEOprofiler', 1040 | 'seoscanners', 1041 | 'SEOsearch', 1042 | 'seositecheckup', 1043 | 'SEOstats', 1044 | 'servernfo', 1045 | 'sexsearcher', 1046 | 'Seznam', 1047 | 'Shelob', 1048 | 'Shodan', 1049 | 'Shoppimon', 1050 | 'ShopWiki', 1051 | 'ShortLinkTranslate', 1052 | 'shortURL lengthener', 1053 | 'shrinktheweb', 1054 | 'Sideqik', 1055 | 'Siege', 1056 | 'SimplePie', 1057 | 'SimplyFast', 1058 | 'Siphon', 1059 | 'SISTRIX', 1060 | 'Site Sucker', 1061 | 'Site-Shot\\/', 1062 | 'Site24x7', 1063 | 'SiteBar', 1064 | 'Sitebeam', 1065 | 'Sitebulb\\/', 1066 | 'SiteCondor', 1067 | 'SiteExplorer', 1068 | 'SiteGuardian', 1069 | 'Siteimprove', 1070 | 'SiteIndexed', 1071 | 'Sitemap(s)? Generator', 1072 | 'SitemapGenerator', 1073 | 'SiteMonitor', 1074 | 'Siteshooter B0t', 1075 | 'SiteSnagger', 1076 | 'SiteSucker', 1077 | 'SiteTruth', 1078 | 'Sitevigil', 1079 | 'sitexy\\.com', 1080 | 'SkypeUriPreview', 1081 | 'Slack\\/', 1082 | 'sli-systems\\.com', 1083 | 'slider\\.com', 1084 | 'slurp', 1085 | 'SlySearch', 1086 | 'SmartDownload', 1087 | 'SMRF URL Expander', 1088 | 'SMUrlExpander', 1089 | 'Snake', 1090 | 'Snappy', 1091 | 'SnapSearch', 1092 | 'Snarfer\\/', 1093 | 'SniffRSS', 1094 | 'sniptracker', 1095 | 'Snoopy', 1096 | 'SnowHaze Search', 1097 | 'sogou web', 1098 | 'SortSite', 1099 | 'Sottopop', 1100 | 'sovereign\\.ai', 1101 | 'SpaceBison', 1102 | 'SpamExperts', 1103 | 'Spammen', 1104 | 'Spanner', 1105 | 'spaziodati', 1106 | 'SPDYCheck', 1107 | 'Specificfeeds', 1108 | 'speedy', 1109 | 'SPEng', 1110 | 'Spinn3r', 1111 | 'spray-can', 1112 | 'Sprinklr ', 1113 | 'spyonweb', 1114 | 'sqlmap', 1115 | 'Sqlworm', 1116 | 'Sqworm', 1117 | 'SSL Labs', 1118 | 'ssl-tools', 1119 | 'StackRambler', 1120 | 'Statastico\\/', 1121 | 'Statically-', 1122 | 'StatusCake', 1123 | 'Steeler', 1124 | 'Stratagems Kumo', 1125 | 'Stripe\\/', 1126 | 'Stroke\\.cz', 1127 | 'StudioFACA', 1128 | 'StumbleUpon', 1129 | 'suchen', 1130 | 'Sucuri', 1131 | 'summify', 1132 | 'SuperHTTP', 1133 | 'Surphace Scout', 1134 | 'Suzuran', 1135 | 'swcd ', 1136 | 'Symfony BrowserKit', 1137 | 'Symfony2 BrowserKit', 1138 | 'Synapse\\/', 1139 | 'Syndirella\\/', 1140 | 'SynHttpClient-Built', 1141 | 'Sysomos', 1142 | 'sysscan', 1143 | 'Szukacz', 1144 | 'T0PHackTeam', 1145 | 'tAkeOut', 1146 | 'Tarantula\\/', 1147 | 'Taringa UGC', 1148 | 'TarmotGezgin', 1149 | 'tchelebi\\.io', 1150 | 'techiaith\\.cymru', 1151 | 'TelegramBot', 1152 | 'Teleport', 1153 | 'Telesoft', 1154 | 'Telesphoreo', 1155 | 'Telesphorep', 1156 | 'Tenon\\.io', 1157 | 'teoma', 1158 | 'terrainformatica', 1159 | 'Test Certificate Info', 1160 | 'testuri', 1161 | 'Tetrahedron', 1162 | 'TextRazor Downloader', 1163 | 'The Drop Reaper', 1164 | 'The Expert HTML Source Viewer', 1165 | 'The Intraformant', 1166 | 'The Knowledge AI', 1167 | 'theinternetrules', 1168 | 'TheNomad', 1169 | 'Thinklab', 1170 | 'Thumbor', 1171 | 'Thumbshots', 1172 | 'ThumbSniper', 1173 | 'timewe\\.net', 1174 | 'TinEye', 1175 | 'Tiny Tiny RSS', 1176 | 'TLSProbe\\/', 1177 | 'Toata', 1178 | 'topster', 1179 | 'touche\\.com', 1180 | 'Traackr\\.com', 1181 | 'tracemyfile', 1182 | 'Trackuity', 1183 | 'TrapitAgent', 1184 | 'Trendiction', 1185 | 'Trendsmap', 1186 | 'trendspottr', 1187 | 'truwoGPS', 1188 | 'TryJsoup', 1189 | 'TulipChain', 1190 | 'Turingos', 1191 | 'Turnitin', 1192 | 'tweetedtimes', 1193 | 'Tweetminster', 1194 | 'Tweezler\\/', 1195 | 'twibble', 1196 | 'Twice', 1197 | 'Twikle', 1198 | 'Twingly', 1199 | 'Twisted PageGetter', 1200 | 'Typhoeus', 1201 | 'ubermetrics-technologies', 1202 | 'uclassify', 1203 | 'UdmSearch', 1204 | 'ultimate_sitemap_parser', 1205 | 'unchaos', 1206 | 'unirest-java', 1207 | 'UniversalFeedParser', 1208 | 'unshortenit', 1209 | 'Unshorten\\.It', 1210 | 'Untiny', 1211 | 'UnwindFetchor', 1212 | 'updated', 1213 | 'updown\\.io daemon', 1214 | 'Upflow', 1215 | 'Uptimia', 1216 | 'URL Verifier', 1217 | 'Urlcheckr', 1218 | 'URLitor', 1219 | 'urlresolver', 1220 | 'Urlstat', 1221 | 'URLTester', 1222 | 'UrlTrends Ranking Updater', 1223 | 'URLy Warning', 1224 | 'URLy\\.Warning', 1225 | 'URL\\/Emacs', 1226 | 'Vacuum', 1227 | 'Vagabondo', 1228 | 'VB Project', 1229 | 'vBSEO', 1230 | 'VCI', 1231 | 'vercel', 1232 | 'via ggpht\\.com GoogleImageProxy', 1233 | 'Virusdie', 1234 | 'visionutils', 1235 | 'vkShare', 1236 | 'VoidEYE', 1237 | 'Voil', 1238 | 'voltron', 1239 | 'voyager\\/', 1240 | 'VSAgent\\/', 1241 | 'VSB-TUO\\/', 1242 | 'Vulnbusters Meter', 1243 | 'VYU2', 1244 | 'w3af\\.org', 1245 | 'W3C-checklink', 1246 | 'W3C-mobileOK', 1247 | 'W3C_Unicorn', 1248 | 'WAC-OFU', 1249 | 'WakeletLinkExpander', 1250 | 'WallpapersHD', 1251 | 'Wallpapers\\/[0-9]+', 1252 | 'wangling', 1253 | 'Wappalyzer', 1254 | 'WatchMouse', 1255 | 'WbSrch\\/', 1256 | 'WDT\\.io', 1257 | 'Web Auto', 1258 | 'Web Collage', 1259 | 'Web Enhancer', 1260 | 'Web Fetch', 1261 | 'Web Fuck', 1262 | 'Web Pix', 1263 | 'Web Sauger', 1264 | 'Web spyder', 1265 | 'Web Sucker', 1266 | 'web-capture\\.net', 1267 | 'Web-sniffer', 1268 | 'Webalta', 1269 | 'Webauskunft', 1270 | 'WebAuto', 1271 | 'WebCapture', 1272 | 'WebClient\\/', 1273 | 'webcollage', 1274 | 'WebCookies', 1275 | 'WebCopier', 1276 | 'WebCorp', 1277 | 'WebDataStats', 1278 | 'WebDoc', 1279 | 'WebEnhancer', 1280 | 'WebFetch', 1281 | 'WebFuck', 1282 | 'WebGazer', 1283 | 'WebGo IS', 1284 | 'WebImageCollector', 1285 | 'WebImages', 1286 | 'WebIndex', 1287 | 'webkit2png', 1288 | 'WebLeacher', 1289 | 'webmastercoffee', 1290 | 'webmon ', 1291 | 'WebPix', 1292 | 'WebReaper', 1293 | 'WebSauger', 1294 | 'webscreenie', 1295 | 'Webshag', 1296 | 'Webshot', 1297 | 'Website Quester', 1298 | 'websitepulse agent', 1299 | 'WebsiteQuester', 1300 | 'Websnapr', 1301 | 'WebSniffer', 1302 | 'Webster', 1303 | 'WebStripper', 1304 | 'WebSucker', 1305 | 'webtech\\/', 1306 | 'WebThumbnail', 1307 | 'Webthumb\\/', 1308 | 'WebWhacker', 1309 | 'WebZIP', 1310 | 'WeLikeLinks', 1311 | 'WEPA', 1312 | 'WeSEE', 1313 | 'wf84', 1314 | 'Wfuzz\\/', 1315 | 'wget', 1316 | 'WhatCMS', 1317 | 'WhatsApp', 1318 | 'WhatsMyIP', 1319 | 'WhatWeb', 1320 | 'WhereGoes\\?', 1321 | 'Whibse', 1322 | 'WhoAPI\\/', 1323 | 'WhoRunsCoinHive', 1324 | 'Whynder Magnet', 1325 | 'Windows-RSS-Platform', 1326 | 'WinHttp-Autoproxy-Service', 1327 | 'WinHTTP\\/', 1328 | 'WinPodder', 1329 | 'wkhtmlto', 1330 | 'wmtips', 1331 | 'Woko', 1332 | 'Wolfram HTTPClient', 1333 | 'woorankreview', 1334 | 'WordPress\\/', 1335 | 'WordupinfoSearch', 1336 | 'Word\\/', 1337 | 'worldping-api', 1338 | 'wotbox', 1339 | 'WP Engine Install Performance API', 1340 | 'WP Rocket', 1341 | 'wpif', 1342 | 'wprecon\\.com survey', 1343 | 'WPScan', 1344 | 'wscheck', 1345 | 'Wtrace', 1346 | 'WWW-Collector-E', 1347 | 'WWW-Mechanize', 1348 | 'WWW::Document', 1349 | 'WWW::Mechanize', 1350 | 'WWWOFFLE', 1351 | 'www\\.monitor\\.us', 1352 | 'x09Mozilla', 1353 | 'x22Mozilla', 1354 | 'XaxisSemanticsClassifier', 1355 | 'XenForo\\/', 1356 | 'Xenu Link Sleuth', 1357 | 'XING-contenttabreceiver', 1358 | 'xpymep([0-9]?)\\.exe', 1359 | 'Y!J-[A-Z][A-Z][A-Z]', 1360 | 'Yaanb', 1361 | 'yacy', 1362 | 'Yahoo Link Preview', 1363 | 'YahooCacheSystem', 1364 | 'YahooMailProxy', 1365 | 'YahooYSMcm', 1366 | 'YandeG', 1367 | 'Yandex(?!Search)', 1368 | 'yanga', 1369 | 'yeti', 1370 | 'Yo-yo', 1371 | 'Yoleo Consumer', 1372 | 'yomins\\.com', 1373 | 'yoogliFetchAgent', 1374 | 'YottaaMonitor', 1375 | 'Your-Website-Sucks', 1376 | 'yourls\\.org', 1377 | 'YoYs\\.net', 1378 | 'YP\\.PL', 1379 | 'Zabbix', 1380 | 'Zade', 1381 | 'Zao', 1382 | 'Zauba', 1383 | 'Zemanta Aggregator', 1384 | 'Zend\\\\Http\\\\Client', 1385 | 'Zend_Http_Client', 1386 | 'Zermelo', 1387 | 'Zeus ', 1388 | 'zgrab', 1389 | 'ZnajdzFoto', 1390 | 'ZnHTTP', 1391 | 'Zombie\\.js', 1392 | 'Zoom\\.Mac', 1393 | 'ZoteroTranslationServer', 1394 | 'ZyBorg', 1395 | '[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)', 1396 | ]; 1397 | } 1398 | } 1399 | -------------------------------------------------------------------------------- /src/lib/crawler/exclusions.ts: -------------------------------------------------------------------------------- 1 | import { Provider } from './provider'; 2 | 3 | export class Exclusions implements Provider { 4 | getAll(): string[] { 5 | return [ 6 | 'Safari.[\\d\\.]*', 7 | 'Firefox.[\\d\\.]*', 8 | ' Chrome.[\\d\\.]*', 9 | 'Chromium.[\\d\\.]*', 10 | 'MSIE.[\\d\\.]', 11 | 'Opera\\/[\\d\\.]*', 12 | 'Mozilla.[\\d\\.]*', 13 | 'AppleWebKit.[\\d\\.]*', 14 | 'Trident.[\\d\\.]*', 15 | 'Windows NT.[\\d\\.]*', 16 | 'Android [\\d\\.]*', 17 | 'Macintosh.', 18 | 'Ubuntu', 19 | 'Linux', 20 | '[ ]Intel', 21 | 'Mac OS X [\\d_]*', 22 | '(like )?Gecko(.[\\d\\.]*)?', 23 | 'KHTML,', 24 | 'CriOS.[\\d\\.]*', 25 | 'CPU iPhone OS ([0-9_])* like Mac OS X', 26 | 'CPU OS ([0-9_])* like Mac OS X', 27 | 'iPod', 28 | 'compatible', 29 | 'x86_..', 30 | 'i686', 31 | 'x64', 32 | 'X11', 33 | 'rv:[\\d\\.]*', 34 | 'Version.[\\d\\.]*', 35 | 'WOW64', 36 | 'Win64', 37 | 'Dalvik.[\\d\\.]*', 38 | ' \\.NET CLR [\\d\\.]*', 39 | 'Presto.[\\d\\.]*', 40 | 'Media Center PC', 41 | 'BlackBerry', 42 | 'Build', 43 | 'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.', 44 | 'Opera', 45 | ' \\.NET[\\d\\.]*', 46 | 'cubot', 47 | '; M bot', 48 | '; CRONO', 49 | '; B bot', 50 | '; IDbot', 51 | '; ID bot', 52 | '; POWER BOT', 53 | 'OCTOPUS-CORE', 54 | ]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/lib/crawler/headers.ts: -------------------------------------------------------------------------------- 1 | import { Provider } from './provider'; 2 | 3 | export class Headers implements Provider { 4 | getAll(): string[] { 5 | return [ 6 | 'USER-AGENT', 7 | 'X-OPERAMINI-PHONE-UA', 8 | 'X-DEVICE-USER-AGENT', 9 | 'X-ORIGINAL-USER-AGENT', 10 | 'X-SKYFIRE-PHONE', 11 | 'X-BOLT-PHONE-UA', 12 | 'DEVICE-STOCK-UA', 13 | 'X-UCBROWSER-DEVICE-UA', 14 | 'FROM', 15 | 'X-SCANNER', 16 | ]; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/lib/crawler/provider.ts: -------------------------------------------------------------------------------- 1 | export interface Provider { 2 | getAll(): string[]; 3 | } 4 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import { NextFunction, Request, Response } from 'express-serve-static-core'; 2 | 3 | import { Crawler } from './lib/crawler'; 4 | 5 | export type Middleware = ( 6 | request: Request, 7 | response: Response, 8 | next: NextFunction 9 | ) => Promise; 10 | 11 | declare module 'express-serve-static-core' { 12 | interface Request { 13 | Crawler: Crawler; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test/lib/crawler.spec.ts: -------------------------------------------------------------------------------- 1 | import assert from 'assert'; 2 | import { Request } from 'express-serve-static-core'; 3 | import * as fs from 'fs'; 4 | import * as readline from 'readline'; 5 | 6 | import { Crawler } from '../../src'; 7 | 8 | declare module 'express-serve-static-core' { 9 | interface Request { 10 | Crawler: Crawler; 11 | } 12 | } 13 | 14 | let crawler = new Crawler(); 15 | 16 | describe('regex-compilation', () => { 17 | it('will join list of patterns with pipes', () => { 18 | assert.strictEqual( 19 | crawler.compileRegex(['some', 'patterns']).source, 20 | 'some|patterns' 21 | ); 22 | assert.strictEqual(crawler.compileRegex(['single']).source, 'single'); 23 | }); 24 | 25 | it('keeps the whitespace', () => { 26 | assert.strictEqual( 27 | crawler.compileRegex([' keep-whitespaces ']).source, 28 | ' keep-whitespaces ' 29 | ); 30 | }); 31 | 32 | it('will accept regex-flags for compilation', () => { 33 | const patterns = ['some', 'patterns']; 34 | assert.strictEqual(crawler.compileRegex(patterns, 'g').flags, 'g'); 35 | assert.strictEqual(crawler.compileRegex(patterns, 'i').flags, 'i'); 36 | }); 37 | 38 | it('should be case insensitive', () => { 39 | assert.strictEqual(crawler.isCrawler('Facebot\\1.0'), true); 40 | assert.strictEqual( 41 | crawler.getMatches(), 42 | 'Facebot', 43 | 'Crawler was not able to identify crawler correctly' 44 | ); 45 | }); 46 | }); 47 | 48 | describe('crawler-identification', () => { 49 | it('should be able to identify crawlers', async () => { 50 | const rl = readline.createInterface({ 51 | input: fs.createReadStream('./test/lib/database/crawlers.txt'), 52 | crlfDelay: Infinity, 53 | }); 54 | 55 | for await (const line of rl) { 56 | assert.strictEqual( 57 | crawler.isCrawler(line), 58 | true, 59 | `${line} is not a crawler` 60 | ); 61 | } 62 | 63 | rl.close(); 64 | }); 65 | 66 | it('should be able to identify devices', async () => { 67 | const rl = readline.createInterface({ 68 | input: fs.createReadStream('./test/lib/database/devices.txt'), 69 | crlfDelay: Infinity, 70 | }); 71 | 72 | for await (const line of rl) { 73 | assert.strictEqual( 74 | crawler.isCrawler(line), 75 | false, 76 | `${line} is not a device` 77 | ); 78 | } 79 | 80 | rl.close(); 81 | }); 82 | 83 | it('should identify the crawler from given headers', async () => { 84 | crawler = new Crawler(undefined, { 85 | host: '127.0.0.1:3000', 86 | 'user-agent': 'curl/7.73.0', 87 | accept: '*/*', 88 | }); 89 | 90 | assert.strictEqual(crawler.isCrawler(), true); 91 | }); 92 | 93 | it('should identify the crawler from request headers', async () => { 94 | crawler = new Crawler({ 95 | headers: { 'user-agent': 'curl/7.73.0', accept: '*/*' }, 96 | } as Request); 97 | 98 | assert.strictEqual(crawler.isCrawler(), true); 99 | }); 100 | 101 | it('should identify the crawler from request headers with exact pattern', async () => { 102 | crawler = new Crawler({ 103 | headers: { 'user-agent': 'b0t', accept: '*/*' }, 104 | } as Request); 105 | 106 | assert.strictEqual(crawler.isCrawler(), true); 107 | }); 108 | 109 | it('should not throw an exception on empty request header', async () => { 110 | crawler = new Crawler({ 111 | headers: { accept: '*/*' }, 112 | } as Request); 113 | 114 | assert.doesNotThrow(() => crawler.isCrawler()); 115 | }); 116 | }); 117 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/tsconfig", 3 | "display": "Node 20", 4 | "_version": "20.1.0", 5 | 6 | "compilerOptions": { 7 | "lib": ["es2023"], 8 | "module": "node16", 9 | "target": "es2022", 10 | 11 | "declaration": true, 12 | "declarationMap": true, 13 | "sourceMap": true, 14 | "emitDeclarationOnly": false, 15 | "outDir": "dist", 16 | 17 | "strict": true, 18 | "esModuleInterop": true, 19 | "skipLibCheck": true, 20 | "moduleResolution": "node16" 21 | }, 22 | "ts-node": { 23 | "esm": true, 24 | }, 25 | 26 | "include": ["src/**/*"], 27 | "exclude": ["node_modules", "dist"] 28 | } --------------------------------------------------------------------------------