├── .gitattributes ├── index.js ├── .npmrc ├── .github ├── dependabot.yml └── workflows │ ├── pull_request.yml │ ├── cron.yml │ └── main.yml ├── .editorconfig ├── .gitignore ├── LICENSE.md ├── CHANGELOG.md ├── scripts ├── fetcher.mjs └── generate.mjs ├── README.md ├── package.json └── index.json /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./index.json') 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | audit=false 2 | fund=false 3 | package-lock=false 4 | prefer-dedupe=true 5 | prefer-offline=false 6 | save-prefix=~ 7 | save=false 8 | strict-peer-dependencies=false 9 | unsafe-perm=true 10 | loglevel=error 11 | shamefully-hoist=true 12 | resolution-mode=highest 13 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: '/' 5 | schedule: 6 | interval: daily 7 | - package-ecosystem: 'github-actions' 8 | directory: '/' 9 | schedule: 10 | # Check for updates to GitHub Actions every weekday 11 | interval: 'daily' 12 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # https://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 2 8 | end_of_line = lf 9 | charset = utf-8 10 | trim_trailing_whitespace = true 11 | insert_final_newline = true 12 | max_line_length = 80 13 | indent_brace_style = 1TBS 14 | spaces_around_operators = true 15 | quote_type = auto 16 | 17 | [package.json] 18 | indent_style = space 19 | indent_size = 2 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ############################ 2 | # npm 3 | ############################ 4 | node_modules 5 | npm-debug.log 6 | .node_history 7 | yarn.lock 8 | package-lock.json 9 | 10 | ############################ 11 | # tmp, editor & OS files 12 | ############################ 13 | .tmp 14 | *.swo 15 | *.swp 16 | *.swn 17 | *.swm 18 | .DS_Store 19 | *# 20 | *~ 21 | .idea 22 | *sublime* 23 | nbproject 24 | 25 | ############################ 26 | # Tests 27 | ############################ 28 | testApp 29 | coverage 30 | .nyc_output 31 | 32 | ############################ 33 | # Other 34 | ############################ 35 | .env 36 | .envrc 37 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | name: pull_request 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | test: 13 | if: github.ref != 'refs/heads/master' 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v6 18 | with: 19 | token: ${{ secrets.GITHUB_TOKEN }} 20 | - name: Setup Node.js 21 | uses: actions/setup-node@v6 22 | with: 23 | node-version: lts/* 24 | - name: Setup PNPM 25 | uses: pnpm/action-setup@v4 26 | with: 27 | version: latest 28 | run_install: true 29 | - name: Test 30 | run: npm test 31 | # - name: Report 32 | # run: npx c8 report --reporter=text-lcov > coverage/lcov.info 33 | # - name: Coverage 34 | # uses: coverallsapp/github-action@main 35 | # with: 36 | # github-token: ${{ secrets.GITHUB_TOKEN }} 37 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2023 Kiko Beats (kikobeats.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/cron.yml: -------------------------------------------------------------------------------- 1 | name: cron 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | min_wait_time: 7 | description: 'Minimum wait time' 8 | required: false 9 | default: 5000 10 | max_concurrency: 11 | description: 'Maximum concurrency' 12 | required: false 13 | default: 1 14 | schedule: 15 | # Cron job every Monday at 12:00 16 | # https://crontab.guru/every-monday 17 | - cron: '0 0 * * MON' 18 | 19 | jobs: 20 | test: 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v6 25 | with: 26 | token: ${{ secrets.GITHUB_TOKEN }} 27 | - name: Setup Node.js 28 | uses: actions/setup-node@v6 29 | with: 30 | node-version: lts/* 31 | - name: Setup PNPM 32 | uses: pnpm/action-setup@v4 33 | with: 34 | version: latest 35 | run_install: true 36 | - name: Cron 37 | run: | 38 | git config --global user.email ${{ secrets.GIT_EMAIL }} 39 | git config --global user.name ${{ secrets.GIT_USERNAME }} 40 | MIN_WAIT_TIME="${{ github.event.inputs.min_wait_time }}" \ 41 | MAX_CONCURRENCY="${{ github.event.inputs.max_concurrency }}" \ 42 | npm run cronjob 43 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 4 | 5 | ### 1.0.34 (2025-08-13) 6 | 7 | ### 1.0.33 (2025-07-28) 8 | 9 | ### 1.0.32 (2025-07-28) 10 | 11 | ### 1.0.31 (2025-05-27) 12 | 13 | ### 1.0.30 (2025-05-26) 14 | 15 | ### 1.0.29 (2025-05-05) 16 | 17 | ### 1.0.28 (2024-12-07) 18 | 19 | ### 1.0.27 (2024-05-08) 20 | 21 | ### 1.0.26 (2024-02-09) 22 | 23 | ### 1.0.25 (2024-02-01) 24 | 25 | ### 1.0.24 (2024-02-01) 26 | 27 | ### 1.0.23 (2024-01-01) 28 | 29 | ### 1.0.22 (2024-01-01) 30 | 31 | ### 1.0.21 (2023-11-25) 32 | 33 | ### 1.0.20 (2023-11-20) 34 | 35 | ### 1.0.19 (2023-11-13) 36 | 37 | ### 1.0.18 (2023-11-09) 38 | 39 | ### 1.0.17 (2023-11-06) 40 | 41 | ### 1.0.16 (2023-10-30) 42 | 43 | ### 1.0.15 (2023-10-24) 44 | 45 | ### 1.0.14 (2023-10-23) 46 | 47 | ### 1.0.13 (2023-10-20) 48 | 49 | ### 1.0.12 (2023-10-16) 50 | 51 | ### 1.0.11 (2023-10-09) 52 | 53 | ### 1.0.10 (2023-10-02) 54 | 55 | ### 1.0.9 (2023-10-01) 56 | 57 | ### 1.0.8 (2023-09-28) 58 | 59 | ### 1.0.7 (2023-09-18) 60 | 61 | ### 1.0.6 (2023-09-11) 62 | 63 | ### 1.0.5 (2023-09-07) 64 | 65 | ### 1.0.4 (2023-09-04) 66 | 67 | ### 1.0.3 (2023-08-28) 68 | 69 | ### 1.0.2 (2023-08-14) 70 | 71 | ### 1.0.1 (2023-08-10) 72 | 73 | ## [1.0.0](https://github.com/Kikobeats/top-crawler-agents/compare/v0.0.1...v1.0.0) (2023-08-06) 74 | 75 | ### 0.0.1 (2023-08-06) 76 | -------------------------------------------------------------------------------- /scripts/fetcher.mjs: -------------------------------------------------------------------------------- 1 | import createBrowser from 'browserless' 2 | import { onExit } from 'signal-exit' 3 | import puppeteer from 'puppeteer' 4 | 5 | const REQ_TIMEOUT = 10000 6 | 7 | let singletonBrowser = null 8 | 9 | const browser = async () => { 10 | if (!singletonBrowser) { 11 | const browser = createBrowser({ puppeteer }) 12 | onExit(browser.close) 13 | singletonBrowser = browser 14 | } 15 | return singletonBrowser.createContext() 16 | } 17 | 18 | export const withPrerender = async (url, userAgent) => { 19 | const browserless = await browser() 20 | const fn = browserless.evaluate( 21 | async (_, response, error) => { 22 | if (error) throw error 23 | const status = response.status() 24 | if (response.status() !== 200) { 25 | throw new TypeError(`Response status: ${status}`) 26 | } 27 | 28 | return { 29 | html: await response.text(), 30 | statusCode: status 31 | } 32 | }, 33 | { 34 | headers: { 'user-agent': userAgent } 35 | } 36 | ) 37 | 38 | const result = await fn(url) 39 | await browserless.destroyContext() 40 | return result 41 | } 42 | 43 | export const withFetch = async (url, userAgent) => { 44 | const controller = new AbortController() 45 | setTimeout(() => controller.abort(), REQ_TIMEOUT) 46 | const res = await fetch(url, { 47 | signal: controller.signal, 48 | headers: { 'user-agent': userAgent, redirect: 'manual' } 49 | }) 50 | const html = await res.text() 51 | return { html, statusCode: res.status } 52 | } 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # top-crawler-agents 2 | 3 | ![Last version](https://img.shields.io/github/tag/Kikobeats/top-crawler-agents.svg?style=flat-square) 4 | [![NPM Status](https://img.shields.io/npm/dm/top-crawler-agents.svg?style=flat-square)](https://www.npmjs.org/package/top-crawler-agents) 5 | 6 | > A list of common crawler user agents useful for retrieving metadata from links
7 | > derivated from [crawler-user-agents](https://github.com/monperrus/crawler-user-agents). 8 | 9 | Some websites (such as [Twitter](https://twitter.com/Kikobeats/status/1687844145019092993)) only return rich HTML metadata if you are identified as popular crawler (like Slack, or WhatsApp). 10 | 11 | The list is derivated from [crawler-user-agents](https://github.com/monperrus/crawler-user-agents) after applying a [script](/scripts/generate.js), so it's always up-to-date to latest changes. 12 | 13 | ## Install 14 | 15 | ```bash 16 | $ npm install top-crawler-agents --save 17 | ``` 18 | 19 | ## Usage 20 | 21 | ```js 22 | const uniqueRandomArray = require('unique-random-array') 23 | 24 | const randomCrawlerAgent = uniqueRandomArray(require('top-crawler-agents')) 25 | 26 | console.log(randomCrawlerAgent()) 27 | // => 'Slackbot-LinkExpanding (+https://api.slack.com/robots)' 28 | ``` 29 | 30 | ## Related 31 | 32 | - [top-user-agents](https://github.com/Kikobeats/top-user-agents) – A list of most common User Agent used on Internet. 33 | 34 | **top-crawler-agents** © [Kiko Beats](https://kikobeats.com), released under the [MIT](https://github.com/Kikobeats/top-crawler-agents/blob/master/LICENSE.md) License.
35 | Authored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/Kikobeats/top-crawler-agents/contributors). 36 | 37 | > [kikobeats.com](https://kikobeats.com) · GitHub [Kiko Beats](https://github.com/Kikobeats) · X [@Kikobeats](https://x.com/Kikobeats) 38 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: main 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | contributors: 10 | if: "${{ github.event.head_commit.message != 'build: contributors' }}" 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v6 15 | with: 16 | fetch-depth: 0 17 | token: ${{ secrets.GITHUB_TOKEN }} 18 | - name: Setup Node.js 19 | uses: actions/setup-node@v6 20 | with: 21 | node-version: lts/* 22 | - name: Contributors 23 | run: | 24 | git config --global user.email ${{ secrets.GIT_EMAIL }} 25 | git config --global user.name ${{ secrets.GIT_USERNAME }} 26 | npm run contributors 27 | - name: Push changes 28 | run: | 29 | git push origin ${{ github.head_ref }} 30 | 31 | release: 32 | if: | 33 | !startsWith(github.event.head_commit.message, 'chore(release):') && 34 | !startsWith(github.event.head_commit.message, 'docs:') && 35 | !startsWith(github.event.head_commit.message, 'ci:') 36 | needs: [contributors] 37 | runs-on: ubuntu-latest 38 | steps: 39 | - name: Checkout 40 | uses: actions/checkout@v6 41 | with: 42 | token: ${{ secrets.GITHUB_TOKEN }} 43 | - name: Setup Node.js 44 | uses: actions/setup-node@v6 45 | with: 46 | node-version: lts/* 47 | - name: Setup PNPM 48 | uses: pnpm/action-setup@v4 49 | with: 50 | version: latest 51 | run_install: true 52 | - name: Test 53 | run: npm test 54 | # - name: Report 55 | # run: npx c8 report --reporter=text-lcov > coverage/lcov.info 56 | # - name: Coverage 57 | # uses: coverallsapp/github-action@main 58 | # with: 59 | # github-token: ${{ secrets.GITHUB_TOKEN }} 60 | - name: Release 61 | env: 62 | GH_TOKEN: ${{ secrets.GH_TOKEN }} 63 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 64 | run: | 65 | git config --global user.email ${{ secrets.GIT_EMAIL }} 66 | git config --global user.name ${{ secrets.GIT_USERNAME }} 67 | git pull origin master 68 | npm run release 69 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "top-crawler-agents", 3 | "description": "A list of common crawler agents used on Internet..", 4 | "homepage": "https://github.com/Kikobeats/top-crawler-agents", 5 | "version": "1.0.34", 6 | "main": "index.js", 7 | "author": { 8 | "email": "josefrancisco.verdu@gmail.com", 9 | "name": "Kiko Beats", 10 | "url": "https://kikobeats.com" 11 | }, 12 | "contributors": [ 13 | { 14 | "name": "omrilotan", 15 | "email": "omrilotan@hotmail.com" 16 | } 17 | ], 18 | "repository": { 19 | "type": "git", 20 | "url": "git+https://github.com/Kikobeats/top-crawler-agents.git" 21 | }, 22 | "bugs": { 23 | "url": "https://github.com/Kikobeats/top-crawler-agents/issues" 24 | }, 25 | "keywords": [ 26 | "agent", 27 | "common", 28 | "popular", 29 | "string", 30 | "top", 31 | "ua", 32 | "user", 33 | "useragent" 34 | ], 35 | "devDependencies": { 36 | "@commitlint/cli": "latest", 37 | "@commitlint/config-conventional": "latest", 38 | "@ksmithut/prettier-standard": "latest", 39 | "bottleneck": "latest", 40 | "browserless": "latest", 41 | "cheerio": "latest", 42 | "ci-publish": "latest", 43 | "crawler-user-agents": "latest", 44 | "finepack": "latest", 45 | "git-authors-cli": "latest", 46 | "github-generate-release": "latest", 47 | "nano-staged": "latest", 48 | "p-every": "latest", 49 | "p-filter": "2", 50 | "puppeteer": "latest", 51 | "simple-git-hooks": "latest", 52 | "standard": "latest", 53 | "standard-markdown": "latest", 54 | "standard-version": "latest" 55 | }, 56 | "engines": { 57 | "node": ">= 18" 58 | }, 59 | "files": [ 60 | "index.js", 61 | "index.json" 62 | ], 63 | "scripts": { 64 | "clean": "rm -rf node_modules", 65 | "contributors": "(npx git-authors-cli && npx finepack && git add package.json && git commit -m 'build: contributors' --no-verify) || true", 66 | "cronjob": "npm run update:crawler-agents && npm run healthcheck", 67 | "healthcheck": "curl -fsS -m 10 --retry 5 -o /dev/null https://hc-ping.com/b7995d4b-75cb-4434-9338-8c1aa4679dc6", 68 | "lint": "standard-markdown README.md", 69 | "postrelease": "npm run release:tags && npm run release:github && (ci-publish || npm publish --access=public)", 70 | "pretest": "npm run lint", 71 | "release": "standard-version -a", 72 | "release:github": "github-generate-release", 73 | "release:tags": "git push --follow-tags origin HEAD:master", 74 | "test": "exit 0", 75 | "update:crawler-agents": "(node scripts/generate.mjs && git add index.json && git commit -m 'build(update): crawlers' --no-verify && git push) || true" 76 | }, 77 | "license": "MIT", 78 | "commitlint": { 79 | "extends": [ 80 | "@commitlint/config-conventional" 81 | ], 82 | "rules": { 83 | "body-max-line-length": [ 84 | 0 85 | ] 86 | } 87 | }, 88 | "nano-staged": { 89 | "*.js": [ 90 | "prettier-standard", 91 | "standard --fix" 92 | ], 93 | "*.md": [ 94 | "standard-markdown" 95 | ], 96 | "package.json": [ 97 | "finepack" 98 | ] 99 | }, 100 | "pnpm": { 101 | "neverBuiltDependencies": [] 102 | }, 103 | "simple-git-hooks": { 104 | "commit-msg": "npx commitlint --edit", 105 | "pre-commit": "npx nano-staged" 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /scripts/generate.mjs: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | import { createRequire } from 'module' 4 | import { writeFile } from 'fs/promises' 5 | import Bottleneck from 'bottleneck' 6 | // import { writeFileSync } from 'fs' 7 | import { styleText } from 'util' 8 | import { load } from 'cheerio' 9 | import pFilter from 'p-filter' 10 | import pEvery from 'p-every' 11 | 12 | import { 13 | withFetch 14 | /* withPrerender */ 15 | } from './fetcher.mjs' 16 | 17 | const CHECK = { true: '✅', false: '❌' } 18 | const MAX_CONCURRENCY = Number(process.env.MAX_CONCURRENCY) || 1 19 | const MIN_WAIT_TIME = Number(process.env.MIN_WAIT_TIME) || 5000 20 | 21 | const VERIFICATIONS = [ 22 | [ 23 | 'https://twitter.com/Kikobeats/status/1687837848802578432', 24 | $ => !!$('meta[property="og:image"]').attr('content'), 25 | withFetch 26 | ] 27 | // [ 28 | // 'https://www.youtube.com/watch?v=vkddaKFgO5g&app=desktop', 29 | // $ => $('title').text() === 'INFINITO AL 40% - YouTube', 30 | // withPrerender 31 | // ] 32 | ] 33 | 34 | const SOURCES = [ 35 | async () => { 36 | const res = await fetch( 37 | 'https://raw.githubusercontent.com/arcjet/well-known-bots/main/well-known-bots.json' 38 | ) 39 | const json = await res.json() 40 | return json.map(item => item.instances.accepted).flat() 41 | }, 42 | () => { 43 | const mod = createRequire(import.meta.url)( 44 | 'crawler-user-agents/crawler-user-agents.json' 45 | ) 46 | const userAgents = mod.map(item => item.instances).flat() 47 | return userAgents 48 | } 49 | ] 50 | 51 | const userAgents = [ 52 | ...new Set((await Promise.all(SOURCES.map(fn => fn()))).flat()) 53 | ].sort((a, b) => a.localeCompare(b)) 54 | 55 | const total = userAgents.length 56 | 57 | if (total === 0) { 58 | console.error('No user agents found to verify.') 59 | process.exit(1) 60 | } 61 | 62 | const limiter = new Bottleneck({ 63 | minTime: MIN_WAIT_TIME 64 | }) 65 | 66 | const verify = (userAgent, index) => { 67 | if (index !== 0) console.log() 68 | console.log(`[${index + 1}/${total}] ${userAgent}\n`) 69 | return pEvery( 70 | VERIFICATIONS, 71 | async ([url, verifyFn, fetcher]) => 72 | limiter.schedule(async () => { 73 | let result = false 74 | let statusCode 75 | try { 76 | const { html, statusCode: _statusCode } = await fetcher( 77 | url, 78 | userAgent 79 | ) 80 | statusCode = _statusCode 81 | result = await verifyFn(load(html)) 82 | } catch (error) { 83 | console.log('ERROR', error) 84 | } 85 | console.log( 86 | ' ' + styleText('gray', `${url} (${statusCode}) ${CHECK[result]}`) 87 | ) 88 | return result 89 | }), 90 | { concurrency: MAX_CONCURRENCY } 91 | ) 92 | } 93 | 94 | console.log() 95 | console.log(`Starting verification of user agents MAX_CONCURRENCY=${MAX_CONCURRENCY} MIN_WAIT_TIME=${MIN_WAIT_TIME}`) 96 | console.log() 97 | 98 | Promise.resolve() 99 | .then(() => 100 | pFilter(userAgents, verify, { 101 | concurrency: MAX_CONCURRENCY 102 | }) 103 | ) 104 | .then(async result => { 105 | if (result.length === 0) { 106 | throw new Error('No user agents passed the verification.') 107 | } 108 | const sorted = result.sort((a, b) => a.localeCompare(b)) 109 | await writeFile('index.json', JSON.stringify(sorted, null, 2)) 110 | console.log(`\nGenerated ${sorted.length} crawlers ✨`) 111 | }).catch(error => { 112 | console.error('Error during verification:', error) 113 | process.exit(1) 114 | }) 115 | -------------------------------------------------------------------------------- /index.json: -------------------------------------------------------------------------------- 1 | [ 2 | "'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)'", 3 | "@LinkArchiver twitter bot", 4 | "2ip bot/1.1 (+http://2ip.io)", 5 | "adbeat_bot", 6 | "AdsBot-Google-Mobile-Apps", 7 | "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)", 8 | "ArchiveTeam ArchiveBot/20170106.02 (wpull 2.0.2)", 9 | "Audisto Crawler (desktop; +https://audisto.com/bot)", 10 | "Audisto Crawler (desktop; essential; +https://audisto.com/bot)", 11 | "Audisto Crawler (mobile; +https://audisto.com/bot)", 12 | "Audisto Crawler (mobile; essential; +https://audisto.com/bot)", 13 | "AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)", 14 | "AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)", 15 | "B2B Bot", 16 | "Baidu-YunGuanCe-Bot(ce.baidu.com)", 17 | "Baidu-YunGuanCe-PerfBot(ce.baidu.com)", 18 | "Baidu-YunGuanCe-ScanBot(ce.baidu.com)", 19 | "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", 20 | "blogmuraBot (+http://www.blogmura.com)", 21 | "bot-pge.chlooe.com/1.0.0 (+http://www.chlooe.com/)", 22 | "BublupBot (+https://www.bublup.com/bublup-bot.html)", 23 | "Clickagy Intelligence Bot v2", 24 | "Cliqzbot/0.1 (+http://cliqz.com +cliqzbot@cliqz.com)", 25 | "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)", 26 | "coccoc", 27 | "coccoc/1.0 ()", 28 | "coccoc/1.0 (http://help.coccoc.com/)", 29 | "coccoc/1.0 (http://help.coccoc.vn/)", 30 | "ContextAd Bot 1.0", 31 | "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)", 32 | "CrunchBot/1.0 (+http://www.leadcrunch.com/crunchbot)", 33 | "datagnionbot (+http://www.datagnion.com/bot.html)", 34 | "deepnoc - https://deepnoc.com/bot", 35 | "Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com", 36 | "DomainStatsBot/1.0 (http://domainstats.io/our-bot)", 37 | "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)", 38 | "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)", 39 | "facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)", 40 | "facebookexternalhit/1.1", 41 | "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", 42 | "Facebot/1.0", 43 | "FeedlyBot/1.0 (http://feedly.com)", 44 | "Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)", 45 | "FreeWebMonitoring SiteChecker/0.2 (+https://www.freewebmonitoring.com/bot.html)", 46 | "FreshpingBot/1.0 (+https://freshping.io/)", 47 | "g2reader-bot/1.0 (+http://www.g2reader.com/)", 48 | "Google-CloudVertexBot", 49 | "Googlebot-News", 50 | "GroupMeBot/1.0", 51 | "http://seewithkids.com/bot", 52 | "INETDEX-BOT/1.5 (Mozilla/5.0; https://inetdex.com/bot.html)", 53 | "infoobot/0.1 (https://www.infoo.nl/bot.html)", 54 | "jpg-newsbot/2.0; (+https://vipnytt.no/bots/)", 55 | "Landau-Media-Spider/1.0(http://bots.landaumedia.de/bot.html)", 56 | "Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)", 57 | "Linguee Bot (http://www.linguee.com/bot)", 58 | "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)", 59 | "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)", 60 | "LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/4.3 +http://www.linkedin.com)", 61 | "LivelapBot/0.2 (http://site.livelap.com/crawler)", 62 | "MauiBot (crawler.feedback+wc@gmail.com)", 63 | "MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php?+)", 64 | "MojeekBot/0.2 (archi; http://www.mojeek.com/bot.html)", 65 | "MoodleBot/1.0", 66 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;.NET CLR 1.0.3705; ContextAd Bot 1.0)", 67 | "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )", 68 | "Mozilla/5.0 (compatible; adbeat_bot; +support@adbeat.com; support@adbeat.com)", 69 | "Mozilla/5.0 (compatible; AddSearchBot/0.9; +http://www.addsearch.com/bot; info@addsearch.com)", 70 | "Mozilla/5.0 (compatible; AdvBot/2.0; +http://advbot.net/bot.html)", 71 | "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)", 72 | "Mozilla/5.0 (compatible; AhrefsBot/5.2; News; +http://ahrefs.com/robot/)", 73 | "Mozilla/5.0 (compatible; AhrefsBot/6.1; +http://ahrefs.com/robot/)", 74 | "Mozilla/5.0 (compatible; AhrefsBot/6.1; News; +http://ahrefs.com/robot/)", 75 | "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)", 76 | "Mozilla/5.0 (compatible; AlphaBot/3.2; +http://alphaseobot.com/bot.html)", 77 | "Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)", 78 | "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", 79 | "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)", 80 | "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)", 81 | "Mozilla/5.0 (compatible; archive.org_bot/heritrix-1.15.4 +http://www.archive.org)", 82 | "Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)", 83 | "Mozilla/5.0 (compatible; BehloolBot/beta; +http://www.webeaver.com/bot)", 84 | "Mozilla/5.0 (compatible; BitSightBot/1.0)", 85 | "Mozilla/5.0 (compatible; BlogTraffic/1.4 Feed-Fetcher; +http://www.blogtraffic.de/rss-bot.html)", 86 | "Mozilla/5.0 (compatible; bnf.fr_bot; +http://bibnum.bnf.fr/robot/bnf.html)", 87 | "Mozilla/5.0 (compatible; bnf.fr_bot; +http://www.bnf.fr/fr/outils/a.dl_web_capture_robot.html)", 88 | "Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)", 89 | "Mozilla/5.0 (compatible; BoxcarBot/1.1; +awesome@boxcar.io)", 90 | "Mozilla/5.0 (compatible; Cincraw/1.0; +http://cincrawdata.net/bot/)", 91 | "Mozilla/5.0 (compatible; Cliqzbot/0.1 +http://cliqz.com/company/cliqzbot)", 92 | "Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot)", 93 | "Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot)", 94 | "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)", 95 | "Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/searchengine)", 96 | "Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)", 97 | "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", 98 | "Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)", 99 | "Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)", 100 | "Mozilla/5.0 (compatible; Digincore bot; https://www.digincore.com/crawler.html for rules and instructions.)", 101 | "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", 102 | "Mozilla/5.0 (compatible; DnyzBot/1.0)", 103 | "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)", 104 | "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)", 105 | "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)", 106 | "Mozilla/5.0 (compatible; ExtLinksBot/1.5 +https://extlinks.com/Bot.html)", 107 | "Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)", 108 | "Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)", 109 | "Mozilla/5.0 (compatible; FemtosearchBot/1.0; http://femtosearch.com)", 110 | "Mozilla/5.0 (compatible; GeedoBot; +http://www.geedo.com/bot.html)", 111 | "Mozilla/5.0 (compatible; Google-InspectionTool/1.0)", 112 | "Mozilla/5.0 (compatible; heritrix/3.1.1-SNAPSHOT-20120116.200628 +http://www.archive.org/details/archive.org_bot)", 113 | "Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)", 114 | "Mozilla/5.0 (compatible; hypestat/1.0; +https://hypestat.com/bot)", 115 | "Mozilla/5.0 (compatible; image.coccoc/1.0; +http://help.coccoc.com/)", 116 | "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/)", 117 | "Mozilla/5.0 (compatible; imagecoccoc/1.0; +http://help.coccoc.com/searchengine)", 118 | "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", 119 | "Mozilla/5.0 (compatible; IstellaBot/1.23.15 +http://www.tiscali.it/)", 120 | "Mozilla/5.0 (compatible; Jooblebot/2.0; Windows NT 6.1; WOW64; +http://jooble.org/jooble-bot) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36", 121 | "Mozilla/5.0 (compatible; KeybaseBot; +https://keybase.io)", 122 | "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", 123 | "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/about/bots/)", 124 | "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)", 125 | "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/about/bots/)", 126 | "Mozilla/5.0 (compatible; linkdexbot/2.1; +http://www.linkdex.com/bots/)", 127 | "Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)", 128 | "Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321", 129 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)", 130 | "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)", 131 | "Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)", 132 | "Mozilla/5.0 (compatible; memorybot/1.21.14 +http://mignify.com/bot.html)", 133 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.1; http://www.majestic12.co.uk/bot.php?+)", 134 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.3; http://www.majestic12.co.uk/bot.php?+)", 135 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.4; http://www.majestic12.co.uk/bot.php?+)", 136 | "Mozilla/5.0 (compatible; MJ12bot/v1.2.5; http://www.majestic12.co.uk/bot.php?+)", 137 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.0; http://www.majestic12.co.uk/bot.php?+)", 138 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.1; http://www.majestic12.co.uk/bot.php?+)", 139 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.2; http://www.majestic12.co.uk/bot.php?+)", 140 | "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)", 141 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.0; http://www.majestic12.co.uk/bot.php?+)", 142 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.1; http://www.majestic12.co.uk/bot.php?+)", 143 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)", 144 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.3; http://www.majestic12.co.uk/bot.php?+)", 145 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.4 (domain ownership verifier); http://www.majestic12.co.uk/bot.php?+)", 146 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)", 147 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)", 148 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.6; http://mj12bot.com/)", 149 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)", 150 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://www.majestic12.co.uk/bot.php?+)", 151 | "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)", 152 | "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html)", 153 | "Mozilla/5.0 (compatible; MojeekBot/0.2; http://www.mojeek.com/bot.html#relaunch)", 154 | "Mozilla/5.0 (compatible; MojeekBot/0.5; http://www.mojeek.com/bot.html)", 155 | "Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)", 156 | "Mozilla/5.0 (compatible; MojeekBot/0.6; http://www.mojeek.com/bot.html)", 157 | "Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; info@monsido.com)", 158 | "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1) Streamline3Bot/1.0", 159 | "Mozilla/5.0 (compatible; Neevabot/1.0; +https://neeva.com/neevabot)", 160 | "Mozilla/5.0 (compatible; NerdByNature.Bot; http://www.nerdbynature.net/bot)", 161 | "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)", 162 | "Mozilla/5.0 (compatible; NIXStatsbot/1.1; +http://www.nixstats.com/bot.html)", 163 | "Mozilla/5.0 (compatible; OdklBot/1.0 like Linux; klass@odnoklassniki.ru)", 164 | "Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)", 165 | "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)", 166 | "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", 167 | "Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)", 168 | "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.1; +https://help.qwant.com/bot)", 169 | "Mozilla/5.0 (compatible; Qwantify/Bleriot/1.2.1; +https://help.qwant.com/bot)", 170 | "Mozilla/5.0 (compatible; RankActiveLinkBot; +https://rankactive.com/resources/rankactive-linkbot)", 171 | "Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)", 172 | "Mozilla/5.0 (compatible; RegionStuttgartBot/1.0; +http://it.region-stuttgart.de/competenzatlas/unternehmen-suchen/)", 173 | "Mozilla/5.0 (compatible; ReverseEngineeringBot/0.1; +https://torus.company/bot.html)", 174 | "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co)", 175 | "Mozilla/5.0 (compatible; RidderBot/1.0; bot@ridder.co) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321", 176 | "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)", 177 | "Mozilla/5.0 (compatible; SBIntuitionsBot/0.1; +https://www.sbintuitions.co.jp/bot/)", 178 | "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)", 179 | "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)", 180 | "Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)", 181 | "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", 182 | "Mozilla/5.0 (compatible; SemrushBot-SA/0.97; +http://www.semrush.com/bot.html)", 183 | "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", 184 | "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)", 185 | "Mozilla/5.0 (compatible; SemrushBot/3~bl; +http://www.semrush.com/bot.html)", 186 | "Mozilla/5.0 (compatible; SemrushBot/6~bl; +http://www.semrush.com/bot.html)", 187 | "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", 188 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test1-1; +http://napoveda.seznam.cz/en/seznambot-intro/)", 189 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test1; +http://napoveda.seznam.cz/en/seznambot-intro/)", 190 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test2; +http://napoveda.seznam.cz/en/seznambot-intro/)", 191 | "Mozilla/5.0 (compatible; SeznamBot/3.2-test4; +http://napoveda.seznam.cz/en/seznambot-intro/)", 192 | "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)", 193 | "Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)", 194 | "Mozilla/5.0 (compatible; spbot/1.0; +http://www.seoprofiler.com/bot/ )", 195 | "Mozilla/5.0 (compatible; spbot/1.1; +http://www.seoprofiler.com/bot/ )", 196 | "Mozilla/5.0 (compatible; spbot/1.2; +http://www.seoprofiler.com/bot/ )", 197 | "Mozilla/5.0 (compatible; spbot/2.0; +http://www.seoprofiler.com/bot/ )", 198 | "Mozilla/5.0 (compatible; spbot/2.0.1; +http://www.seoprofiler.com/bot/ )", 199 | "Mozilla/5.0 (compatible; spbot/2.0.2; +http://www.seoprofiler.com/bot/ )", 200 | "Mozilla/5.0 (compatible; spbot/2.0.3; +http://www.seoprofiler.com/bot/ )", 201 | "Mozilla/5.0 (compatible; spbot/2.0.4; +http://www.seoprofiler.com/bot )", 202 | "Mozilla/5.0 (compatible; spbot/2.1; +http://www.seoprofiler.com/bot )", 203 | "Mozilla/5.0 (compatible; spbot/3.0; +http://www.seoprofiler.com/bot )", 204 | "Mozilla/5.0 (compatible; spbot/3.1; +http://www.seoprofiler.com/bot )", 205 | "Mozilla/5.0 (compatible; spbot/4.0; +http://www.seoprofiler.com/bot )", 206 | "Mozilla/5.0 (compatible; spbot/4.0.1; +http://www.seoprofiler.com/bot )", 207 | "Mozilla/5.0 (compatible; spbot/4.0.2; +http://www.seoprofiler.com/bot )", 208 | "Mozilla/5.0 (compatible; spbot/4.0.3; +http://www.seoprofiler.com/bot )", 209 | "Mozilla/5.0 (compatible; spbot/4.0.4; +http://www.seoprofiler.com/bot )", 210 | "Mozilla/5.0 (compatible; spbot/4.0.5; +http://www.seoprofiler.com/bot )", 211 | "Mozilla/5.0 (compatible; spbot/4.0.6; +http://www.seoprofiler.com/bot )", 212 | "Mozilla/5.0 (compatible; spbot/4.0.7; +http://OpenLinkProfiler.org/bot )", 213 | "Mozilla/5.0 (compatible; spbot/4.0.7; +https://www.seoprofiler.com/bot )", 214 | "Mozilla/5.0 (compatible; spbot/4.0.8; +http://OpenLinkProfiler.org/bot )", 215 | "Mozilla/5.0 (compatible; spbot/4.0.9; +http://OpenLinkProfiler.org/bot )", 216 | "Mozilla/5.0 (compatible; spbot/4.0a; +http://www.seoprofiler.com/bot )", 217 | "Mozilla/5.0 (compatible; spbot/4.0b; +http://www.seoprofiler.com/bot )", 218 | "Mozilla/5.0 (compatible; spbot/4.1.0; +http://OpenLinkProfiler.org/bot )", 219 | "Mozilla/5.0 (compatible; spbot/4.2.0; +http://OpenLinkProfiler.org/bot )", 220 | "Mozilla/5.0 (compatible; spbot/4.3.0; +http://OpenLinkProfiler.org/bot )", 221 | "Mozilla/5.0 (compatible; spbot/4.4.0; +http://OpenLinkProfiler.org/bot )", 222 | "Mozilla/5.0 (compatible; spbot/4.4.1; +http://OpenLinkProfiler.org/bot )", 223 | "Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )", 224 | "Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )", 225 | "Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )", 226 | "Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )", 227 | "Mozilla/5.0 (compatible; spbot/5.0.3; +http://OpenLinkProfiler.org/bot )", 228 | "Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)", 229 | "Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)", 230 | "Mozilla/5.0 (compatible; StorygizeBot; http://www.storygize.com)", 231 | "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)", 232 | "Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html; Linux; Android 4; iPhone; CPU iPhone OS 6_0_1 like Mac OS X)", 233 | "Mozilla/5.0 (compatible; t3versionsBot/1.0; +https://www.t3versions.com/bot)", 234 | "Mozilla/5.0 (compatible; TinEye-bot/1.31; +http://www.tineye.com/crawler.html)", 235 | "Mozilla/5.0 (compatible; trovitBot 1.0; +http://www.trovit.com/bot.html)", 236 | "Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html)", 237 | "Mozilla/5.0 (compatible; Vigil/1.0; +http://vigil-app.com/bot.html)", 238 | "Mozilla/5.0 (compatible; VoluumDSP-content-bot/2.0; +dsp-dev@codewise.com)", 239 | "Mozilla/5.0 (compatible; vuhuvBot/1.0; +http://vuhuv.com/bot.html)", 240 | "Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)", 241 | "Mozilla/5.0 (compatible; XoviBot/2.0; +http://www.xovibot.net/)", 242 | "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)", 243 | "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", 244 | "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)", 245 | "Mozilla/5.0 (compatible; Yellowbrandprotectionbot/1.0; +https://www.yellowbp.com/bot.html)", 246 | "Mozilla/5.0 (compatible; Yeti/1.1; +http://naver.me/bot)", 247 | "Mozilla/5.0 (compatible; yoozBot-2.2; http://yooz.ir; info@yooz.ir)", 248 | "Mozilla/5.0 (compatible; zenback bot; powered by logly +http://corp.logly.co.jp/)", 249 | "Mozilla/5.0 (compatible; ZuperlistBot/1.0)", 250 | "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)", 251 | "Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", 252 | "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)", 253 | "Mozilla/5.0 (compatible) Ai2Bot-Dolma (+https://www.allenai.org/crawler)", 254 | "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)", 255 | "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)", 256 | "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)", 257 | "Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)", 258 | "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0)", 259 | "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; AspiegelBot)", 260 | "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", 261 | "Mozilla/5.0 (Linux) (compatible; AlexandriaOrgBot/1.0; +https://www.alexandria.org/bot.html)", 262 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", 263 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)", 264 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36 (compatible; KosmioBot/1.0; +http://kosm.io/bot.html)", 265 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0", 266 | "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0", 267 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0", 268 | "Mozilla/5.0 (Windows NT 5.1; U; Win64; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", 269 | "Mozilla/5.0 (Windows NT 6.1; rv:38.0) Gecko/20100101 Firefox/38.0 (IndeedBot 1.1)", 270 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://url-classification.io/wiki/index.php?title=URL_server_crawler) KStandBot/1.0", 271 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +http://www.komodia.com/newwiki/index.php/URL_server_crawler) KomodiaBot/1.0", 272 | "Mozilla/5.0 (Windows NT 6.1; Win64; x64; +https://www.ubtsupport.com/legal/Streamline3Bot.php) Streamline3Bot/1.0", 273 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6 - James BOT - WebCrawler http://cognitiveseo.com/bot.html", 274 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)", 275 | "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)", 276 | "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11", 277 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/87.0.4280.88 YextBot/Java Safari/537.36", 278 | "Mozilla/5.0 (X11; U; Linux Core i7-4980HQ; de; rv:32.0; compatible; JobboerseBot; http://www.jobboerse.com/bot.htm) Gecko/20100101 Firefox/38.0", 279 | "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0 / GnowitNewsbot / Contact information at http://www.gnowit.com", 280 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", 281 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Linespider/1.1; +https://lin.ee/4dwXkTH) Chrome/W.X.Y.Z Safari/537.36", 282 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)", 283 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", 284 | "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot", 285 | "Mozilla/5.0+(compatible;+PiplBot;+http://www.pipl.com/bot/)", 286 | "Neticle Crawler v1.0 ( https://neticle.com/bot/en/ )", 287 | "NINJA bot", 288 | "NutchCVS/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)", 289 | "OdklBot/1.0 (share@odnoklassniki.ru)", 290 | "OutclicksBot/2 +https://www.outclicks.net/agent/gIYbZ38dfAuhZkrFVl7sJBFOUhOVct6J1SvxgmBZgCe", 291 | "OutclicksBot/2 +https://www.outclicks.net/agent/p2i4sNUh7eylJF1S6SGgRs5mP40ExlYvsr9GBxVQG6h", 292 | "OutclicksBot/2 +https://www.outclicks.net/agent/PryJzTl8POCRHfvEUlRN5FKtZoWDQOBEvFJ2wh6KH5J", 293 | "OutclicksBot/2 +https://www.outclicks.net/agent/VjzDygCuk4ubNmg40ZMbFqT0sIh7UfOKk8s8ZMiupUR", 294 | "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)", 295 | "Pinterest/0.2 (+http://www.pinterest.com/bot.html)", 296 | "PiplBot (+http://www.pipl.com/bot/)", 297 | "psbot-image (+http://www.picsearch.com/bot.html)", 298 | "psbot-page (+http://www.picsearch.com/bot.html)", 299 | "psbot/0.1 (+http://www.picsearch.com/bot.html)", 300 | "RSSingBot (http://www.rssing.com)", 301 | "RyteBot/1.0.0 (+https://bot.ryte.com/)", 302 | "SBL-BOT (http://sbl.net)", 303 | "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2", 304 | "SemanticScholarBot/1.0 (+http://s2.allenai.org/bot.html)", 305 | "SEMrushBot", 306 | "SenutoBot/1.0 (compatible; SenutoBot/1.0; +https://www.senuto.com/)", 307 | "SeobilityBot (SEO Tool; https://www.seobility.net/sites/bot.html)", 308 | "SEOlizer/1.1 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13 (+https://www.seolizer.de/bot.html)", 309 | "SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)", 310 | "serpstatbot/1.0 (advanced backlink tracking bot; curl/7.58.0; http://serpstatbot.com/; abuse@serpstatbot.com)", 311 | "serpstatbot/1.0 (advanced backlink tracking bot; http://serpstatbot.com/; abuse@serpstatbot.com)", 312 | "Slackbot 1.0 (+https://api.slack.com/robots)", 313 | "Slackbot-LinkExpanding (+https://api.slack.com/robots)", 314 | "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)", 315 | "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots", 316 | "SummalyBot/5.1.0", 317 | "Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1162088860", 318 | "Synapse (bot; +https://github.com/matrix-org/synapse)", 319 | "TangibleeBot/1.0.0.0 (http://tangiblee.com/bot)", 320 | "TelegramBot (like TwitterBot)", 321 | "TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)", 322 | "Twitterbot/0.1", 323 | "Twitterbot/1.0", 324 | "webzio (+https://webz.io/bot.html)", 325 | "webzio-extended (+https://webz.io/bot.html)", 326 | "WeSEE:Search/0.1 (Alpha, http://www.wesee.com/en/support/bot/)", 327 | "WhatsApp", 328 | "WhatsApp/0.3.4479 N", 329 | "WhatsApp/0.3.4679 N", 330 | "WhatsApp/0.3.4941 N", 331 | "WhatsApp/2.12.15/i", 332 | "WhatsApp/2.12.16/i", 333 | "WhatsApp/2.12.17/i", 334 | "WhatsApp/2.12.449 A", 335 | "WhatsApp/2.12.453 A", 336 | "WhatsApp/2.12.510 A", 337 | "WhatsApp/2.12.540 A", 338 | "WhatsApp/2.12.548 A", 339 | "WhatsApp/2.12.555 A", 340 | "WhatsApp/2.12.556 A", 341 | "WhatsApp/2.16.1/i", 342 | "WhatsApp/2.16.13 A", 343 | "WhatsApp/2.16.2/i", 344 | "WhatsApp/2.16.42 A", 345 | "WhatsApp/2.16.57 A", 346 | "WhatsApp/2.19.175 A", 347 | "WhatsApp/2.19.244 A", 348 | "WhatsApp/2.19.258 A", 349 | "WhatsApp/2.19.308 A", 350 | "WhatsApp/2.19.330 A", 351 | "WhatsApp/2.19.92 i", 352 | "Wotbox/2.01 (+http://www.wotbox.com/bot/)", 353 | "yacybot (-global; amd64 FreeBSD 9.2-RELEASE-p10; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 354 | "yacybot (-global; amd64 Linux 2.6.32-042stab111.11; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 355 | "yacybot (-global; amd64 Linux 2.6.32-042stab116.1; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 356 | "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 357 | "yacybot (-global; amd64 Linux 3.10.0-229.4.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", 358 | "yacybot (-global; amd64 Linux 3.13.0-61-generic; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 359 | "yacybot (-global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.8.0_111; Europe/de) http://yacy.net/bot.html", 360 | "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 361 | "yacybot (-global; amd64 Linux 3.19.0-15-generic; java 1.8.0_45-internal; Europe/de) http://yacy.net/bot.html", 362 | "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 363 | "yacybot (-global; amd64 Linux 3.2.0-4-amd64; java 1.7.0_67; Europe/en) http://yacy.net/bot.html", 364 | "yacybot (-global; amd64 Linux 4.4.0-57-generic; java 9-internal; Europe/en) http://yacy.net/bot.html", 365 | "yacybot (-global; amd64 Linux 5.2.11-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", 366 | "yacybot (-global; amd64 Linux 5.2.8-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", 367 | "yacybot (-global; amd64 Linux 5.2.9-Jinsol; java 12.0.2; Europe/en) http://yacy.net/bot.html", 368 | "yacybot (-global; amd64 Windows 8 6.2; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", 369 | "yacybot (-global; amd64 Windows 8.1 6.3; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", 370 | "yacybot (/global; amd64 FreeBSD 10.3-RELEASE-p7; java 1.7.0_95; GMT/en) http://yacy.net/bot.html", 371 | "yacybot (/global; amd64 FreeBSD 10.3-RELEASE; java 1.8.0_77; GMT/en) http://yacy.net/bot.html", 372 | "yacybot (/global; amd64 Linux 2.6.32-042stab093.4; java 1.7.0_65; Etc/en) http://yacy.net/bot.html", 373 | "yacybot (/global; amd64 Linux 2.6.32-042stab094.8; java 1.7.0_79; America/en) http://yacy.net/bot.html", 374 | "yacybot (/global; amd64 Linux 2.6.32-042stab108.8; java 1.7.0_91; America/en) http://yacy.net/bot.html", 375 | "yacybot (/global; amd64 Linux 2.6.32-573.3.1.el6.x86_64; java 1.7.0_85; Europe/en) http://yacy.net/bot.html", 376 | "yacybot (/global; amd64 Linux 3.10.0-229.7.2.el7.x86_64; java 1.8.0_45; Europe/en) http://yacy.net/bot.html", 377 | "yacybot (/global; amd64 Linux 3.10.0-327.22.2.el7.x86_64; java 1.7.0_101; Etc/en) http://yacy.net/bot.html", 378 | "yacybot (/global; amd64 Linux 3.11.10-21-desktop; java 1.7.0_51; America/en) http://yacy.net/bot.html", 379 | "yacybot (/global; amd64 Linux 3.12.1; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 380 | "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", 381 | "yacybot (/global; amd64 Linux 3.13.0-042stab093.4; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 382 | "yacybot (/global; amd64 Linux 3.13.0-45-generic; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 383 | "yacybot (/global; amd64 Linux 3.13.0-74-generic; java 1.7.0_91; Europe/en) http://yacy.net/bot.html", 384 | "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/de) http://yacy.net/bot.html", 385 | "yacybot (/global; amd64 Linux 3.13.0-83-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", 386 | "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", 387 | "yacybot (/global; amd64 Linux 3.13.0-85-generic; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", 388 | "yacybot (/global; amd64 Linux 3.13.0-88-generic; java 1.7.0_101; Europe/en) http://yacy.net/bot.html", 389 | "yacybot (/global; amd64 Linux 3.14-0.bpo.1-amd64; java 1.7.0_55; Europe/de) http://yacy.net/bot.html", 390 | "yacybot (/global; amd64 Linux 3.14.32-xxxx-grs-ipv6-64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 391 | "yacybot (/global; amd64 Linux 3.16-0.bpo.2-amd64; java 1.7.0_65; Europe/en) http://yacy.net/bot.html", 392 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_111; Europe/de) http://yacy.net/bot.html", 393 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; America/en) http://yacy.net/bot.html", 394 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_75; Europe/en) http://yacy.net/bot.html", 395 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/de) http://yacy.net/bot.html", 396 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_79; Europe/en) http://yacy.net/bot.html", 397 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_91; Europe/de) http://yacy.net/bot.html", 398 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_95; Europe/en) http://yacy.net/bot.html", 399 | "yacybot (/global; amd64 Linux 3.16.0-4-amd64; java 1.8.0_111; Europe/en) http://yacy.net/bot.html", 400 | "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)", 401 | "ZoomBot (Linkbot 1.0 http://suite.seozoom.it/bot.html)", 402 | "ZoominfoBot (zoominfobot at zoominfo dot com)" 403 | ] --------------------------------------------------------------------------------