├── test ├── resources │ ├── empty.txt │ └── filter.txt ├── integration │ ├── dnscheck.test.js │ ├── filelinter.test.js │ ├── urlfilter.test.js │ └── linter.test.js └── mocked │ ├── mockresponse.js │ ├── filelinter.test.js │ ├── urlfilter.test.js │ ├── dnscheck.test.js │ └── linter.test.js ├── .eslintignore ├── jest.config.js ├── .gitignore ├── .husky └── pre-commit ├── bamboo-specs ├── bamboo.yaml ├── permissions.yaml ├── deploy.yaml ├── increment.yaml ├── test.yaml └── build.yaml ├── tools └── build-txt.js ├── .github └── workflows │ ├── test.yml │ └── lint.yml ├── .eslintrc.js ├── LICENSE ├── package.json ├── src ├── utils.js ├── dnscheck.js ├── urlfilter.js ├── fetchdomains.js ├── cli.js ├── filelinter.js └── linter.js ├── CHANGELOG.md └── README.md /test/resources/empty.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | silent: true, 4 | }; 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .idea 3 | .DS_Store 4 | .eslintcache 5 | dist 6 | node_modules/ 7 | 8 | .npmrc 9 | *.tgz 10 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | echo "Running pre-commit hook, make sure that pnpm is installed" 2 | 3 | pnpm run lint 4 | 5 | pnpm run test 6 | -------------------------------------------------------------------------------- /bamboo-specs/bamboo.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | !include 'build.yaml' 3 | 4 | --- 5 | !include 'deploy.yaml' 6 | 7 | --- 8 | !include 'increment.yaml' 9 | 10 | --- 11 | !include 'permissions.yaml' 12 | 13 | --- 14 | !include 'test.yaml' 15 | -------------------------------------------------------------------------------- /test/resources/filter.txt: -------------------------------------------------------------------------------- 1 | ! Use this file when testing the CLI interface. 2 | ||example.org^$third-party 3 | ||example.org^$domain=example.notexisting 4 | ||example.org^$domain=~example.notexisting 5 | example.notexisting##banner 6 | ||anotherdeaddomain.examplee^ 7 | 8 | ! Проверяем, что с кодировкой все останется в порядке после перезаписи. 9 | -------------------------------------------------------------------------------- /bamboo-specs/permissions.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | deployment: 4 | name: dead-domains-linter - deploy 5 | deployment-permissions: 6 | - groups: 7 | - extensions-developers 8 | permissions: 9 | - view 10 | environment-permissions: 11 | - npmjs: 12 | - groups: 13 | - extensions-developers 14 | permissions: 15 | - view 16 | - deploy 17 | -------------------------------------------------------------------------------- /tools/build-txt.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const { version } = require('../package.json'); 4 | 5 | const PATH = '../dist'; 6 | const FILENAME = 'build.txt'; 7 | 8 | const buildTxt = () => { 9 | const dir = path.resolve(__dirname, PATH); 10 | 11 | if (!fs.existsSync(dir)) { 12 | fs.mkdirSync(dir); 13 | } 14 | 15 | const content = `version=${version}`; 16 | fs.writeFileSync(path.resolve(__dirname, PATH, FILENAME), content); 17 | }; 18 | 19 | buildTxt(); 20 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | node-version: [18.x, 22.x] 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Use Node.js 18 | uses: actions/setup-node@v4 19 | with: 20 | node-version: ${{ matrix.node-version }} 21 | - name: Install pnpm 22 | uses: pnpm/action-setup@v4 23 | with: 24 | version: 10.12.4 25 | - run: pnpm install --frozen-lockfile 26 | - run: pnpm test 27 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | node-version: [18.x, 22.x] 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Use Node.js 18 | uses: actions/setup-node@v4 19 | with: 20 | node-version: ${{ matrix.node-version }} 21 | - name: Install pnpm 22 | uses: pnpm/action-setup@v4 23 | with: 24 | version: 10.12.4 25 | - run: pnpm install --frozen-lockfile 26 | - run: pnpm run lint 27 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | 'airbnb-base', 4 | 'plugin:jsdoc/recommended', 5 | ], 6 | parser: '@babel/eslint-parser', 7 | parserOptions: { 8 | requireConfigFile: false, 9 | }, 10 | env: { 11 | browser: false, 12 | node: true, 13 | jest: true, 14 | }, 15 | rules: { 16 | 'max-len': [ 17 | 'error', 18 | { 19 | code: 120, 20 | ignoreUrls: true, 21 | }, 22 | ], 23 | indent: ['error', 4, { SwitchCase: 1 }], 24 | 'import/prefer-default-export': 'off', 25 | 'arrow-body-style': 'off', 26 | 'import/no-extraneous-dependencies': 'off', 27 | 'jsdoc/tag-lines': [ 28 | 'warn', 29 | 'any', 30 | { 31 | startLines: 1, 32 | }, 33 | ], 34 | }, 35 | }; 36 | -------------------------------------------------------------------------------- /test/integration/dnscheck.test.js: -------------------------------------------------------------------------------- 1 | const dnscheck = require('../../src/dnscheck'); 2 | 3 | describe('dnscheck', () => { 4 | it('check a known existing domain', async () => { 5 | const result = await dnscheck.checkDomain('example.org'); 6 | 7 | expect(result).toBe(true); 8 | }); 9 | 10 | it('check a known non-existing domain', async () => { 11 | const result = await dnscheck.checkDomain('example.nonexistingdomain'); 12 | 13 | expect(result).toBe(false); 14 | }); 15 | 16 | it('check a domain that only has a www. record', async () => { 17 | const noWwwExists = await dnscheck.domainExists('city.kawasaki.jp'); 18 | // Make sure that there's no A record for the domain. 19 | // If it appears, we'll need to change the domain for this test. 20 | expect(noWwwExists).toBe(false); 21 | 22 | const result = await dnscheck.checkDomain('city.kawasaki.jp'); 23 | 24 | expect(result).toBe(true); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 AdGuard Software Ltd. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /bamboo-specs/deploy.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | deployment: 4 | name: dead-domains-linter - deploy 5 | source-plan: AJL-DDLBUILD 6 | release-naming: ${bamboo.inject.version} 7 | environments: 8 | - npmjs 9 | 10 | npmjs: 11 | docker: 12 | image: adguard/node-ssh:18.13--3 13 | triggers: [] 14 | tasks: 15 | - checkout: 16 | force-clean-build: true 17 | - artifact-download: 18 | artifacts: 19 | - name: dead-domains-linter.tgz 20 | - script: 21 | interpreter: SHELL 22 | scripts: 23 | - |- 24 | set -e 25 | set -x 26 | 27 | # Fix mixed logs 28 | exec 2>&1 29 | 30 | ls -alt 31 | 32 | export NPM_TOKEN=${bamboo.npmSecretToken} 33 | echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc 34 | npm publish dead-domains-linter.tgz --access public 35 | requirements: 36 | - adg-docker: "true" 37 | notifications: 38 | - events: 39 | - deployment-started-and-finished 40 | recipients: 41 | - webhook: 42 | name: Deploy webhook 43 | url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo 44 | -------------------------------------------------------------------------------- /test/integration/filelinter.test.js: -------------------------------------------------------------------------------- 1 | const fileLinter = require('../../src/filelinter'); 2 | 3 | describe('File linter', () => { 4 | it('test a simple automatic run', async () => { 5 | const fileResult = await fileLinter.lintFile('test/resources/filter.txt', { 6 | auto: true, 7 | ignoreDomains: new Set(), 8 | }); 9 | 10 | expect(fileResult).toBeDefined(); 11 | expect(fileResult.listAst).toBeDefined(); 12 | expect(fileResult.results).toBeDefined(); 13 | expect(fileResult.results).toHaveLength(4); 14 | }); 15 | 16 | it('should ignore domains in ignoreDomains set with real API', async () => { 17 | const fileResult = await fileLinter.lintFile('test/resources/filter.txt', { 18 | auto: true, 19 | ignoreDomains: new Set(['anotherdeaddomain.examplee']), 20 | }); 21 | 22 | expect(fileResult).toBeDefined(); 23 | expect(fileResult.results).toHaveLength(3); 24 | }, 30000); 25 | 26 | it('should handle empty file', async () => { 27 | const fileResult = await fileLinter.lintFile('test/resources/empty.txt', { 28 | auto: true, 29 | ignoreDomains: new Set(), 30 | }); 31 | 32 | expect(fileResult).toBeNull(); 33 | }, 10000); 34 | }); 35 | -------------------------------------------------------------------------------- /bamboo-specs/increment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | plan: 4 | project-key: AJL 5 | key: DDLINCR 6 | name: dead-domains-linter - increment 7 | variables: 8 | dockerContainer: adguard/node-ssh:18.13--3 9 | 10 | stages: 11 | - Increment: 12 | manual: false 13 | final: false 14 | jobs: 15 | - Increment 16 | 17 | Increment: 18 | key: INCR 19 | docker: 20 | image: "${bamboo.dockerContainer}" 21 | volumes: 22 | ${system.PNPM_DIR}: "${bamboo.cachePnpm}" 23 | other: 24 | clean-working-dir: true 25 | tasks: 26 | - checkout: 27 | force-clean-build: true 28 | - script: 29 | interpreter: SHELL 30 | scripts: 31 | - |- 32 | set -e 33 | set -x 34 | 35 | # Fix mixed logs 36 | exec 2>&1 37 | 38 | npm run increment 39 | - any-task: 40 | plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.commit 41 | configuration: 42 | commitMessage: "skipci: Automatic increment build number" 43 | selectedRepository: defaultRepository 44 | requirements: 45 | - adg-docker: "true" 46 | 47 | branches: 48 | create: manually 49 | delete: never 50 | link-to-jira: true 51 | 52 | labels: [] 53 | other: 54 | concurrent-build-plugin: system-default 55 | -------------------------------------------------------------------------------- /test/integration/urlfilter.test.js: -------------------------------------------------------------------------------- 1 | const urlfilter = require('../../src/urlfilter'); 2 | 3 | describe('urlfilter', () => { 4 | it('check a domain that we know does exist', async () => { 5 | const result = await urlfilter.findDeadDomains(['example.org']); 6 | 7 | expect(result).toEqual([]); 8 | }); 9 | 10 | it('check a domain that we know does NOT exist', async () => { 11 | const result = await urlfilter.findDeadDomains(['example.atatatata.baababbaba']); 12 | 13 | expect(result).toEqual(['example.atatatata.baababbaba']); 14 | }); 15 | 16 | it('check two domains, one exists, one not', async () => { 17 | const result = await urlfilter.findDeadDomains(['example.org', 'example.atatatata.baababbaba']); 18 | 19 | expect(result).toEqual(['example.atatatata.baababbaba']); 20 | }); 21 | 22 | it('check an fqdn domain name', async () => { 23 | const result = await urlfilter.findDeadDomains(['example.notexisting.']); 24 | 25 | expect(result).toEqual(['example.notexisting.']); 26 | }); 27 | 28 | it('checks lots of domains using two chunks', async () => { 29 | const domains = []; 30 | for (let i = 0; i < 10; i += 1) { 31 | domains.push(`example${i}.notexistingtld`); 32 | } 33 | 34 | const result = await urlfilter.findDeadDomains(domains, 5); 35 | 36 | expect(result).toEqual(domains); 37 | }, 30000); 38 | }); 39 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@adguard/dead-domains-linter", 3 | "version": "1.0.33", 4 | "description": "Simple tool to check adblock filtering rules for dead domains.", 5 | "keywords": [ 6 | "adblock", 7 | "adguard", 8 | "filter", 9 | "linter", 10 | "list", 11 | "ublock" 12 | ], 13 | "author": "AdGuard Software Ltd.", 14 | "license": "MIT", 15 | "repository": { 16 | "type": "git", 17 | "url": "git+https://github.com/AdguardTeam/DeadDomainsLinter.git" 18 | }, 19 | "bugs": { 20 | "url": "https://github.com/AdguardTeam/DeadDomainsLinter/issues" 21 | }, 22 | "homepage": "https://github.com/AdguardTeam/DeadDomainsLinter#readme", 23 | "files": [ 24 | "src" 25 | ], 26 | "bin": { 27 | "dead-domains-linter": "src/cli.js" 28 | }, 29 | "engines": { 30 | "node": ">=18" 31 | }, 32 | "dependencies": { 33 | "@adguard/agtree": "^1.1.7", 34 | "consola": "3.2.3", 35 | "glob": "^10.3.10", 36 | "node-fetch": "^2.7.0", 37 | "tldts": "^6.1.4", 38 | "yargs": "^17.7.2" 39 | }, 40 | "devDependencies": { 41 | "@babel/core": "^7.23.9", 42 | "@babel/eslint-parser": "^7.23.9", 43 | "eslint": "^8.56.0", 44 | "eslint-config-airbnb-base": "^15.0.0", 45 | "eslint-plugin-import": "^2.29.1", 46 | "eslint-plugin-jsdoc": "^48.0.4", 47 | "husky": "^9.0.6", 48 | "jest": "^29.7.0", 49 | "jsdoc": "^4.0.2" 50 | }, 51 | "scripts": { 52 | "lint": "eslint . --cache", 53 | "test": "jest --runInBand --detectOpenHandles --verbose .", 54 | "increment": "npm version patch --no-git-tag-version", 55 | "prepare": "husky", 56 | "build-txt": "node tools/build-txt.js" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | const tldts = require('tldts'); 2 | 3 | /** 4 | * Helper function that takes an array and returns a new one without any 5 | * duplicate items. 6 | * 7 | * @param {Array} arr - THe array to check for duplicates. 8 | * @returns {Array} Returns a new array without duplicates. 9 | */ 10 | function unique(arr) { 11 | return [...new Set([].concat(...arr))]; 12 | } 13 | 14 | // A list of TLD that we ignore and don't check for existence for technical 15 | // reasons. 16 | const ALLOW_TLD = new Set([ 17 | 'onion', // Tor 18 | 'lib', 'coin', 'bazar', 'emc', // EmerCoin 19 | 'bit', // Namecoin 20 | 'sats', 'ord', 'gm', // SATS domains 21 | ]); 22 | 23 | /** 24 | * Checks if the given domain is valid for our dead domains check. 25 | * 26 | * @param {string} domain - The domain name to check. 27 | * @returns {boolean} Returns true if the domain is valid, false otherwise. 28 | */ 29 | function validDomain(domain) { 30 | const result = tldts.parse(domain); 31 | 32 | if (!result?.domain) { 33 | return false; 34 | } 35 | 36 | if (result.isIp) { 37 | // IP addresses cannot be verified too so just ignore them too. 38 | return false; 39 | } 40 | 41 | if (ALLOW_TLD.has(result.publicSuffix)) { 42 | // Do not check TLDs that are in use, but we cannot check them for 43 | // existence. 44 | return false; 45 | } 46 | 47 | if (result.domainWithoutSuffix === '') { 48 | // Ignore top-level domains to avoid false positives on things like: 49 | // https://github.com/AdguardTeam/DeadDomainsLinter/issues/6 50 | return false; 51 | } 52 | 53 | return true; 54 | } 55 | 56 | module.exports = { 57 | unique, 58 | validDomain, 59 | }; 60 | -------------------------------------------------------------------------------- /test/mocked/mockresponse.js: -------------------------------------------------------------------------------- 1 | const createSuccessResponse = (deadDomains, activeDomains = []) => { 2 | const responseData = {}; 3 | 4 | deadDomains.forEach((domain) => { 5 | responseData[domain] = { 6 | info: { 7 | domain_name: domain, 8 | registered_domain: domain, 9 | registered_domain_used_last_24_hours: false, 10 | used_last_24_hours: false, 11 | }, 12 | matches: [], 13 | }; 14 | }); 15 | 16 | activeDomains.forEach((domain) => { 17 | responseData[domain] = { 18 | info: { 19 | domain_name: domain, 20 | registered_domain: domain, 21 | registered_domain_used_last_24_hours: true, 22 | used_last_24_hours: true, 23 | }, 24 | matches: [], 25 | }; 26 | }); 27 | 28 | return { 29 | status: 200, 30 | ok: true, 31 | headers: { get: () => 'application/json' }, 32 | json: jest.fn().mockResolvedValue(responseData), 33 | }; 34 | }; 35 | 36 | const createRateLimitedResponse = (retryAfterValue) => ({ 37 | status: 429, 38 | ok: false, 39 | headers: { 40 | get: jest.fn((headerName) => { 41 | const headers = { 42 | 'retry-after': retryAfterValue, 43 | 'content-type': 'application/json', 44 | }; 45 | return headers[headerName.toLowerCase()]; 46 | }), 47 | }, 48 | json: jest.fn().mockResolvedValue({ 49 | error: 'Too many requests', 50 | message: 'Rate limit exceeded', 51 | }), 52 | }); 53 | 54 | module.exports = { 55 | createSuccessResponse, 56 | createRateLimitedResponse, 57 | }; 58 | -------------------------------------------------------------------------------- /bamboo-specs/test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | plan: 4 | project-key: AJL 5 | key: DDLTEST 6 | name: dead-domains-linter - tests 7 | variables: 8 | dockerContainer: adguard/node-ssh:18.13--3 9 | 10 | stages: 11 | - Test: 12 | manual: false 13 | final: false 14 | jobs: 15 | - Test 16 | 17 | Test: 18 | key: TEST 19 | docker: 20 | image: "${bamboo.dockerContainer}" 21 | volumes: 22 | ${system.PNPM_DIR}: "${bamboo.cachePnpm}" 23 | tasks: 24 | - checkout: 25 | force-clean-build: true 26 | - script: 27 | interpreter: SHELL 28 | scripts: 29 | - |- 30 | set -e 31 | set -x 32 | 33 | # Fix mixed logs 34 | exec 2>&1 35 | 36 | pnpm install --ignore-scripts --no-optional 37 | 38 | pnpm run lint 39 | pnpm run test 40 | final-tasks: 41 | - script: 42 | interpreter: SHELL 43 | scripts: 44 | - |- 45 | set -x 46 | set -e 47 | 48 | # Fix mixed logs 49 | exec 2>&1 50 | 51 | ls -la 52 | 53 | echo "Size before cleanup:" && du -h | tail -n 1 54 | rm -rf node_modules 55 | echo "Size after cleanup:" && du -h | tail -n 1 56 | requirements: 57 | - adg-docker: "true" 58 | 59 | branches: 60 | create: for-pull-request 61 | delete: 62 | after-deleted-days: "1" 63 | after-inactive-days: "5" 64 | link-to-jira: true 65 | 66 | notifications: 67 | - events: 68 | - plan-status-changed 69 | recipients: 70 | - webhook: 71 | name: Build webhook 72 | url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo 73 | 74 | labels: [] 75 | other: 76 | concurrent-build-plugin: system-default 77 | -------------------------------------------------------------------------------- /src/dnscheck.js: -------------------------------------------------------------------------------- 1 | const dns = require('dns'); 2 | const { promisify } = require('util'); 3 | 4 | const resolver = new dns.Resolver(); 5 | 6 | // Note, that we don't use AdGuard DNS servers here in order to not add checked 7 | // domains to the next domains snapshot. 8 | // 9 | // TODO(ameshkov): Consider making the DNS server configurable. 10 | resolver.setServers([ 11 | '8.8.8.8', 12 | ]); 13 | 14 | const resolveAsync = promisify(resolver.resolve).bind(resolver); 15 | 16 | /** 17 | * Checks if the domain has an A record. 18 | * 19 | * @param {string} domain - Domain name to check with a DNS query. 20 | * @returns {Promise} Returns true if the domain has an A record. 21 | */ 22 | async function domainExists(domain) { 23 | try { 24 | const addresses = await resolveAsync(domain, 'A'); 25 | 26 | return addresses.length > 0; 27 | } catch { 28 | return false; 29 | } 30 | } 31 | 32 | /** 33 | * Checks if the domain name exists with one or more DNS queries. 34 | * 35 | * @param {string} domain - Domain name to check. 36 | * @returns {Promise} Returns true if the domain is considered alive. 37 | */ 38 | async function checkDomain(domain) { 39 | let exists = await domainExists(domain); 40 | 41 | if (exists) { 42 | return true; 43 | } 44 | 45 | if (domain.startsWith('www.')) { 46 | // If this is a www. domain name, there's no need to doublecheck it. 47 | return false; 48 | } 49 | 50 | // Double-check a www. version of a domain name. We do this because there 51 | // are some cases when it's necessary: 52 | // https://github.com/AdguardTeam/DeadDomainsLinter/issues/16 53 | exists = domainExists(`www.${domain}`); 54 | 55 | return exists; 56 | } 57 | 58 | module.exports = { 59 | checkDomain, 60 | domainExists, 61 | }; 62 | -------------------------------------------------------------------------------- /src/urlfilter.js: -------------------------------------------------------------------------------- 1 | const punycode = require('node:punycode'); 2 | const { fetchWithRetry, trimFqdn } = require('./fetchdomains'); 3 | 4 | const CHUNK_SIZE = 25; 5 | 6 | /** 7 | * This function looks for dead domains among the specified ones. It uses a web 8 | * service to do that. 9 | * 10 | * @param {Array} domains domains to check. 11 | * @param {number} chunkSize configures the size of chunks for checking large 12 | * arrays. 13 | * @returns {Promise>} returns the list of dead domains. 14 | */ 15 | async function findDeadDomains(domains, chunkSize = CHUNK_SIZE) { 16 | const result = []; 17 | 18 | // Split the domains array into chunks 19 | const chunks = []; 20 | for (let i = 0; i < domains.length; i += chunkSize) { 21 | chunks.push(domains.slice(i, i + chunkSize)); 22 | } 23 | 24 | // Compose and send requests for each chunk 25 | // eslint-disable-next-line no-restricted-syntax 26 | for (const chunk of chunks) { 27 | try { 28 | // eslint-disable-next-line no-await-in-loop 29 | const response = await fetchWithRetry(chunk); 30 | // eslint-disable-next-line no-await-in-loop 31 | const data = await response.json(); 32 | 33 | if (data.error) { 34 | throw new Error(data.error); 35 | } 36 | 37 | // Iterate over the domains in the chunk 38 | // eslint-disable-next-line no-restricted-syntax 39 | for (const domain of chunk) { 40 | const domainData = data[punycode.toASCII(trimFqdn(domain))]; 41 | if (domainData && domainData.info.registered_domain_used_last_24_hours === false) { 42 | result.push(domain); 43 | } 44 | } 45 | } catch (ex) { 46 | // Re-throw to add information about the URL. 47 | throw new Error(`Failed to fetch domains ${ex}`); 48 | } 49 | } 50 | 51 | return result; 52 | } 53 | 54 | module.exports = { 55 | findDeadDomains, 56 | }; 57 | -------------------------------------------------------------------------------- /test/mocked/filelinter.test.js: -------------------------------------------------------------------------------- 1 | jest.mock('node-fetch'); 2 | const fetch = require('node-fetch'); 3 | const fileLinter = require('../../src/filelinter'); 4 | const { createSuccessResponse } = require('./mockresponse'); 5 | 6 | describe('File linter with mocked API', () => { 7 | beforeEach(() => { 8 | fetch.mockReset(); 9 | }); 10 | 11 | it('test a simple automatic run with mocked API', async () => { 12 | fetch.mockResolvedValue(createSuccessResponse( 13 | ['example.notexisting', 'anotherdeaddomain.examplee'], 14 | ['example.org'], 15 | )); 16 | 17 | const fileResult = await fileLinter.lintFile('test/resources/filter.txt', { 18 | auto: true, 19 | ignoreDomains: new Set(), 20 | }); 21 | 22 | expect(fileResult).toBeDefined(); 23 | expect(fileResult.listAst).toBeDefined(); 24 | expect(fileResult.results).toBeDefined(); 25 | 26 | // Should find 4 issues: 27 | // 1. ||example.org^$domain=example.notexisting (dead domain in $domain) 28 | // 2. ||example.org^$domain=~example.notexisting (dead negated domain in $domain) 29 | // 3. example.notexisting##banner (dead domain in cosmetic rule) 30 | // 4. ||anotherdeaddomain.examplee^ (dead domain in network rule) 31 | expect(fileResult.results).toHaveLength(4); 32 | }); 33 | 34 | it('should ignore domains in ignoreDomains set', async () => { 35 | fetch.mockResolvedValue(createSuccessResponse( 36 | ['example.notexisting', 'anotherdeaddomain.examplee'], 37 | ['example.org'], 38 | )); 39 | 40 | const fileResult = await fileLinter.lintFile('test/resources/filter.txt', { 41 | auto: true, 42 | ignoreDomains: new Set(['example.notexisting']), 43 | }); 44 | 45 | expect(fileResult).toBeDefined(); 46 | 47 | // Should only find 1 issue now (the rule with 'anotherdeaddomain.examplee' which is not ignored) 48 | // The rules with example.notexisting should be ignored 49 | expect(fileResult.results).toHaveLength(1); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /test/mocked/urlfilter.test.js: -------------------------------------------------------------------------------- 1 | jest.mock('node-fetch'); 2 | 3 | const fetch = require('node-fetch'); 4 | const urlfilter = require('../../src/urlfilter'); 5 | const { createRateLimitedResponse, createSuccessResponse } = require('./mockresponse'); 6 | 7 | describe('urlfilter tests with mocked api calls', () => { 8 | beforeEach(() => { 9 | fetch.mockReset(); 10 | jest.useFakeTimers(); 11 | }); 12 | 13 | afterEach(() => { 14 | jest.useRealTimers(); 15 | }); 16 | 17 | const testRetryAfter = async (retryAfterValue, domain = 'example.notexisting') => { 18 | fetch.mockResolvedValueOnce(createRateLimitedResponse(retryAfterValue)); 19 | fetch.mockResolvedValueOnce(createSuccessResponse([domain])); 20 | 21 | const promise = urlfilter.findDeadDomains([domain]); 22 | expect(fetch).toHaveBeenCalledTimes(1); 23 | await jest.advanceTimersByTimeAsync(2000); 24 | const result = await promise; 25 | 26 | expect(fetch).toHaveBeenCalledTimes(2); 27 | expect(result).toEqual([domain]); 28 | }; 29 | 30 | it('should handle 429 with retry-after header with seconds', async () => { 31 | await testRetryAfter('2'); 32 | }); 33 | 34 | it('should handle 429 with retry-after header with Date', async () => { 35 | await testRetryAfter(new Date(Date.now() + 2000)); 36 | }); 37 | 38 | it('check a domain that we know does exist', async () => { 39 | fetch.mockResolvedValueOnce(createSuccessResponse([], ['example.org'])); 40 | 41 | const result = await urlfilter.findDeadDomains(['example.org']); 42 | expect(result).toEqual([]); 43 | }); 44 | 45 | it('check a domain that we know does NOT exist', async () => { 46 | fetch.mockResolvedValueOnce(createSuccessResponse(['example.atatatata.baababbaba'])); 47 | 48 | const result = await urlfilter.findDeadDomains(['example.atatatata.baababbaba']); 49 | expect(result).toEqual(['example.atatatata.baababbaba']); 50 | }); 51 | 52 | it('check an fqdn domain name', async () => { 53 | fetch.mockResolvedValueOnce(createSuccessResponse(['example.notexisting'])); 54 | 55 | const result = await urlfilter.findDeadDomains(['example.notexisting.']); 56 | expect(result).toEqual(['example.notexisting.']); 57 | }); 58 | 59 | it('checks lots of domains using two chunks', async () => { 60 | const domains = []; 61 | for (let i = 0; i < 10; i += 1) { 62 | domains.push(`example${i}.notexistingtld`); 63 | } 64 | 65 | // Mock two API calls (for two chunks) 66 | fetch.mockResolvedValueOnce(createSuccessResponse(domains.slice(0, 5))); 67 | fetch.mockResolvedValueOnce(createSuccessResponse(domains.slice(5))); 68 | 69 | const result = await urlfilter.findDeadDomains(domains, 5); 70 | expect(result).toEqual(domains); 71 | }); 72 | }); 73 | -------------------------------------------------------------------------------- /bamboo-specs/build.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | plan: 4 | project-key: AJL 5 | key: DDLBUILD 6 | name: dead-domains-linter - build 7 | variables: 8 | dockerContainer: adguard/node-ssh:18.13--3 9 | 10 | stages: 11 | - Build: 12 | manual: false 13 | final: false 14 | jobs: 15 | - Build 16 | 17 | Build: 18 | key: BUILD 19 | other: 20 | clean-working-dir: true 21 | docker: 22 | image: "${bamboo.dockerContainer}" 23 | volumes: 24 | ${system.PNPM_DIR}: "${bamboo.cachePnpm}" 25 | tasks: 26 | - checkout: 27 | force-clean-build: true 28 | - script: 29 | interpreter: SHELL 30 | scripts: 31 | - |- 32 | set -e 33 | set -x 34 | 35 | # Fix mixed logs 36 | exec 2>&1 37 | 38 | # Install dependencies at the beginning. 39 | pnpm install --ignore-scripts --no-optional 40 | 41 | # Prepare the script that will be injected into the build. 42 | pnpm run build-txt 43 | - inject-variables: 44 | file: dist/build.txt 45 | scope: RESULT 46 | namespace: inject 47 | - script: 48 | interpreter: SHELL 49 | scripts: 50 | - |- 51 | #!/bin/bash 52 | set -e 53 | set -x 54 | 55 | # Fix mixed logs 56 | exec 2>&1 57 | 58 | # Fail if the version was not incremented. 59 | if [ "$(git tag -l "v${bamboo.inject.version}")" ]; then 60 | echo "Build failed!" 61 | echo "Error: Tag v${bamboo.inject.version} already exists. Increment version before build" 62 | exit 1 63 | fi 64 | 65 | npm pack && mv $(ls adguard-dead-domains-linter-*.tgz) dead-domains-linter.tgz 66 | - any-task: 67 | plugin-key: com.atlassian.bamboo.plugins.vcs:task.vcs.tagging 68 | configuration: 69 | selectedRepository: defaultRepository 70 | tagName: v${bamboo.inject.version} 71 | final-tasks: 72 | - script: 73 | interpreter: SHELL 74 | scripts: 75 | - |- 76 | set -x 77 | set -e 78 | 79 | # Fix mixed logs 80 | exec 2>&1 81 | 82 | ls -la 83 | 84 | echo "Size before cleanup:" && du -h | tail -n 1 85 | rm -rf node_modules 86 | echo "Size after cleanup:" && du -h | tail -n 1 87 | artifacts: 88 | - name: dead-domains-linter.tgz 89 | location: ./ 90 | pattern: dead-domains-linter.tgz 91 | shared: true 92 | required: true 93 | requirements: 94 | - adg-docker: "true" 95 | 96 | triggers: [] 97 | 98 | branches: 99 | create: manually 100 | delete: never 101 | link-to-jira: true 102 | 103 | notifications: 104 | - events: 105 | - plan-status-changed 106 | recipients: 107 | - webhook: 108 | name: Build webhook 109 | url: http://prod.jirahub.service.eu.consul/v1/webhook/bamboo 110 | 111 | labels: [] 112 | 113 | other: 114 | concurrent-build-plugin: system-default 115 | -------------------------------------------------------------------------------- /test/mocked/dnscheck.test.js: -------------------------------------------------------------------------------- 1 | jest.mock('dns', () => { 2 | const actualDns = jest.requireActual('dns'); 3 | 4 | const mockResolver = { 5 | resolve: jest.fn(), 6 | setServers: jest.fn(), 7 | }; 8 | 9 | return { 10 | ...actualDns, 11 | Resolver: jest.fn(() => mockResolver), 12 | }; 13 | }); 14 | 15 | const dns = require('dns'); 16 | const dnscheck = require('../../src/dnscheck'); 17 | 18 | describe('dnscheck mocked tests', () => { 19 | let mockResolver; 20 | 21 | beforeEach(() => { 22 | mockResolver = new dns.Resolver(); 23 | jest.clearAllMocks(); 24 | }); 25 | 26 | it('check a known existing domain with mocked DNS', async () => { 27 | // Mock successful DNS resolution 28 | mockResolver.resolve.mockImplementation((domain, rrtype, callback) => { 29 | callback(null, ['93.184.216.34']); 30 | }); 31 | 32 | const result = await dnscheck.checkDomain('example.org'); 33 | expect(result).toBe(true); 34 | expect(mockResolver.resolve).toHaveBeenCalledWith('example.org', 'A', expect.any(Function)); 35 | }); 36 | 37 | it('check a known non-existing domain with mocked DNS', async () => { 38 | // Mock DNS resolution failure 39 | mockResolver.resolve.mockImplementation((domain, rrtype, callback) => { 40 | callback(new Error('ENOTFOUND')); 41 | }); 42 | 43 | const result = await dnscheck.checkDomain('example.nonexistingdomain'); 44 | expect(result).toBe(false); 45 | expect(mockResolver.resolve).toHaveBeenCalledWith('example.nonexistingdomain', 'A', expect.any(Function)); 46 | }); 47 | 48 | it('check a domain that only has a www. record with mocked DNS', async () => { 49 | // Mock: base domain fails, www version succeeds 50 | mockResolver.resolve.mockImplementation((domain, rrtype, callback) => { 51 | if (domain === 'city.kawasaki.jp') { 52 | callback(new Error('ENOTFOUND')); 53 | } else if (domain === 'www.city.kawasaki.jp') { 54 | callback(null, ['192.0.2.1']); 55 | } else { 56 | callback(new Error('Unexpected domain')); 57 | } 58 | }); 59 | 60 | const noWwwExists = await dnscheck.domainExists('city.kawasaki.jp'); 61 | expect(noWwwExists).toBe(false); 62 | 63 | const result = await dnscheck.checkDomain('city.kawasaki.jp'); 64 | expect(result).toBe(true); 65 | 66 | expect(mockResolver.resolve).toHaveBeenCalledWith('city.kawasaki.jp', 'A', expect.any(Function)); 67 | expect(mockResolver.resolve).toHaveBeenCalledWith('www.city.kawasaki.jp', 'A', expect.any(Function)); 68 | }); 69 | 70 | it('should not check www version for www domains', async () => { 71 | // Mock DNS resolution failure 72 | mockResolver.resolve.mockImplementation((domain, rrtype, callback) => { 73 | callback(new Error('ENOTFOUND')); 74 | }); 75 | 76 | const result = await dnscheck.checkDomain('www.example.nonexisting'); 77 | expect(result).toBe(false); 78 | 79 | // Should only be called once for the www domain 80 | expect(mockResolver.resolve).toHaveBeenCalledTimes(1); 81 | expect(mockResolver.resolve).toHaveBeenCalledWith('www.example.nonexisting', 'A', expect.any(Function)); 82 | }); 83 | }); 84 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Dead Domains Linter Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog], and this project adheres to [Semantic Versioning]. 6 | 7 | [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/ 8 | [Semantic Versioning]: https://semver.org/spec/v2.0.0.html 9 | 10 | ## [1.0.33] - 2025-09-01 11 | 12 | ### Changed 13 | 14 | - Linter now respects retry-after header for requests to adtidy API [#43]. 15 | - Non ascii domains are now converted to punycode and checked [#35]. 16 | 17 | [#35]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/35 18 | [#43]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/43 19 | 20 | ## [1.0.28] - 2025-07-09 21 | 22 | ### Added 23 | 24 | - Option to add a file with domains to ignore when running [#33]. 25 | 26 | [1.0.28]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.22...v1.0.28 27 | [#33]: https://github.com/AdguardTeam/DeadDomainsLinter/pull/33 28 | 29 | ## [1.0.22] - 2024-12-26 30 | 31 | ### Fixed 32 | 33 | - `consola.info is not a function` error [#32]. 34 | 35 | [1.0.22]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.19...v1.0.22 36 | [#32]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/32 37 | 38 | ## [1.0.19] - 2024-02-08 39 | 40 | ### Changed 41 | 42 | - Requests to the urlfilter service so that only domain info was checked 43 | without testing which lists match the domain, it should speed up the process. 44 | 45 | [1.0.19]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.18...v1.0.19 46 | 47 | ## [1.0.18] - 2024-02-01 48 | 49 | ### Fixed 50 | 51 | - Issue with importing a list of domains [#23]. 52 | 53 | [1.0.18]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.16...v1.0.18 54 | [#23]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/23 55 | 56 | ## [1.0.16] - 2024-01-31 57 | 58 | ### Added 59 | 60 | - Option to use a pre-defined list of dead domains from a file [#20]. 61 | - Option to export the list of dead domains to a file [#8]. 62 | 63 | ### Fixed 64 | 65 | - Issue with keeping negated domains in a network rule [#19]. 66 | 67 | [1.0.16]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.13...v1.0.16 68 | [#8]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/8 69 | [#19]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/19 70 | [#20]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/20 71 | 72 | ## [1.0.13] - 2024-01-31 73 | 74 | ### Fixed 75 | 76 | - Issue with some rarely visited domains marked as dead [#16]. 77 | - Issue with rules that target IP ranges [#17]. 78 | - Issue with checking FQDN in rules [#18]. 79 | 80 | [1.0.13]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.8...v1.0.13 81 | [#16]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/16 82 | [#17]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/17 83 | [#18]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/18 84 | 85 | ## [1.0.8] - 2024-01-31 86 | 87 | ### Fixed 88 | 89 | - Issue with extracting domains from some URL patterns [#11]. 90 | - Issue with testing custom TLD [#13]. 91 | 92 | [1.0.8]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.6...v1.0.8 93 | [#11]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/11 94 | [#13]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/13 95 | 96 | ## [1.0.6] - 2024-01-29 97 | 98 | ### Added 99 | 100 | - Option to comment the rule out instead of removing it [#4]. 101 | 102 | ### Changed 103 | 104 | - Speed up the build by running several checks in parallel [#2]. 105 | 106 | ### Fixed 107 | 108 | - Issue with incorrect line numbers [#1]. 109 | - Issue with counting IPv4 addresses as dead domains [#5]. 110 | - Issue with suggesting removing TLDs and extension IDs [#6]. 111 | 112 | [1.0.6]: https://github.com/AdguardTeam/DeadDomainsLinter/compare/v1.0.4...v1.0.6 113 | [#1]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/1 114 | [#2]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/2 115 | [#4]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/4 116 | [#5]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/5 117 | [#6]: https://github.com/AdguardTeam/DeadDomainsLinter/issues/6 118 | -------------------------------------------------------------------------------- /src/fetchdomains.js: -------------------------------------------------------------------------------- 1 | const dns = require('dns'); 2 | const consola = require('consola'); 3 | const https = require('https'); 4 | const fetch = require('node-fetch'); 5 | const punycode = require('node:punycode'); 6 | 7 | /** 8 | * 503 - Service Unavailable 9 | * 429 - Too Many Requests 10 | */ 11 | const CODES_TO_RETRY = new Set([503, 429]); 12 | 13 | const DECIMAL_BASE = 10; 14 | const ONE_SECOND_MS = 1000; 15 | 16 | /** 17 | * 2 retries for the first request and request after receiving retry-after header. 18 | */ 19 | const DEFAULT_MAX_ATTEMPTS = 2; 20 | const URLFILTER_URL = 'https://urlfilter.adtidy.org/v2/checkDomains'; 21 | 22 | /** 23 | * Default agent for requests to adtidy API 24 | */ 25 | const httpAgent = new https.Agent({ 26 | keepAlive: true, 27 | // eslint-disable-next-line no-use-before-define 28 | lookup: dnsLookup, 29 | }); 30 | 31 | /** 32 | * When using native node fetch it is easy to run into ENOTFOUND errors when 33 | * there are many parallel requests. In order to avoid that, we use node-fetch 34 | * with a custom DNS lookup function that caches resolution result permanently. 35 | * In addition to that, we use a semaphore-like approach to forbid parallel 36 | * DNS queries. 37 | */ 38 | const dnsProcessing = {}; 39 | const dnsCache = {}; 40 | 41 | /** 42 | * Custom DNS lookup function that caches resolution result permanently. 43 | * 44 | * @param {string} hostname - The hostname to resolve. 45 | * @param {object} options - The options object. 46 | * @param {boolean} options.all - If true, return all resolved addresses. 47 | * @param {Function} cb - The callback function. 48 | */ 49 | function dnsLookup(hostname, options, cb) { 50 | const cached = dnsCache[hostname]; 51 | if (cached) { 52 | if (options.all) { 53 | cb(null, cached); 54 | } else { 55 | const addr = cached[0]; 56 | cb(null, addr.address, addr.family); 57 | } 58 | 59 | return; 60 | } 61 | 62 | if (dnsProcessing[hostname]) { 63 | // If a query for this hostname is already processing, wait until it's 64 | // finished. 65 | setTimeout(() => { 66 | dnsLookup(hostname, options, cb); 67 | }, 10); 68 | 69 | return; 70 | } 71 | 72 | dnsProcessing[hostname] = true; 73 | dns.lookup(hostname, { all: true, family: 4 }, (err, addresses) => { 74 | delete dnsProcessing[hostname]; 75 | 76 | if (err === null) { 77 | dnsCache[hostname] = addresses; 78 | } 79 | 80 | if (options.all) { 81 | cb(err, addresses); 82 | } else { 83 | if (addresses.length === 0) { 84 | cb(err, addresses); 85 | } 86 | const addr = addresses[0]; 87 | cb(err, addr.address, addr.family); 88 | } 89 | }); 90 | } 91 | 92 | /** 93 | * Removes trailing dot from an fully qualified domain name. The reason for 94 | * that is that urlfilter service does not know how to work with FQDN. 95 | * 96 | * @param {string} domain - The domain name to trim. 97 | * @returns {string} The domain name without trailing dot. 98 | */ 99 | function trimFqdn(domain) { 100 | return domain.endsWith('.') ? domain.slice(0, -1) : domain; 101 | } 102 | 103 | /** 104 | * Parses `Retry-After` header (seconds or HTTP date). 105 | * 106 | * @param {string} retryAfter - Header value. 107 | * @returns {number} Delay in milliseconds. 108 | */ 109 | function parseRetryAfter(retryAfter) { 110 | if (/^\d+$/.test(retryAfter)) { 111 | return parseInt(retryAfter, DECIMAL_BASE) * ONE_SECOND_MS; // Seconds to ms 112 | } 113 | const date = new Date(retryAfter); 114 | return !Number.isNaN(date.getTime()) ? date - Date.now() : null; 115 | } 116 | 117 | /** 118 | * Fetches a URL with retries respecting `Retry-After` headers. 119 | * 120 | * @param {string[]} domains - List of domains to fetch. 121 | * @param {number} [maxAttempts] - Maximum retry attempts 122 | * @throws {Error} If all attempts fail or fetch encounters network errors. 123 | * @returns {Promise} Fetch response. 124 | */ 125 | async function fetchWithRetry(domains, maxAttempts = DEFAULT_MAX_ATTEMPTS) { 126 | const url = new URL(`${URLFILTER_URL}`); 127 | url.searchParams.append('filter', 'none'); 128 | 129 | domains.forEach((domain) => { 130 | const asciiDomain = punycode.toASCII(domain); 131 | url.searchParams.append('domain', encodeURIComponent(trimFqdn(asciiDomain))); 132 | }); 133 | 134 | for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { 135 | // eslint-disable-next-line no-await-in-loop 136 | const response = await fetch(url, { 137 | agent: httpAgent, 138 | }); 139 | 140 | if (response.ok) { 141 | return response; 142 | } 143 | 144 | const retryAfter = response.headers.get('Retry-After'); 145 | if (!CODES_TO_RETRY.has(response.status)) { 146 | throw new Error(`Failed to fetch domains response code - ${response.status}`); 147 | } 148 | if (!retryAfter) { 149 | throw new Error(`Fetch status - ${response.status}, but no retry-after received for ${url}`); 150 | } 151 | 152 | const delayMs = parseRetryAfter(retryAfter); 153 | if (delayMs) { 154 | consola.info(`Retry required (attempt ${attempt}): Waiting ${delayMs}ms`); 155 | // eslint-disable-next-line no-await-in-loop 156 | await new Promise((resolve) => { 157 | setTimeout(resolve, delayMs); 158 | }); 159 | } else { 160 | throw new Error(`Unable to parse retry-after header -${retryAfter}`); 161 | } 162 | } 163 | throw Error(`Fetch domains failed: ${url}, tried ${maxAttempts} times`); 164 | } 165 | 166 | module.exports = { 167 | fetchWithRetry, 168 | trimFqdn, 169 | }; 170 | -------------------------------------------------------------------------------- /src/cli.js: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | const fs = require('fs'); 4 | const consola = require('consola'); 5 | // eslint-disable-next-line import/no-unresolved 6 | const consolaUtils = require('consola/utils'); 7 | const glob = require('glob'); 8 | const packageJson = require('../package.json'); 9 | const utils = require('./utils'); 10 | const fileLinter = require('./filelinter'); 11 | 12 | // eslint-disable-next-line import/order 13 | const { argv } = require('yargs') 14 | .usage('Usage: $0 [options]') 15 | .example( 16 | '$0 -i **/*.txt', 17 | 'scan all .txt files in the current directory and subdirectories in the interactive mode', 18 | ) 19 | .example( 20 | '$0 -a -i filter.txt', 21 | 'scan filter.txt and automatically apply suggested fixes', 22 | ) 23 | .option('input', { 24 | alias: 'i', 25 | type: 'string', 26 | description: 'glob expression that selects files that the tool will scan.', 27 | }) 28 | .option('dnscheck', { 29 | type: 'boolean', 30 | description: 'Double-check dead domains with a DNS query.', 31 | }) 32 | .option('commentout', { 33 | type: 'boolean', 34 | description: 'Comment out rules instead of removing them.', 35 | }) 36 | .option('export', { 37 | type: 'string', 38 | description: 'Export dead domains to the specified file instead of modifying the files.', 39 | }) 40 | .option('import', { 41 | type: 'string', 42 | description: 'Import dead domains from the specified file and skip other checks.', 43 | }) 44 | .option('ignore', { 45 | type: 'string', 46 | description: 'File with domains to ignore.', 47 | }) 48 | .option('auto', { 49 | alias: 'a', 50 | type: 'boolean', 51 | description: 'Automatically apply suggested fixes without asking the user.', 52 | }) 53 | .option('show', { 54 | alias: 's', 55 | type: 'boolean', 56 | description: 'Show suggestions without applying them.', 57 | }) 58 | .option('verbose', { 59 | alias: 'v', 60 | type: 'boolean', 61 | description: 'Run with verbose logging', 62 | }) 63 | .default('input', '**/*.txt') 64 | .default('dnscheck', true) 65 | .default('commentout', false) 66 | .default('auto', false) 67 | .default('show', false) 68 | .default('verbose', false) 69 | .version() 70 | .help('h') 71 | .alias('h', 'help'); 72 | 73 | if (argv.verbose) { 74 | // trace level. 75 | consola.level = 5; 76 | } 77 | 78 | /** 79 | * Extracts the list of dead domains from raw linting results. 80 | * 81 | * @param {import('./filelinter').FileResult} fileResult - Result of linting 82 | * the file. 83 | * @returns {Array} Array of dead domains. 84 | */ 85 | function getDeadDomains(fileResult) { 86 | if (!fileResult || !fileResult.results) { 87 | return []; 88 | } 89 | 90 | return fileResult.results.map((result) => { 91 | return result.linterResult.deadDomains; 92 | }).reduce((acc, val) => { 93 | return acc.concat(val); 94 | }, []); 95 | } 96 | 97 | /** 98 | * Entry point into the CLI program logic. 99 | */ 100 | async function main() { 101 | consola.info(`Starting ${packageJson.name} v${packageJson.version}`); 102 | 103 | const globExpression = argv.input; 104 | const files = glob.globSync(globExpression); 105 | const plural = files.length > 1 || files.length === 0; 106 | 107 | let predefinedDomains; 108 | if (argv.import) { 109 | consola.info(`Importing dead domains from ${argv.import}, other checks will be skipped`); 110 | 111 | try { 112 | predefinedDomains = fs.readFileSync(argv.import).toString() 113 | .split(/\r?\n/) 114 | .map((line) => line.trim()) 115 | .filter((line) => line !== ''); 116 | } catch (ex) { 117 | consola.error(`Failed to read from ${argv.import}: ${ex}`); 118 | 119 | process.exit(1); 120 | } 121 | 122 | consola.info(`Imported ${predefinedDomains.length} dead domains`); 123 | } 124 | 125 | consola.info(`Found ${files.length} file${plural ? 's' : ''} matching ${globExpression}`); 126 | 127 | let ignoreDomainsList = []; 128 | if (argv.ignore) { 129 | consola.info(`Importing domains to ignore from ${argv.ignore}`); 130 | try { 131 | ignoreDomainsList = fs.readFileSync(argv.ignore).toString() 132 | .split(/\r?\n/) 133 | .map((line) => line.trim()) 134 | .filter((line) => line !== ''); 135 | } catch (ex) { 136 | consola.error(`Failed to read from ${argv.ignore}: ${ex}`); 137 | 138 | process.exit(1); 139 | } 140 | consola.info(`Imported ${ignoreDomainsList.length} domains to ignore`); 141 | } 142 | const ignoreDomains = new Set(ignoreDomainsList); 143 | // This array is used when export is enabled. 144 | const deadDomains = []; 145 | 146 | for (let i = 0; i < files.length; i += 1) { 147 | const file = files[i]; 148 | 149 | try { 150 | consola.info(consolaUtils.colorize('bold', `Processing file ${file}`)); 151 | 152 | const linterOptions = { 153 | show: argv.show, 154 | auto: argv.auto || !!argv.export, 155 | useDNS: argv.dnscheck, 156 | commentOut: argv.commentout, 157 | deadDomains: predefinedDomains, 158 | ignoreDomains, 159 | }; 160 | 161 | // eslint-disable-next-line no-await-in-loop 162 | const fileResult = await fileLinter.lintFile(file, linterOptions); 163 | 164 | if (fileResult !== null) { 165 | if (argv.export) { 166 | deadDomains.push(...getDeadDomains(fileResult)); 167 | } else { 168 | // eslint-disable-next-line no-await-in-loop 169 | await fileLinter.applyFileChanges(file, fileResult, linterOptions); 170 | } 171 | } 172 | } catch (ex) { 173 | consola.error(`Failed to process ${file} due to ${ex}`); 174 | 175 | process.exit(1); 176 | } 177 | } 178 | 179 | if (argv.export) { 180 | consola.info(`Exporting the list of dead domains to ${argv.export}`); 181 | const uniqueDomains = utils.unique(deadDomains); 182 | fs.writeFileSync(argv.export, uniqueDomains.join('\n')); 183 | } 184 | 185 | consola.success('Finished successfully'); 186 | } 187 | 188 | main(); 189 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dead Domains Linter 2 | 3 | [![npm-badge]][npm-url] [![license-badge]][license-url] 4 | 5 | This is a simple tool that checks adblock filtering rules for dead domains. 6 | 7 | In the future, it should be replaced with an [AGLint rule][aglintrule]. 8 | 9 | [aglintrule]: https://github.com/AdguardTeam/AGLint/issues/194 10 | [npm-badge]: https://img.shields.io/npm/v/@adguard/dead-domains-linter 11 | [npm-url]: https://www.npmjs.com/package/@adguard/dead-domains-linter 12 | [license-badge]: https://img.shields.io/github/license/AdGuardTeam/DeadDomainsLinter 13 | [license-url]: https://github.com/AdguardTeam/DeadDomainsLinter/blob/master/LICENSE 14 | 15 | ## How to use 16 | 17 | ### Installation and update 18 | 19 | First of all, install the dead-domains-linter: 20 | 21 | ```shell 22 | npm i -g @adguard/dead-domains-linter 23 | ``` 24 | 25 | > [!NOTE] 26 | > If you have it installed and need to update to a newer version, run this 27 | > command: 28 | > 29 | > ```shell 30 | > npm update -g @adguard/dead-domains-linter 31 | > ``` 32 | 33 | ### Interactive mode 34 | 35 | By default it runs in interactive mode, scans the current directory and all its 36 | subdirectories for `*.txt` files, and asks the user to apply suggested changes. 37 | 38 | Just run it in the directory with your filter lists to see how it works: 39 | 40 | ```shell 41 | dead-domains-linter 42 | ``` 43 | 44 | Here's how the interactive mode looks like: 45 | ![dead-domain-linter](https://cdn.adtidy.org/website/github.com/DeadDomainsLinter/default-config.png) 46 | 47 | You can specify a custom glob expression to select files that the tool will 48 | scan: 49 | 50 | ```shell 51 | dead-domains-linter -i filter.txt 52 | ``` 53 | 54 | ### Automatic and show-only mode 55 | 56 | You can allow it to automatically apply suggestions by passing the `--auto` 57 | flag: 58 | 59 | ```shell 60 | dead-domains-linter --auto 61 | ``` 62 | 63 | Alternatively, you can run it in the "show only" mode: 64 | 65 | ```shell 66 | dead-domains-linter --show 67 | ``` 68 | 69 | ### Commenting rules out instead of removing them 70 | 71 | One more useful feature would be to comment out filter rules that contain dead 72 | domains instead of removing them. You can enable this feature by passing the 73 | `--commentout=true` flag: 74 | 75 | ```shell 76 | dead-domains-linter --commentout 77 | ``` 78 | 79 | ### Ignoring domains 80 | 81 | If there are specific domains you want the tool to ignore and treat as valid, 82 | you can use the --ignore flag and provide a file containing a list of domains to be excluded. 83 | 84 | ### Exporting and using a pre-defined list of domains 85 | 86 | Instead of immediately modifying the filter list, you may opt to export the 87 | list of dead domains so that you could carefully review it. For instance, the 88 | command below scans `filter.txt` for dead domains and exports this list to 89 | `domains.txt`. 90 | 91 | ```shell 92 | dead-domains-linter -i filter.txt --export=domains.txt 93 | ``` 94 | 95 | When you finish the review and clean up the list, you can make the tool use it 96 | exclusively to modify filter lists. For instance, the command below scans 97 | `filter.txt` for all domains that are in `domains.txt` and removes them from 98 | the filter list. 99 | 100 | ```shell 101 | dead-domains-linter -i filter.txt --import=domains.txt --auto 102 | ``` 103 | 104 | ### Disabling DNS check 105 | 106 | > [!IMPORTANT] 107 | > Please read this if you maintain a filter list with a large number of users. 108 | 109 | The tool relies on AdGuard DNS snapshot of the Internet domains that represents 110 | all domains used by 100M+ AdGuard DNS users for the last 24 hours. Using this 111 | snapshot is a good way to find dead domains, but it alone may not be 100% 112 | accurate and it can produce false positives for really rarely visited domains. 113 | This is why the tool also double-checks dead domains with a DNS query. 114 | 115 | If your filter list does not have a large number of dead domains, we recommend 116 | disabling that double-check by running the tool with the `--dnscheck=false` 117 | flag: 118 | 119 | ```shell 120 | dead-domains-linter --dnscheck=false 121 | ``` 122 | 123 | > [!NOTE] 124 | > Actually, AdGuard [filter policy][filterpolicy] requires that the website 125 | > should be popular enough to be added to the filter list. So there's a great 126 | > chance that even when the tool produced a false positive when running with 127 | > `--dnscheck=false`, this domain anyways does not qualify for the filter list. 128 | 129 | [filterpolicy]: https://adguard.com/kb/general/ad-filtering/filter-policy/ 130 | 131 | ### Full usage info 132 | 133 | ```shell 134 | Usage: dead-domains-linter [options] 135 | 136 | Options: 137 | -i, --input glob expression that selects files that the tool will scan. 138 | [string] [default: "**/*.txt"] 139 | --dnscheck Double-check dead domains with a DNS query. 140 | [boolean] [default: true] 141 | --commentout Comment out rules instead of removing them. 142 | [boolean] [default: false] 143 | --export Export dead domains to the specified file instead of 144 | modifying the files. [string] 145 | --import Import dead domains from the specified file and skip other 146 | checks. [string] 147 | --ignore File with domains to ignore. [string] 148 | -a, --auto Automatically apply suggested fixes without asking the user. 149 | [boolean] [default: false] 150 | -s, --show Show suggestions without applying them. 151 | [boolean] [default: false] 152 | -v, --verbose Run with verbose logging [boolean] [default: false] 153 | --version Show version number [boolean] 154 | -h, --help Show help [boolean] 155 | 156 | Examples: 157 | dead-domains-linter -i **/*.txt scan all .txt files in the current 158 | directory and subdirectories in the 159 | interactive mode 160 | dead-domains-linter -a -i filter.txt scan filter.txt and automatically apply 161 | suggested fixes 162 | ``` 163 | 164 | ## How to develop 165 | 166 | First, install [pnpm](https://pnpm.io/): `npm install -g pnpm`. 167 | 168 | Then you can use the following commands: 169 | 170 | * `pnpm install` - install dependencies. 171 | * `pnpm run lint` - lint the code. 172 | * `pnpm run test` - run the unit-tests. 173 | -------------------------------------------------------------------------------- /test/mocked/linter.test.js: -------------------------------------------------------------------------------- 1 | jest.mock('node-fetch'); 2 | const fetch = require('node-fetch'); 3 | const agtree = require('@adguard/agtree'); 4 | const punycode = require('node:punycode'); 5 | const checker = require('../../src/linter'); 6 | 7 | describe('Linter mocked tests', () => { 8 | beforeEach(() => { 9 | fetch.mockReset(); 10 | }); 11 | 12 | const testLintRule = (rule, expected, ignoreDomains = new Set()) => { 13 | return async () => { 14 | const deadDomainsToMock = expected?.deadDomains.map((domain) => punycode.toASCII(domain)) || []; 15 | 16 | fetch.mockImplementation(async (url) => { 17 | const urlObj = new URL(url); 18 | const requestedDomains = urlObj.searchParams.getAll('domain'); 19 | const responseData = {}; 20 | 21 | requestedDomains.forEach((domain) => { 22 | const isDead = deadDomainsToMock.includes(domain); 23 | responseData[domain] = { 24 | info: { 25 | domain_name: domain, 26 | registered_domain: domain, 27 | registered_domain_used_last_24_hours: !isDead, 28 | used_last_24_hours: !isDead, 29 | }, 30 | matches: [], 31 | }; 32 | }); 33 | 34 | return { 35 | status: 200, 36 | ok: true, 37 | headers: { get: () => 'application/json' }, 38 | json: jest.fn().mockResolvedValue(responseData), 39 | }; 40 | }); 41 | 42 | const ast = agtree.RuleParser.parse(rule); 43 | const result = await checker.lintRule(ast, { useDNS: false, ignoreDomains }); 44 | 45 | if (expected === null) { 46 | expect(result).toEqual(null); 47 | return; 48 | } 49 | 50 | if (expected.remove) { 51 | expect(result.suggestedRule).toEqual(null); 52 | } else { 53 | const ruleText = agtree.RuleParser.generate(result.suggestedRule); 54 | expect(ruleText).toEqual(expected.suggestedRuleText); 55 | } 56 | 57 | expect(result.deadDomains).toEqual(expected.deadDomains); 58 | }; 59 | }; 60 | 61 | it('suggest removing rule with a dead domain in the pattern', testLintRule( 62 | '||example.notexistingdomain^', 63 | { remove: true, deadDomains: ['example.notexistingdomain'] }, 64 | )); 65 | 66 | it('suggest removing rule with a dead domain in the pattern URL', testLintRule( 67 | '||example.notexistingdomain/thisissomepath/tosomewhere', 68 | { remove: true, deadDomains: ['example.notexistingdomain'] }, 69 | )); 70 | 71 | it('ignore removing rule with a dead domain in the pattern URL', testLintRule( 72 | '||example.notexistingdomain/thisissomepath/tosomewhere', 73 | null, 74 | new Set(['example.notexistingdomain']), 75 | )); 76 | 77 | it('do not suggest removing IP addresses', testLintRule( 78 | '||1.2.3.4^', 79 | null, 80 | )); 81 | 82 | it('do not suggest removing .onion domains', testLintRule( 83 | '||example.onion^', 84 | null, 85 | )); 86 | 87 | it('suggest removing dead non ASCII domain from modifier', testLintRule( 88 | '||example.org^$domain=ппример2.рф', 89 | { remove: true, deadDomains: ['ппример2.рф'] }, 90 | )); 91 | 92 | it('do nothing with a simple rule with existing domain', testLintRule( 93 | '||example.org^$third-party', 94 | null, 95 | )); 96 | 97 | it('suggest removing negated domain from $domain', testLintRule( 98 | '||example.org^$domain=example.org|example.notexistingdomain', 99 | { suggestedRuleText: '||example.org^$domain=example.org', deadDomains: ['example.notexistingdomain'] }, 100 | )); 101 | 102 | it('suggest removing the whole rule when all permitted domains are dead', testLintRule( 103 | '||example.org^$domain=example.notexisting1|example.notexisting2', 104 | { remove: true, deadDomains: ['example.notexisting1', 'example.notexisting2'] }, 105 | )); 106 | 107 | it('ignore dead domain as part of $domain modifier', testLintRule( 108 | '||example.org^$domain=example.notexisting1|google.com|example.notexisting2', 109 | { 110 | suggestedRuleText: '||example.org^$domain=example.notexisting1|google.com', 111 | deadDomains: ['example.notexisting2'], 112 | }, 113 | new Set(['example.notexisting1']), 114 | )); 115 | 116 | // Cosmetic rules tests 117 | it('suggest removing an element hiding rule which was only for dead domains', testLintRule( 118 | 'example.notexistingdomain##banner', 119 | { remove: true, deadDomains: ['example.notexistingdomain'] }, 120 | )); 121 | 122 | it('ignore removing an element hiding rule for dead domains', testLintRule( 123 | 'example.notexistingdomain##banner', 124 | null, 125 | new Set(['example.notexistingdomain']), 126 | )); 127 | 128 | it('keep the rule if there are permitted domains left', testLintRule( 129 | 'example.org,example.notexistingdomain##banner', 130 | { suggestedRuleText: 'example.org##banner', deadDomains: ['example.notexistingdomain'] }, 131 | )); 132 | 133 | it('keep the rule if all dead domains were negated', testLintRule( 134 | '~example.notexistingdomain##banner', 135 | { suggestedRuleText: '##banner', deadDomains: ['example.notexistingdomain'] }, 136 | )); 137 | 138 | it('suggest removing a scriptlet rule', testLintRule( 139 | 'example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', 140 | { remove: true, deadDomains: ['example.notexistingdomain'] }, 141 | )); 142 | 143 | it('ignore removing a scriptlet rule with ignore domain', testLintRule( 144 | 'example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', 145 | null, 146 | new Set(['example.notexistingdomain']), 147 | )); 148 | 149 | it('suggest modifying a scriptlet rule', testLintRule( 150 | 'example.org,example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', 151 | { 152 | suggestedRuleText: 'example.org#%#//scriptlet("set-constant", "a", "1")', 153 | deadDomains: ['example.notexistingdomain'], 154 | }, 155 | )); 156 | 157 | it('ignore modifying a scriptlet rule with ignore domain', testLintRule( 158 | 'example.org,example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', 159 | null, 160 | new Set(['example.notexistingdomain']), 161 | )); 162 | }); 163 | -------------------------------------------------------------------------------- /src/filelinter.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-await-in-loop */ 2 | const fs = require('fs'); 3 | const consola = require('consola'); 4 | // eslint-disable-next-line import/no-unresolved 5 | const consolaUtils = require('consola/utils'); 6 | const agtree = require('@adguard/agtree'); 7 | const linter = require('./linter'); 8 | 9 | // Filter list lines are processing in parallel in chunks of this size. For 10 | // now we chose 10 as the test run shows it to provide good enough results 11 | // without overloading the web service. 12 | // 13 | // TODO(ameshkov): Consider making it configurable. 14 | const PARALLEL_CHUNK_SIZE = 10; 15 | 16 | /** 17 | * Represents options for the linter. 18 | * 19 | * @typedef {object} FileLintOptions 20 | * 21 | * @property {boolean} show - If true, the linter will only show suggested 22 | * changes, but will not confirm them. 23 | * @property {boolean} auto - If true, the linter will automatically count all 24 | * suggested changes as confirm. 25 | * @property {boolean} useDNS - If true, the linter doublecheck results received 26 | * from the urlfilter web service with a DNS query. 27 | * @property {boolean} commentOut - If true, the linter will suggest commenting 28 | * a rule out instead of removing it. 29 | * @property {Array} deadDomains - Pre-defined list of dead domains. If 30 | * it is specified, skip all other checks. 31 | * @property {Set} ignoreDomains - Set of domains to ignore. 32 | */ 33 | 34 | /** 35 | * Helper function that checks the "automatic" flag first before asking user. 36 | * 37 | * @param {string} message - Question to ask the user in the prompt. 38 | * @param {FileLintOptions} options - Configuration for this linter run. 39 | * @returns {Promise} True if the user confirmed the action, false 40 | * otherwise. 41 | */ 42 | async function confirm(message, options) { 43 | if (options.show) { 44 | consola.info(`${message}: declined automatically`); 45 | 46 | return false; 47 | } 48 | 49 | if (options.auto) { 50 | consola.info(`${message}: confirmed automatically`); 51 | 52 | return true; 53 | } 54 | 55 | const answer = await consola.prompt(message, { 56 | type: 'confirm', 57 | }); 58 | 59 | return answer; 60 | } 61 | 62 | /** 63 | * Represents result of processing a rule AST. 64 | * 65 | * @typedef {object} AstResult 66 | * 67 | * @property {string} line - Text of the rule that's was processed. 68 | * @property {number} lineNumber - Number of that line. 69 | * @property {import('./linter').LinterResult} linterResult - Result of linting 70 | * that line. 71 | */ 72 | 73 | /** 74 | * Process the rule AST from the specified file and returns the linting result 75 | * or null if nothing needs to be changed. 76 | * 77 | * @param {string} file - Path to the file that's being processed. 78 | * @param {agtree.AnyRule} ast - AST of the rule that's being processed. 79 | * @param {FileLintOptions} options - Configuration for this linter run. 80 | * @returns {Promise} Returns null if nothing needs to be changed or 81 | * AstResult if the linter found any issues. 82 | */ 83 | async function processRuleAst(file, ast, options) { 84 | const line = ast.raws.text; 85 | const lineNumber = ast.loc.start.line; 86 | 87 | try { 88 | consola.verbose(`Processing ${file}:${lineNumber}: ${line}`); 89 | 90 | const linterResult = await linter.lintRule(ast, { 91 | useDNS: options.useDNS, 92 | deadDomains: options.deadDomains, 93 | ignoreDomains: options.ignoreDomains, 94 | }); 95 | 96 | // If the result is empty, the line can be simply skipped. 97 | if (!linterResult) { 98 | return null; 99 | } 100 | 101 | if (linterResult.suggestedRule === null && options.commentOut) { 102 | const suggestedRuleText = `! commented out by dead-domains-linter: ${line}`; 103 | linterResult.suggestedRule = agtree.RuleParser.parse(suggestedRuleText); 104 | } 105 | 106 | return { 107 | line, 108 | lineNumber, 109 | linterResult, 110 | }; 111 | } catch (ex) { 112 | consola.warn(`Failed to process line ${lineNumber} due to ${ex}, skipping it`); 113 | 114 | return null; 115 | } 116 | } 117 | 118 | /** 119 | * Process the filter list AST and returns a list of changes that are confirmed 120 | * by the user. 121 | * 122 | * @param {string} file - Path to the file that's being processed. 123 | * @param {agtree.FilterList} listAst - AST of the filter list to process. 124 | * @param {FileLintOptions} options - Configuration for this linter run. 125 | * 126 | * @returns {Promise>} Returns the list of changes that are confirmed. 127 | */ 128 | async function processListAst(file, listAst, options) { 129 | consola.start(`Analyzing ${listAst.children.length} rules`); 130 | 131 | let processing = 0; 132 | let analyzedRules = 0; 133 | let issuesCount = 0; 134 | 135 | const processingResults = await Promise.all(listAst.children.map((ast) => { 136 | return (async () => { 137 | // Using a simple semaphore-like construction to limit the number of 138 | // parallel processing tasks. 139 | while (processing >= PARALLEL_CHUNK_SIZE) { 140 | // Waiting for 10ms until the next check. 10ms is an arbitrarily 141 | // chosen value, there's no big difference between 100-10-1. 142 | 143 | await new Promise((resolve) => { setTimeout(resolve, 10); }); 144 | } 145 | 146 | processing += 1; 147 | try { 148 | const result = await processRuleAst(file, ast, options); 149 | if (result !== null) { 150 | issuesCount += 1; 151 | } 152 | 153 | return result; 154 | } finally { 155 | analyzedRules += 1; 156 | processing -= 1; 157 | 158 | if (analyzedRules % 100 === 0) { 159 | consola.info(`Analyzed ${analyzedRules} rules, found ${issuesCount} issues`); 160 | } 161 | } 162 | })(); 163 | })); 164 | 165 | const results = processingResults.filter((res) => res !== null); 166 | 167 | consola.success(`Found ${results.length} issues`); 168 | 169 | // Sort the results by line number in ascending order. 170 | results.sort((a, b) => a.lineNumber - b.lineNumber); 171 | 172 | // Now ask the user whether the changes are allowed. 173 | const allowedResults = []; 174 | for (let i = 0; i < results.length; i += 1) { 175 | const result = results[i]; 176 | const { suggestedRule, deadDomains } = result.linterResult; 177 | const suggestedRuleText = suggestedRule === null ? '' : suggestedRule.raws.text; 178 | 179 | consola.info(`Found dead domains in a rule: ${deadDomains.join(', ')}`); 180 | consola.info(consolaUtils.colorize('red', `- ${result.lineNumber}: ${result.line}`)); 181 | consola.info(consolaUtils.colorize('green', `+ ${result.lineNumber}: ${suggestedRuleText}`)); 182 | 183 | const confirmed = await confirm('Apply suggested fix?', options); 184 | if (confirmed) { 185 | allowedResults.push(result); 186 | } 187 | } 188 | 189 | return allowedResults; 190 | } 191 | 192 | /** 193 | * Result of linting the file. 194 | * 195 | * @typedef {object} FileResult 196 | * 197 | * @property {agtree.FilterList} listAst - AST of the filter list. 198 | * @property {Array} results - List of changes to apply to the filter 199 | * list. 200 | */ 201 | 202 | /** 203 | * Lints the specified file and returns the resulting list of changes and 204 | * the original file AST. 205 | * 206 | * @param {string} file - Path to the file that the program should process. 207 | * @param {FileLintOptions} options - Configuration for this linter run. 208 | * @returns {Promise} Object with the file linting result or 209 | * null if there is nothing to change. 210 | */ 211 | async function lintFile(file, options) { 212 | const content = fs.readFileSync(file, 'utf8'); 213 | 214 | // Parsing the whole filter list. 215 | const listAst = agtree.FilterListParser.parse(content); 216 | 217 | if (!listAst.children || listAst.children.length === 0) { 218 | consola.info(`No rules found in ${file}`); 219 | 220 | return null; 221 | } 222 | 223 | const results = await processListAst(file, listAst, options); 224 | 225 | if (results.length === 0) { 226 | consola.info(`No changes to ${file}`); 227 | 228 | return null; 229 | } 230 | 231 | return { 232 | listAst, 233 | results, 234 | }; 235 | } 236 | 237 | /** 238 | * Asks for the user permission to change the file. 239 | * 240 | * @param {string} file - Path to the file being analyzed. 241 | * @param {FileResult} fileResult - Result of linting the file. 242 | * @param {FileLintOptions} options - Configuration for this linter run. 243 | * @returns {Promise} True if the user confirmed the changes. 244 | */ 245 | async function confirmFileChanges(file, fileResult, options) { 246 | const { results } = fileResult; 247 | 248 | // Count the number of lines that are to be removed. 249 | const cntRemove = results.reduce((cnt, res) => { 250 | return res.linterResult.suggestedRule === null ? cnt + 1 : cnt; 251 | }, 0); 252 | const cntModify = results.reduce((cnt, res) => { 253 | return res.linterResult.suggestedRule !== null ? cnt + 1 : cnt; 254 | }, 0); 255 | 256 | const summaryMsg = `${consolaUtils.colorize('bold', `Summary for ${file}:`)}\n` 257 | + `${cntRemove} line${cntRemove.length > 1 || cntRemove.length === 0 ? 's' : ''} will be removed.\n` 258 | + `${cntModify} line${cntModify.length > 1 || cntModify.length === 0 ? 's' : ''} will be modified.`; 259 | 260 | consola.box(summaryMsg); 261 | 262 | const confirmed = await confirm('Apply modifications to the file?', options); 263 | 264 | return confirmed; 265 | } 266 | 267 | /** 268 | * Applies confirmed changes to the file. 269 | * 270 | * @param {string} file - Path to the file. 271 | * @param {FileResult} fileResult - Result of linting the file. 272 | * @param {FileLintOptions} options - Configuration for this linter run. 273 | */ 274 | async function applyFileChanges(file, fileResult, options) { 275 | const confirmed = await confirmFileChanges(file, fileResult, options); 276 | 277 | if (!confirmed) { 278 | consola.info(`Skipping file ${file}`); 279 | 280 | return; 281 | } 282 | 283 | consola.info(`Applying modifications to ${file}`); 284 | 285 | const { listAst, results } = fileResult; 286 | 287 | // Sort result by lineNumber descending so that we could use it for the 288 | // original array modification. 289 | results.sort((a, b) => b.lineNumber - a.lineNumber); 290 | 291 | // Go through the results array in and either remove or modify the 292 | // lines. 293 | for (let i = 0; i < results.length; i += 1) { 294 | const result = results[i]; 295 | const lineIdx = result.lineNumber - 1; 296 | 297 | if (result.linterResult.suggestedRule === null) { 298 | listAst.children.splice(lineIdx, 1); 299 | } else { 300 | listAst.children[lineIdx] = result.linterResult.suggestedRule; 301 | } 302 | } 303 | 304 | // Generate a new filter list contents, use raw text when it's 305 | // available in a rule AST. 306 | const newContents = agtree.FilterListParser.generate(listAst, true); 307 | 308 | // Update the filter list file. 309 | fs.writeFileSync(file, newContents); 310 | } 311 | 312 | module.exports = { 313 | lintFile, 314 | applyFileChanges, 315 | }; 316 | -------------------------------------------------------------------------------- /src/linter.js: -------------------------------------------------------------------------------- 1 | const agtree = require('@adguard/agtree'); 2 | const urlfilter = require('./urlfilter'); 3 | const dnscheck = require('./dnscheck'); 4 | const utils = require('./utils'); 5 | 6 | /** 7 | * A list of network rule modifiers that are to be scanned for dead domains. 8 | */ 9 | const DOMAIN_MODIFIERS = ['domain', 'denyallow', 'from', 'to']; 10 | 11 | /** 12 | * Regular expression that matches the domain in a network rule pattern. 13 | */ 14 | const PATTERN_DOMAIN_REGEX = (() => { 15 | const startStrings = [ 16 | '||', '//', '://', 'http://', 'https://', '|http://', '|https://', 17 | 'ws://', 'wss://', '|ws://', '|wss://', 18 | ]; 19 | 20 | const startRegex = startStrings.map((str) => { 21 | return str.replace(/\//g, '\\/').replace(/\|/g, '\\|'); 22 | }).join('|'); 23 | 24 | return new RegExp(`^(@@)?(${startRegex})([a-z0-9-\\p{L}.]+)(\\^|\\/)`, 'iu'); 25 | })(); 26 | 27 | /** 28 | * Attempts to extract the domain from the network rule pattern. If the pattern 29 | * does not contain a domain, returns null. 30 | * 31 | * @param {agtree.NetworkRule} ast - The rule AST. 32 | * @returns {string | null} The domain extracted from the AST. 33 | */ 34 | function extractDomainFromPattern(ast) { 35 | if (!ast.pattern) { 36 | return null; 37 | } 38 | 39 | // Ignore regular expressions as we cannot be sure if it's for a hostname 40 | // or not. 41 | // 42 | // TODO(ameshkov): Handle the most common cases like (negative lookahead + 43 | // a list of domains). 44 | if (ast.pattern.value.startsWith('/') && ast.pattern.value.endsWith('/')) { 45 | return null; 46 | } 47 | 48 | const match = ast.pattern.value.match(PATTERN_DOMAIN_REGEX); 49 | if (match && utils.validDomain(match[3])) { 50 | const domain = match[3]; 51 | 52 | return domain; 53 | } 54 | 55 | return null; 56 | } 57 | 58 | /** 59 | * Represents a domain that is used in the rule. It can be a negated domain. 60 | * 61 | * @typedef {object} RuleDomain 62 | * 63 | * @property {string} domain - The domain name. 64 | * @property {boolean} negated - True if the domain is negated. 65 | */ 66 | 67 | /** 68 | * Extracts an array of domains from the network rule modifier. 69 | * 70 | * @param {agtree.Modifier} modifier - The modifier that contains the domains. 71 | * @returns {Array} The list of domains extracted from the modifier. 72 | */ 73 | function extractModifierDomains(modifier) { 74 | if (!modifier.value.value) { 75 | return []; 76 | } 77 | 78 | const domainList = agtree.DomainListParser.parse(modifier.value.value, agtree.PIPE_MODIFIER_SEPARATOR); 79 | return domainList.children.map((domain) => { 80 | return { 81 | domain: domain.value, 82 | negated: domain.exception, 83 | }; 84 | }); 85 | } 86 | 87 | /** 88 | * Extracts domains from a network rule AST. 89 | * 90 | * @param {agtree.NetworkRule} ast - The AST of a network rule to extract 91 | * domains from. 92 | * @returns {Array} The list of all domains that are used by this rule. 93 | */ 94 | function extractNetworkRuleDomains(ast) { 95 | const domains = []; 96 | 97 | const patternDomain = extractDomainFromPattern(ast); 98 | if (patternDomain) { 99 | domains.push(patternDomain); 100 | } 101 | 102 | if (!ast.modifiers) { 103 | // No modifiers in the rule, return right away. 104 | return domains; 105 | } 106 | 107 | for (let i = 0; i < ast.modifiers.children.length; i += 1) { 108 | const modifier = ast.modifiers.children[i]; 109 | 110 | if (DOMAIN_MODIFIERS.includes(modifier.modifier.value)) { 111 | const modifierDomains = extractModifierDomains(modifier) 112 | .map((domain) => domain.domain); 113 | 114 | domains.push(...modifierDomains); 115 | } 116 | } 117 | 118 | return utils.unique(domains).filter(utils.validDomain); 119 | } 120 | 121 | /** 122 | * Extracts domains from a cosmetic rule AST. 123 | * 124 | * @param {agtree.CosmeticRule} ast - The AST of a cosmetic rule to extract 125 | * domains from. 126 | * @returns {Array} The list of all domains that are used by this rule. 127 | */ 128 | function extractCosmeticRuleDomains(ast) { 129 | // TODO(ameshkov): Extract and analyze cosmetic rules modifiers too. 130 | 131 | if (!ast.domains || ast.domains.length === 0) { 132 | return []; 133 | } 134 | 135 | const domains = ast.domains.children 136 | .map((domain) => domain.value) 137 | .filter(utils.validDomain); 138 | 139 | return utils.unique(domains); 140 | } 141 | 142 | /** 143 | * This function goes through the rule AST and extracts domains from it. 144 | * 145 | * @param {agtree.AnyRule} ast - The AST of the rule to extract domains from. 146 | * @returns {Array} The list of all domains that are used by this rule. 147 | */ 148 | function extractRuleDomains(ast) { 149 | switch (ast.category) { 150 | case 'Network': 151 | return extractNetworkRuleDomains(ast); 152 | case 'Cosmetic': 153 | return extractCosmeticRuleDomains(ast); 154 | default: 155 | return []; 156 | } 157 | } 158 | 159 | /** 160 | * Modifies the cosmetic rule AST and removes the rule elements that contain the 161 | * dead domains. 162 | * 163 | * @param {agtree.CosmeticRule} ast - The network rule AST to modify and remove 164 | * dead domains. 165 | * @param {Array} deadDomains - A list of dead domains. 166 | * 167 | * @returns {agtree.AnyRule|null} Returns AST of the rule text with suggested 168 | * modification. Returns null if the rule must be removed. 169 | */ 170 | function modifyCosmeticRule(ast, deadDomains) { 171 | if (!ast.domains || ast.domains.children.length === 0) { 172 | // Do nothing if there are no domains in the rule. In theory, it 173 | // shouldn't happen, but check it just in case. 174 | return ast; 175 | } 176 | 177 | const newAst = structuredClone(ast); 178 | 179 | const hasPermittedDomains = newAst.domains.children.some((domain) => !domain.exception); 180 | 181 | // Go through ast.domains and remove those that contain dead domains. 182 | // Iterate in the reverse order to keep the indices correct. 183 | for (let i = newAst.domains.children.length - 1; i >= 0; i -= 1) { 184 | const domain = newAst.domains.children[i]; 185 | 186 | if (deadDomains.includes(domain.value)) { 187 | newAst.domains.children.splice(i, 1); 188 | } 189 | } 190 | 191 | const hasPermittedDomainsAfterFilter = newAst.domains.children.some((domain) => !domain.exception); 192 | if (hasPermittedDomains && !hasPermittedDomainsAfterFilter) { 193 | // Suggest removing the whole rule if it had permitted domains before, 194 | // but does not have it anymore. 195 | // 196 | // Example: 197 | // example.org##banner -> ##banner 198 | // 199 | // The rule must be removed in this case as otherwise its 200 | // scope will change to global. 201 | return null; 202 | } 203 | 204 | if (newAst.domains.children.length === 0 && newAst.exception) { 205 | // Suggest removing the whole rule if this is an exception rule. 206 | // 207 | // Example: 208 | // example.org#@#banner -> #@#banner 209 | return null; 210 | } 211 | 212 | // Update raws because it can be used to serialize the filter list back when 213 | // applying changes. 214 | newAst.raws.text = agtree.RuleParser.generate(newAst); 215 | 216 | return newAst; 217 | } 218 | 219 | /** 220 | * Modifies the network rule AST and removes the rule elements that contain the 221 | * dead domains. 222 | * 223 | * @param {agtree.NetworkRule} ast - The network rule AST to modify and remove 224 | * dead domains. 225 | * @param {Array} deadDomains - A list of dead domains. 226 | * 227 | * @returns {agtree.AnyRule|null} Returns AST of the rule text with suggested 228 | * modification. Returns null if the rule must be removed. 229 | */ 230 | function modifyNetworkRule(ast, deadDomains) { 231 | const patternDomain = extractDomainFromPattern(ast); 232 | if (deadDomains.includes(patternDomain)) { 233 | // Suggest completely removing the rule if it contains a dead domain in 234 | // the pattern. 235 | return null; 236 | } 237 | 238 | if (!ast.modifiers) { 239 | // No modifiers in the rule, nothing to do. 240 | return ast; 241 | } 242 | 243 | // Clone the AST and apply all modifications to the cloned version. 244 | const newAst = structuredClone(ast); 245 | 246 | const modifierIdxToRemove = []; 247 | 248 | // Go through the network rule modifiers and remove the dead domains from 249 | // them. Depending on the result, remove the whole modifier or even suggest 250 | // removing the whole rule. 251 | for (let i = 0; i < newAst.modifiers.children.length; i += 1) { 252 | const modifier = newAst.modifiers.children[i]; 253 | 254 | if (DOMAIN_MODIFIERS.includes(modifier.modifier.value)) { 255 | const modifierDomains = extractModifierDomains(modifier); 256 | 257 | // Check if modifierDomains had at least one non-negated domain. 258 | const hasPermittedDomains = modifierDomains.some((domain) => !domain.negated); 259 | 260 | // Remove the dead domains from the modifier. 261 | const filteredDomains = modifierDomains.filter( 262 | (domain) => !deadDomains.includes(domain.domain), 263 | ); 264 | 265 | // Check if filteredDomains now has at least one non-negated domain. 266 | const hasPermittedDomainsAfterFilter = filteredDomains.some((domain) => !domain.negated); 267 | 268 | if (hasPermittedDomains && !hasPermittedDomainsAfterFilter) { 269 | // Suggest completely removing the rule if there are no 270 | // permitted domains left now. 271 | // 272 | // Example: 273 | // ||example.org^$domain=example.org -> ||example.org^ 274 | // 275 | // The rule must be removed in this case as otherwise its 276 | // scope will change to global. 277 | return null; 278 | } 279 | 280 | if (filteredDomains.length === 0) { 281 | modifierIdxToRemove.push(i); 282 | } 283 | 284 | // TODO(ameshkov): Refactor extractModifierDomains so that we could 285 | // use DomainListParser.generate here. 286 | modifier.value.value = filteredDomains.map( 287 | (domain) => { 288 | return domain.negated ? `${agtree.NEGATION_MARKER}${domain.domain}` : domain.domain; 289 | }, 290 | ).join(agtree.PIPE_MODIFIER_SEPARATOR); 291 | } 292 | } 293 | 294 | // If there were any modifiers that should be removed, remove them. 295 | if (modifierIdxToRemove.length > 0) { 296 | // Remove the modifiers in reverse order to keep the indices correct. 297 | for (let i = modifierIdxToRemove.length - 1; i >= 0; i -= 1) { 298 | newAst.modifiers.children.splice(modifierIdxToRemove[i], 1); 299 | } 300 | } 301 | 302 | if (newAst.modifiers.children.length === 0) { 303 | // No modifiers left in the rule, make the whole node undefined. 304 | newAst.modifiers = undefined; 305 | } 306 | 307 | newAst.raws.text = agtree.RuleParser.generate(newAst); 308 | 309 | return newAst; 310 | } 311 | 312 | /** 313 | * Modifies the rule AST and removes the rule elements that contain the dead 314 | * domains. 315 | * 316 | * @param {agtree.AnyRule} ast - The rule AST to modify and remove dead domains. 317 | * @param {Array} deadDomains - A list of dead domains. 318 | * 319 | * @returns {agtree.AnyRule|null} Returns AST of the rule text with suggested 320 | * modification. Returns null if the rule must be removed. 321 | * @throws {Error} Throws an error if the rule AST if for unexpected category. 322 | */ 323 | function modifyRule(ast, deadDomains) { 324 | switch (ast.category) { 325 | case 'Network': 326 | return modifyNetworkRule(ast, deadDomains); 327 | case 'Cosmetic': 328 | return modifyCosmeticRule(ast, deadDomains); 329 | default: 330 | throw new Error(`Unsupported rule category: ${ast.category}`); 331 | } 332 | } 333 | 334 | // Cache for the results of the domains check. The key is the domain name, the 335 | // value is true for alive domains, false for dead. 336 | const domainsCheckCache = {}; 337 | 338 | /** 339 | * Goes through the list of domains that needs to be checked and uses the 340 | * urlfilter web service to check which of them are dead. 341 | * 342 | * @param {Array} domains - Parts of the rule with domains. 343 | * @param {LintOptions} options - Configuration for the linting process. 344 | * @returns {Promise>} A list of dead domains. 345 | */ 346 | async function findDeadDomains(domains, options) { 347 | const deadDomains = []; 348 | const domainsToCheck = []; 349 | 350 | // If we have a pre-defined list of domains, skip all other checks and just 351 | // go through it. 352 | if (options.deadDomains && options.deadDomains.length > 0) { 353 | domains.forEach((domain) => { 354 | if (options.deadDomains.includes(domain) && !options.ignoreDomains.has(domain)) { 355 | deadDomains.push(domain); 356 | } 357 | }); 358 | 359 | return utils.unique(deadDomains); 360 | } 361 | 362 | const nonIgnoredDomains = domains.filter((domain) => !options.ignoreDomains.has(domain)); 363 | // eslint-disable-next-line no-restricted-syntax 364 | for (const domain of utils.unique(nonIgnoredDomains)) { 365 | if (Object.prototype.hasOwnProperty.call(domainsCheckCache, domain)) { 366 | if (domainsCheckCache[domain] === false) { 367 | deadDomains.push(domain); 368 | } 369 | } else { 370 | domainsToCheck.push(domain); 371 | } 372 | } 373 | 374 | const checkResult = await urlfilter.findDeadDomains(domainsToCheck); 375 | 376 | if (options.useDNS) { 377 | // eslint-disable-next-line no-restricted-syntax 378 | for (const domain of checkResult) { 379 | // eslint-disable-next-line no-await-in-loop 380 | const dnsRecordExists = await dnscheck.checkDomain(domain); 381 | 382 | if (!dnsRecordExists) { 383 | deadDomains.push(domain); 384 | } 385 | } 386 | } else { 387 | deadDomains.push(...checkResult); 388 | } 389 | 390 | // eslint-disable-next-line no-restricted-syntax 391 | for (const domain of domainsToCheck) { 392 | if (deadDomains.includes(domain)) { 393 | domainsCheckCache[domain] = false; 394 | } else { 395 | domainsCheckCache[domain] = true; 396 | } 397 | } 398 | 399 | return deadDomains; 400 | } 401 | 402 | /** 403 | * Configures the linting process. 404 | * 405 | * @typedef {object} LintOptions 406 | * 407 | * @property {boolean} useDNS - If true, use a DNS query to doublecheck domains 408 | * returned by the urlfilter web service. 409 | * @property {Array} deadDomains - Pre-defined list of dead domains. If 410 | * it is specified, skip all other checks. 411 | * @property {Set} ignoreDomains - Set of domains to ignore. 412 | */ 413 | 414 | /** 415 | * Result of the dead domains check. Contains the list of dead domains found 416 | * in the rule and the suggested rule text after removing dead domains. 417 | * 418 | * @typedef {object} LinterResult 419 | * 420 | * @property {agtree.AnyRule|null} suggestedRule - AST of the suggested rule 421 | * after removing dead domains. If the whole rule should be removed, this field 422 | * is null. 423 | * @property {Array} deadDomains - A list of dead domains in the rule. 424 | */ 425 | 426 | /** 427 | * This function parses the input rule, extracts all domain names that can be 428 | * found there and checks if they are alive using the urlfilter web service. 429 | * 430 | * @param {string} ast - AST of the rule that we're going to check. 431 | * @param {LintOptions} options - Configuration for the linting process. 432 | * @returns {Promise} Result of the rule linting. 433 | */ 434 | async function lintRule(ast, options) { 435 | const domains = extractRuleDomains(ast); 436 | if (!domains || domains.length === 0) { 437 | return null; 438 | } 439 | 440 | const deadDomains = await findDeadDomains(domains, options); 441 | 442 | if (!deadDomains || deadDomains.length === 0) { 443 | return null; 444 | } 445 | 446 | const suggestedRule = modifyRule(ast, deadDomains); 447 | 448 | return { 449 | suggestedRule, 450 | deadDomains, 451 | }; 452 | } 453 | 454 | module.exports = { lintRule }; 455 | -------------------------------------------------------------------------------- /test/integration/linter.test.js: -------------------------------------------------------------------------------- 1 | const agtree = require('@adguard/agtree'); 2 | const checker = require('../../src/linter'); 3 | 4 | const testLintRule = (rule, expected, useDNS = false, ignoreDomains = new Set()) => { 5 | return async () => { 6 | const ast = agtree.RuleParser.parse(rule); 7 | const result = await checker.lintRule(ast, { useDNS, ignoreDomains }); 8 | 9 | if (expected === null) { 10 | expect(result).toEqual(null); 11 | 12 | return; 13 | } 14 | 15 | if (expected.remove) { 16 | expect(result.suggestedRule).toEqual(null); 17 | } else { 18 | const ruleText = agtree.RuleParser.generate(result.suggestedRule); 19 | expect(ruleText).toEqual(expected.suggestedRuleText); 20 | } 21 | 22 | expect(result.deadDomains).toEqual(expected.deadDomains); 23 | }; 24 | }; 25 | 26 | describe('Linter', () => { 27 | describe('Lint with DNS double-check', () => { 28 | it( 29 | 'suggest removing rule with a dead (with DNS double-check) domain in the pattern', 30 | testLintRule('||example.notexistingdomain^', { 31 | remove: true, 32 | deadDomains: ['example.notexistingdomain'], 33 | }, true), 34 | ); 35 | }); 36 | 37 | describe('Network rules', () => { 38 | it( 39 | 'suggest removing rule with a dead domain in the pattern', 40 | testLintRule('||example.notexistingdomain^', { 41 | remove: true, 42 | deadDomains: ['example.notexistingdomain'], 43 | }), 44 | ); 45 | 46 | it( 47 | 'suggest removing rule with a dead domain in the pattern URL', 48 | testLintRule('||example.notexistingdomain/thisissomepath/tosomewhere', { 49 | remove: true, 50 | deadDomains: ['example.notexistingdomain'], 51 | }), 52 | ); 53 | 54 | it( 55 | 'ignore removing rule with a dead domain in the pattern URL', 56 | testLintRule( 57 | '||example.notexistingdomain/thisissomepath/tosomewhere', 58 | null, 59 | false, 60 | new Set(['example.notexistingdomain']), 61 | ), 62 | ); 63 | 64 | it( 65 | 'suggest removing rule with a dead domain in the pattern from exception rule', 66 | testLintRule('@@||example.notexistingdomain/thisissomepath/tosomewhere', { 67 | remove: true, 68 | deadDomains: ['example.notexistingdomain'], 69 | }), 70 | ); 71 | 72 | it( 73 | 'suggest removing rule with a dead FQDN in the pattern', 74 | testLintRule('||example.notexistingdomain.^', { 75 | remove: true, 76 | deadDomains: ['example.notexistingdomain.'], 77 | }), 78 | ); 79 | 80 | it( 81 | 'suggest removing rule with a dead domain in the pattern with ://', 82 | testLintRule('://example.notexistingdomain/thisissomepath/tosomewhere', { 83 | remove: true, 84 | deadDomains: ['example.notexistingdomain'], 85 | }), 86 | ); 87 | 88 | it( 89 | 'suggest removing rule with a dead domain with an WebSocket URL pattern', 90 | testLintRule('wss://example.notexistingdomain/thisissomepath/tosomewhere', { 91 | remove: true, 92 | deadDomains: ['example.notexistingdomain'], 93 | }), 94 | ); 95 | 96 | it( 97 | 'suggest removing rule with a dead domain with an URL pattern', 98 | testLintRule('|https://example.notexistingdomain/thisissomepath/tosomewhere', { 99 | remove: true, 100 | deadDomains: ['example.notexistingdomain'], 101 | }), 102 | ); 103 | 104 | it( 105 | 'do not suggest removing IP addresses', 106 | testLintRule('||1.2.3.4^', null), 107 | ); 108 | 109 | it( 110 | 'do not suggest removing IP ranges', 111 | testLintRule('||203.195.121.$popup', null), 112 | ); 113 | 114 | it( 115 | 'do not suggest removing .onion domains', 116 | testLintRule('||example.onion^', null), 117 | ); 118 | 119 | it( 120 | 'do not suggest removing .lib domains', 121 | testLintRule('||example.lib^', null), 122 | ); 123 | 124 | it( 125 | 'do not suggest removing non ASCII domains', 126 | testLintRule('||поддерживаю.рф^', null), 127 | ); 128 | 129 | it( 130 | 'do not suggest removing non ASCII domains from modifier', 131 | testLintRule('||example.org^$domain=поддерживаю.рф', null), 132 | ); 133 | 134 | it( 135 | 'suggest removing dead non ASCII domain from modifier', 136 | testLintRule('||example.org^$domain=ппример2.рф', { 137 | remove: true, 138 | deadDomains: ['ппример2.рф'], 139 | }), 140 | ); 141 | 142 | it( 143 | 'do not suggest removing rules that target browser extensions', 144 | testLintRule('@@||evernote.com^$domain=pioclpoplcdbaefihamjohnefbikjilc', null), 145 | ); 146 | 147 | it( 148 | 'do nothing with a simple rule with existing domain', 149 | testLintRule('||example.org^$third-party', null), 150 | ); 151 | 152 | it( 153 | 'do not break $domain with TLD patterns', 154 | testLintRule('||example.org^$domain=google.*', null), 155 | ); 156 | 157 | it( 158 | 'do not break $domain with regexes', 159 | testLintRule('||example.org^$domain=/some.randomstring/', null), 160 | ); 161 | 162 | it( 163 | 'do not break $domain with regexes with pipes', 164 | testLintRule('||example.org^$domain=/(^\\|.+\\.)c\\.(com\\|org)\\$/|example.notexistingdomain', { 165 | suggestedRuleText: '||example.org^$domain=/(^\\|.+\\.)c\\.(com\\|org)\\$/', 166 | deadDomains: ['example.notexistingdomain'], 167 | }), 168 | ); 169 | 170 | it( 171 | 'do nothing check rules without domains', 172 | testLintRule('$script,third-party', null), 173 | ); 174 | 175 | it( 176 | 'do not break on comments', 177 | testLintRule('! this is a comment', null), 178 | ); 179 | 180 | it( 181 | 'do not break on empty lines', 182 | testLintRule('', null), 183 | ); 184 | 185 | it( 186 | 'suggest removing negated domain from $domain', 187 | testLintRule('||example.org^$domain=example.org|example.notexistingdomain', { 188 | suggestedRuleText: '||example.org^$domain=example.org', 189 | deadDomains: ['example.notexistingdomain'], 190 | }), 191 | ); 192 | 193 | it( 194 | 'suggest removing negated domain from $domain and keep TLD pattern', 195 | testLintRule('||example.org^$domain=example.*|example.notexistingdomain', { 196 | suggestedRuleText: '||example.org^$domain=example.*', 197 | deadDomains: ['example.notexistingdomain'], 198 | }), 199 | ); 200 | 201 | it( 202 | 'suggest removing negated domain from $domain and keep regex pattern', 203 | testLintRule('||example.org^$domain=/example.test/|example.notexistingdomain', { 204 | suggestedRuleText: '||example.org^$domain=/example.test/', 205 | deadDomains: ['example.notexistingdomain'], 206 | }), 207 | ); 208 | 209 | it( 210 | 'suggest removing the whole rule when all permitted domains are dead', 211 | testLintRule('||example.org^$domain=example.notexisting1|example.notexisting2', { 212 | remove: true, 213 | deadDomains: ['example.notexisting1', 'example.notexisting2'], 214 | }), 215 | ); 216 | 217 | it( 218 | 'ignore dead domain as part of $domain modifier', 219 | testLintRule( 220 | '||example.org^$domain=example.notexisting1|google.com|example.notexisting2', 221 | { 222 | suggestedRuleText: '||example.org^$domain=example.notexisting1|google.com', 223 | deadDomains: ['example.notexisting2'], 224 | }, 225 | false, 226 | new Set(['example.notexisting1']), 227 | ), 228 | ); 229 | 230 | it( 231 | 'suggest removing one negated domain of two from a network rule', 232 | testLintRule('||example.org^$domain=~example.org|~example.notexisting', { 233 | suggestedRuleText: '||example.org^$domain=~example.org', 234 | deadDomains: ['example.notexisting'], 235 | }), 236 | ); 237 | 238 | it( 239 | 'suggest removing the whole $domain modifier when it had only dead negated domains', 240 | testLintRule('||example.org^$domain=~example.notexisting1|~example.notexisting2', { 241 | suggestedRuleText: '||example.org^', 242 | deadDomains: ['example.notexisting1', 'example.notexisting2'], 243 | }), 244 | ); 245 | 246 | it( 247 | 'suggest keeping $domain modifier when a permitted domain is alive', 248 | testLintRule('||example.org^$domain=example.org|~example.notexisting1|~example.notexisting2,third-party', { 249 | suggestedRuleText: '||example.org^$domain=example.org,third-party', 250 | deadDomains: ['example.notexisting1', 'example.notexisting2'], 251 | }), 252 | ); 253 | 254 | it( 255 | 'ignore with $domain modifier', 256 | testLintRule('||example.org^$domain=example.org|~example.notexisting1|~example.notexisting2,third-party', { 257 | suggestedRuleText: '||example.org^$domain=example.org|~example.notexisting1,third-party', 258 | deadDomains: ['example.notexisting2'], 259 | }, false, new Set(['example.notexisting1'])), 260 | ); 261 | 262 | it( 263 | 'suggest removing the whole rule when domains in $denyallow are dead', 264 | testLintRule('||example.org^$denyallow=example.notexisting1', { 265 | remove: true, 266 | deadDomains: ['example.notexisting1'], 267 | }), 268 | ); 269 | 270 | it( 271 | 'ignore removing rule with ignored dead domain in the pattern from exception rule', 272 | testLintRule( 273 | '@@||example.notexistingdomain/thisissomepath/tosomewhere', 274 | null, 275 | true, 276 | new Set(['example.notexistingdomain']), 277 | ), 278 | ); 279 | }); 280 | 281 | describe('Cosmetic rules', () => { 282 | it( 283 | 'do not break rules without domains', 284 | testLintRule('##banner', null), 285 | ); 286 | 287 | it( 288 | 'do not break domains with TLD pattern', 289 | testLintRule('google.*##banner', null), 290 | ); 291 | 292 | it( 293 | 'suggest removing an element hiding rule which was only for dead domains', 294 | testLintRule('example.notexistingdomain##banner', { 295 | remove: true, 296 | deadDomains: ['example.notexistingdomain'], 297 | }), 298 | ); 299 | 300 | it( 301 | 'ignore removing an element hiding rule which was th dead domains', 302 | testLintRule( 303 | 'example.notexistingdomain##banner', 304 | null, 305 | false, 306 | new Set(['example.notexistingdomain']), 307 | ), 308 | ); 309 | 310 | it( 311 | 'keep the rule if there are permitted domains left', 312 | testLintRule('example.org,example.notexistingdomain##banner', { 313 | suggestedRuleText: 'example.org##banner', 314 | deadDomains: ['example.notexistingdomain'], 315 | }), 316 | ); 317 | 318 | it( 319 | 'keep the rule if all dead domains were negated', 320 | testLintRule('~example.notexistingdomain##banner', { 321 | suggestedRuleText: '##banner', 322 | deadDomains: ['example.notexistingdomain'], 323 | }), 324 | ); 325 | 326 | it( 327 | 'suggest removing the whole rule if it was an exception rule', 328 | testLintRule('~example.notexistingdomain#@#banner', { 329 | remove: true, 330 | deadDomains: ['example.notexistingdomain'], 331 | }), 332 | ); 333 | 334 | it( 335 | 'suggest removing one negated domain of two from a network rule', 336 | testLintRule('~example.org,~example.notexisting##banner', { 337 | suggestedRuleText: '~example.org##banner', 338 | deadDomains: ['example.notexisting'], 339 | }), 340 | ); 341 | 342 | it( 343 | 'suggest removing a CSS injection rule', 344 | testLintRule('example.notexistingdomain#$#banner { height: 0px; }', { 345 | remove: true, 346 | deadDomains: ['example.notexistingdomain'], 347 | }), 348 | ); 349 | 350 | it( 351 | 'suggest removing a CSS injection exception rule', 352 | testLintRule('example.notexistingdomain#@$#banner { height: 0px; }', { 353 | remove: true, 354 | deadDomains: ['example.notexistingdomain'], 355 | }), 356 | ); 357 | 358 | it( 359 | 'suggest modifying a CSS injection rule', 360 | testLintRule('example.org,example.notexistingdomain#$#banner { height: 0px; }', { 361 | suggestedRuleText: 'example.org#$#banner { height: 0px; }', 362 | deadDomains: ['example.notexistingdomain'], 363 | }), 364 | ); 365 | 366 | it( 367 | 'suggest removing an extended CSS rule', 368 | testLintRule('example.notexistingdomain#?#banner', { 369 | remove: true, 370 | deadDomains: ['example.notexistingdomain'], 371 | }), 372 | ); 373 | 374 | it( 375 | 'suggest removing an extended CSS exception rule', 376 | testLintRule('example.notexistingdomain#@?#banner', { 377 | remove: true, 378 | deadDomains: ['example.notexistingdomain'], 379 | }), 380 | ); 381 | 382 | it( 383 | 'suggest modifying an extended CSS rule', 384 | testLintRule('example.org,example.notexistingdomain#?#banner', { 385 | suggestedRuleText: 'example.org#?#banner', 386 | deadDomains: ['example.notexistingdomain'], 387 | }), 388 | ); 389 | 390 | it( 391 | 'suggest removing a scriptlet rule', 392 | testLintRule('example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', { 393 | remove: true, 394 | deadDomains: ['example.notexistingdomain'], 395 | }), 396 | ); 397 | 398 | it( 399 | 'ignore removing a scriptlet rule with ignore domain', 400 | testLintRule( 401 | 'example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', 402 | null, 403 | false, 404 | new Set(['example.notexistingdomain']), 405 | ), 406 | ); 407 | 408 | it( 409 | 'suggest removing a scriptlet exception rule', 410 | testLintRule('example.notexistingdomain#@%#//scriptlet("set-constant", "a", "1")', { 411 | remove: true, 412 | deadDomains: ['example.notexistingdomain'], 413 | }), 414 | ); 415 | 416 | it( 417 | 'ignore modifying a scriptlet rule with ignore domain', 418 | testLintRule( 419 | 'example.org,example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', 420 | null, 421 | false, 422 | new Set(['example.notexistingdomain']), 423 | ), 424 | ); 425 | 426 | it( 427 | 'suggest modifying a scriptlet rule', 428 | testLintRule('example.org,example.notexistingdomain#%#//scriptlet("set-constant", "a", "1")', { 429 | suggestedRuleText: 'example.org#%#//scriptlet("set-constant", "a", "1")', 430 | deadDomains: ['example.notexistingdomain'], 431 | }), 432 | ); 433 | 434 | it( 435 | 'suggest removing an HTML filtering rule', 436 | testLintRule('example.notexistingdomain$$banner', { 437 | remove: true, 438 | deadDomains: ['example.notexistingdomain'], 439 | }), 440 | ); 441 | 442 | it( 443 | 'suggest removing an HTML filtering exception rule', 444 | testLintRule('example.notexistingdomain$@$banner', { 445 | remove: true, 446 | deadDomains: ['example.notexistingdomain'], 447 | }), 448 | ); 449 | 450 | it( 451 | 'suggest modifying an HTML filtering rule', 452 | testLintRule('example.org,example.notexistingdomain$$banner', { 453 | suggestedRuleText: 'example.org$$banner', 454 | deadDomains: ['example.notexistingdomain'], 455 | }), 456 | ); 457 | 458 | it( 459 | 'ignore all CSS dead domains when they are in the ignore list', 460 | testLintRule( 461 | 'example.org,example.notexistingdomain$$banner', 462 | null, 463 | false, 464 | new Set(['example.notexistingdomain', 'example.org']), 465 | ), 466 | ); 467 | 468 | it( 469 | 'ignore some CSS dead domains when they are in the ignore list', 470 | testLintRule( 471 | 'example2.notexistingdomain,example.notexistingdomain$$banner', 472 | { 473 | suggestedRuleText: 'example.notexistingdomain$$banner', 474 | deadDomains: ['example2.notexistingdomain'], 475 | }, 476 | false, 477 | new Set(['example.notexistingdomain']), 478 | ), 479 | ); 480 | }); 481 | }); 482 | --------------------------------------------------------------------------------