├── .dockerignore ├── .eslintignore ├── .eslintrc ├── .github ├── CODEOWNERS ├── renovate.json └── workflows │ ├── build.yml │ ├── codeql-analysis.yml │ ├── integration-tests.yml │ ├── regression-tests.yml │ └── unit-tests.yml ├── .gitignore ├── Brocfile.js ├── Dockerfile.ci ├── LICENSE ├── README.md ├── broccoli ├── Brocfile.brave.js ├── Brocfile.node.js ├── Brocfile.webextension.js ├── config.js ├── env.js ├── modules-tree.js ├── modules │ ├── broccoli-webpack.js │ ├── bundles-tree.js │ ├── content-script-imports.js │ ├── content-tests-imports.js │ ├── dist-tree.js │ ├── integration-tests-imports.js │ └── modules-list.js └── specific-tree.js ├── configs ├── ci │ ├── integration-tests.js │ └── unit-tests.js ├── common │ ├── subprojects │ │ └── bundles.js │ └── urls.js ├── extension.js ├── module.js ├── production.js └── sandbox.js ├── fern.js ├── fern ├── commands │ ├── build.js │ ├── lint.js │ ├── pack.js │ ├── serve.js │ └── test.js ├── common.js └── reporter.js ├── modules ├── .gitkeep ├── content-script-tests │ ├── build-config.json │ ├── sources │ │ ├── background.es │ │ └── content.es │ └── tests │ │ └── integration │ │ ├── content-script-test.es │ │ └── messaging-test.es ├── core │ ├── build-config.json │ ├── sources │ │ ├── LRU.ts │ │ ├── app.es │ │ ├── app │ │ │ ├── life-cycle.es │ │ │ ├── module-errors.es │ │ │ ├── module.es │ │ │ └── service.es │ │ ├── array-buffer-utils.ts │ │ ├── background.es │ │ ├── base │ │ │ └── background.es │ │ ├── bloom-filter.es │ │ ├── browser.es │ │ ├── console.es │ │ ├── content-script.bundle.es │ │ ├── content-script.es │ │ ├── content-tests.bundle.es │ │ ├── content.es │ │ ├── content │ │ │ ├── actions-manager.es │ │ │ ├── console.es │ │ │ ├── glob.es │ │ │ ├── helpers.es │ │ │ ├── i18n.es │ │ │ ├── match-patterns.es │ │ │ ├── ready-promise.es │ │ │ ├── register.es │ │ │ └── run.es │ │ ├── crypto │ │ │ ├── pkcs-conversion.es │ │ │ ├── random.es │ │ │ └── utils.es │ │ ├── decorators.es │ │ ├── encoding.es │ │ ├── event-emitter.es │ │ ├── events.es │ │ ├── globals-window.es │ │ ├── gzip.es │ │ ├── helpers │ │ │ ├── default-map.ts │ │ │ ├── defer.es │ │ │ ├── dynamic-data-view.es │ │ │ ├── fixed-size-cache.es │ │ │ ├── md5.es │ │ │ ├── remote-action-provider.es │ │ │ ├── sleep.es │ │ │ ├── string-cache.ts │ │ │ ├── strip-api-headers.es │ │ │ ├── timeout.es │ │ │ └── wait.es │ │ ├── history-service.es │ │ ├── http.es │ │ ├── kord.es │ │ ├── kord │ │ │ └── inject.es │ │ ├── lint.es │ │ ├── logger.es │ │ ├── message-queue.es │ │ ├── platform.es │ │ ├── prefs.es │ │ ├── request-sanitizer.es │ │ ├── services.es │ │ ├── services │ │ │ └── pacemaker.es │ │ ├── settings.es │ │ ├── time.es │ │ ├── timers.es │ │ ├── tlds.es │ │ ├── url.ts │ │ ├── webrequest.es │ │ └── zlib.es │ └── tests │ │ ├── http-server.es │ │ ├── integration │ │ ├── helpers.es │ │ └── http-test.es │ │ ├── test-helpers.es │ │ └── unit │ │ ├── app-test.es │ │ ├── app │ │ └── module-test.es │ │ ├── bloom-filter-test.es │ │ ├── content │ │ └── match-patterns-test.es │ │ ├── data │ │ └── encoding-data.json │ │ ├── decorators-test.es │ │ ├── encoding-data.es │ │ ├── encoding-test.es │ │ ├── events-test.es │ │ ├── fixed-size-cache-test.es │ │ ├── helpers-md5-test.es │ │ ├── helpers │ │ └── strip-api-headers-test.es │ │ ├── http-test.es │ │ ├── message-queue-test.es │ │ ├── pkcs-conversion-test.es │ │ ├── prefs-test.es │ │ ├── services │ │ └── pacemaker-test.es │ │ ├── url-test.es │ │ ├── utils │ │ ├── dexie.es │ │ ├── fs.es │ │ └── url-parser.es │ │ └── wait-test.es ├── fetcher │ ├── build-config.json │ └── sources │ │ ├── background.es │ │ ├── logger.es │ │ └── manager.es ├── hpnv2 │ ├── build-config.json │ ├── sources │ │ ├── README.md │ │ ├── background.es │ │ ├── config-loader.es │ │ ├── constants.es │ │ ├── database.es │ │ ├── digest.es │ │ ├── endpoints.es │ │ ├── errors.es │ │ ├── group-signer.es │ │ ├── logger.es │ │ ├── manager.es │ │ ├── message-throttler.es │ │ ├── star.es │ │ ├── trusted-clock.es │ │ ├── utils.es │ │ ├── window.es │ │ ├── worker-common.es │ │ ├── worker.asmjs.bundle.es │ │ └── worker.wasm.bundle.es │ └── tests │ │ └── unit │ │ ├── config-loader-test.es │ │ ├── digest-test.es │ │ ├── manager-test.es │ │ ├── message-throttler-test.es │ │ ├── trusted-clock-test.es │ │ └── utils-test.es ├── integration-tests │ ├── build-config.json │ ├── dist │ │ ├── index.html │ │ ├── setup.js │ │ └── tap-reporter.js │ └── sources │ │ ├── background.es │ │ ├── initialize-test-helpers.es │ │ └── run.bundle.es ├── web-discovery-project │ ├── build-config.json │ ├── dist │ │ ├── .gitkeep │ │ ├── anonpatterns.json │ │ └── patterns.json │ ├── sources │ │ ├── README.md │ │ ├── ad-detection.es │ │ ├── background.es │ │ ├── content-extractor.es │ │ ├── content.es │ │ ├── doublefetch-handler.es │ │ ├── fallback-dns.es │ │ ├── html-helpers.es │ │ ├── logger.es │ │ ├── network.es │ │ ├── remote-resource-fetcher.ts │ │ ├── remote-resource-watcher.ts │ │ ├── safebrowsing-endpoint.es │ │ ├── signature-verifier.ts │ │ ├── styles │ │ │ └── .gitkeep │ │ ├── web-discovery-project-patterns-loader.ts │ │ ├── web-discovery-project.es │ │ └── window.es │ └── tests │ │ ├── integration │ │ ├── doublefetch-test.es │ │ ├── utility-regression-test.es │ │ └── web-discovery-project-test.es │ │ └── unit │ │ ├── ad-detection-test.es │ │ ├── content-extractor-test.es │ │ ├── content-test.es │ │ ├── doublefetch-handler-test.es │ │ ├── fallback-dns-test.es │ │ ├── fixtures │ │ ├── content-extractor │ │ │ ├── am │ │ │ │ ├── fussballschuh-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── gardening-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ └── test-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ ├── bing │ │ │ │ ├── donald-trump-2018-05-28 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ └── img-search-2020-02-23 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ ├── empty-page │ │ │ │ ├── page.html.gz │ │ │ │ └── scenario.json │ │ │ ├── go │ │ │ │ ├── angela-merkel-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── cricket-icc-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── elon-musk-twitter-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── f1-2023-09-27 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── goetze-wm-2014-tor-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── green-apple-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── grossformat-laserdrucker-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── konrad-zuse-bilder-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── kuckucksuhr-kaufen-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── nba-2023-09-27 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── nirvana-polly-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── passauer-dom-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── sneezing-panda-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── sq-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ ├── trump-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ │ └── von-der-leyen-2023-10-10 │ │ │ │ │ ├── page.html.gz │ │ │ │ │ └── scenario.json │ │ │ ├── patterns-anon.json │ │ │ ├── patterns.json │ │ │ └── yahoo │ │ │ │ └── donald-trump-2018-05-28 │ │ │ │ ├── page.html.gz │ │ │ │ └── scenario.json │ │ └── content-test │ │ │ ├── android-user-agent-page-with-ads-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ │ ├── android-user-agent-page-without-ads-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ │ ├── coffee-ads-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ │ ├── flight-page-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ │ ├── gardening-shoes-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ │ ├── page-with-no-ads-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ │ ├── potato-ads-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ │ └── shoe-ads-2019-07-24 │ │ │ ├── expected-ads.json │ │ │ └── page.html.gz │ │ ├── generate-fixtures.js │ │ ├── generate-fixtures.sh │ │ ├── network-test.es │ │ ├── remote-resource-watcher-test.es │ │ ├── signature-verifier-test.es │ │ └── web-discovery-project-test.es ├── webextension-specific │ └── sources │ │ ├── app.bundle.es │ │ └── background.es └── webrequest-pipeline │ ├── sources │ ├── background.es │ ├── logger.es │ ├── page-store.es │ ├── page.ts │ ├── pipeline.es │ └── webrequest-context.es │ └── tests │ ├── integration │ └── webrequest-pipeline-test.es │ └── unit │ └── pipeline-test.es ├── package-lock.json ├── package.json ├── patches └── broccoli+3.5.2.patch ├── platforms └── webextension │ ├── browser.es │ ├── console.es │ ├── content-communication-manager.es │ ├── content │ └── globals.es │ ├── crypto.es │ ├── fast-content-app-state-injection.es │ ├── fetch.es │ ├── globals-chrome.es │ ├── globals-window.es │ ├── globals.es │ ├── gzip.es │ ├── history-service.es │ ├── integration-tests │ ├── test-launcher.es │ └── test-page-sources.es │ ├── kv-store.es │ ├── lib │ ├── dexie.es │ ├── tldts.es │ └── zlib.es │ ├── location-change-observer.es │ ├── permission-manager.es │ ├── platform.es │ ├── prefs.es │ ├── resource-loader-storage.es │ ├── runtime.es │ ├── tabs.es │ ├── test-helpers │ └── helpers.es │ ├── text-decoder.es │ ├── text-encoder.es │ ├── timers.es │ ├── web-discovery-project │ ├── doublefetch.es │ ├── opentabs.es │ └── storage.es │ ├── webnavigation.es │ ├── webrequest.es │ ├── windows.es │ ├── worker.es │ └── xmlhttprequest.es ├── run_tests_in_docker.sh ├── specific ├── node │ └── index.js └── web-discovery-project │ ├── assets │ └── brave.png │ └── manifest.json ├── testem.json ├── tests ├── run_tests.sh ├── runners │ ├── brave-web-ext.js │ ├── firefox-web-ext.js │ ├── launchers │ │ ├── brave-web-ext.js │ │ ├── firefox-web-ext.js │ │ └── test-options.js │ ├── tap-result-streamer.js │ ├── test-runner-common.js │ └── unit-node.js └── test-server.js ├── tsconfig.json └── update-brave.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | modules/web-discovery-project/sources/web-discovery-project.es 2 | modules/core/dist/EventUtils.js 3 | specific/node/index.js 4 | build/ 5 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "extends": [ 4 | "eslint:recommended", 5 | "plugin:compat/recommended", 6 | "prettier" 7 | ], 8 | "parser": "@babel/eslint-parser", 9 | "parserOptions": { 10 | "requireConfigFile": false 11 | }, 12 | "env": { 13 | "browser": true, 14 | "commonjs": false, 15 | "es6": true, 16 | "mocha": true, 17 | "node": true, 18 | "shared-node-browser": true, 19 | "webextensions": true, 20 | "worker": true 21 | }, 22 | "globals": {}, 23 | "rules": { 24 | "compat/compat": "error", 25 | "import/extensions": "off", 26 | "no-underscore-dangle": "off", 27 | "import/no-unresolved": "off", 28 | "no-restricted-globals": ["error", "Worker"], 29 | }, 30 | "overrides": [ 31 | { 32 | "files": ["**/*.ts"], 33 | "extends": [ 34 | "eslint:recommended", 35 | "plugin:compat/recommended", 36 | "plugin:@typescript-eslint/recommended", 37 | "plugin:@typescript-eslint/recommended-requiring-type-checking" 38 | ], 39 | "parser": "@typescript-eslint/parser", 40 | "parserOptions": { 41 | "project": "./tsconfig.json" 42 | }, 43 | "plugins": ["@typescript-eslint"], 44 | "rules": { 45 | "lines-between-class-members": "off", 46 | "@typescript-eslint/no-unsafe-call": "off", 47 | "@typescript-eslint/no-unsafe-member-access": "off", 48 | "@typescript-eslint/no-unsafe-assignment": "off", 49 | "@typescript-eslint/no-unsafe-return": "off", 50 | "@typescript-eslint/restrict-template-expressions": "off", 51 | "@typescript-eslint/no-unsafe-argument": "off", 52 | "@typescript-eslint/no-explicit-any": "off" 53 | } 54 | } 55 | ] 56 | } 57 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @remusao 2 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "local>brave/renovate-config" 4 | ], 5 | "ignoreDeps": ["ansi-regex", "systemjs", "eslint"], 6 | "bumpVersion": "patch", 7 | "labels": [ 8 | "PR: Dependencies :nut_and_bolt:", 9 | "renovate" 10 | ], 11 | "reviewers": [ 12 | "remusao" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build configs 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | build: 11 | name: Trigger build 12 | runs-on: ubuntu-24.04 13 | strategy: 14 | matrix: 15 | node-version: [22] 16 | config: ["configs/extension.js", "configs/module.js"] 17 | 18 | steps: 19 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 20 | 21 | - name: Use Node.js ${{ matrix.node-version }} 22 | uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 23 | with: 24 | node-version: ${{ matrix.node-version }} 25 | 26 | - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 27 | with: 28 | path: ~/.npm 29 | key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} 30 | restore-keys: | 31 | ${{ runner.os }}-node- 32 | 33 | - run: | 34 | npm install 35 | 36 | - run: node --unhandled-rejections=strict fern.js build ${{ matrix.config }} 37 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [main] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [main] 20 | schedule: 21 | - cron: "23 1 * * 4" 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: ["javascript"] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@2c779ab0d087cd7fe7b826087247c2c81f27bfa6 # v3.26.5 72 | -------------------------------------------------------------------------------- /.github/workflows/integration-tests.yml: -------------------------------------------------------------------------------- 1 | name: Integration tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-24.04 12 | steps: 13 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 14 | - run: ./run_tests_in_docker.sh "configs/ci/integration-tests.js -l brave-web-ext --grep UtilityRegression -i --brave /opt/brave.com/brave/brave-browser" 15 | -------------------------------------------------------------------------------- /.github/workflows/regression-tests.yml: -------------------------------------------------------------------------------- 1 | name: Regression tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-24.04 12 | steps: 13 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 14 | - run: ./run_tests_in_docker.sh "configs/ci/integration-tests.js -l brave-web-ext --grep UtilityRegression --brave /opt/brave.com/brave/brave-browser" 15 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-24.04 12 | strategy: 13 | matrix: 14 | node-version: [22] 15 | steps: 16 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 17 | 18 | - name: Use Node.js ${{ matrix.node-version }} 19 | uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 20 | with: 21 | node-version: ${{ matrix.node-version }} 22 | 23 | - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 24 | with: 25 | path: ~/.npm 26 | key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} 27 | restore-keys: | 28 | ${{ runner.os }}-node- 29 | 30 | - run: npm install 31 | 32 | - run: ./node_modules/.bin/eslint . 33 | 34 | - run: node --unhandled-rejections=strict fern.js build configs/ci/unit-tests.js --include-tests 35 | 36 | - run: node --unhandled-rejections=strict fern.js test configs/ci/unit-tests.js -l unit-node --environment testing --no-build --ci report.xml 37 | 38 | - if: ${{ github.actor != 'dependabot[bot]' && github.actor != 'renovate[bot]' }} 39 | run: npm audit 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.eslintcache 2 | /.vscode 3 | /build 4 | /node_modules 5 | /brave 6 | /testem.log 7 | /mocha.log 8 | -------------------------------------------------------------------------------- /Brocfile.js: -------------------------------------------------------------------------------- 1 | const buildConfig = require("./broccoli/config"); 2 | const brocfile = buildConfig.brocfile || "Brocfile." + buildConfig.platform; 3 | const platformBrocfile = require("./broccoli/" + brocfile); 4 | module.exports = platformBrocfile; 5 | -------------------------------------------------------------------------------- /Dockerfile.ci: -------------------------------------------------------------------------------- 1 | FROM node:lts-bookworm 2 | 3 | ENV DEBIAN_FRONTEND=noninteractive 4 | 5 | ENV PATH "/usr/local/ssl/bin:$PATH" 6 | 7 | RUN mkdir /app 8 | RUN chown node:node -R /app 9 | 10 | # Prevent errors when running xvfb as node user 11 | RUN mkdir /tmp/.X11-unix \ 12 | && chmod 1777 /tmp/.X11-unix \ 13 | && chown root /tmp/.X11-unix 14 | 15 | # Expose port for VNC 16 | EXPOSE 5900 17 | 18 | # Install Brave 19 | RUN wget 'https://github.com/brave/brave-browser/releases/download/v1.60.114/brave-browser_1.60.114_amd64.deb' -O /home/node/brave.deb \ 20 | && wget 'https://brave-browser-apt-release.s3.brave.com/pool/main/b/brave-keyring/brave-keyring_1.13-1.deb' -O /home/node/brave-keyring.deb \ 21 | && apt-get update --no-install-recommends \ 22 | && dpkg --install /home/node/brave-keyring.deb \ 23 | && apt-get install -y libnss3 openbox xvfb x11vnc libgbm1 libasound2 fonts-liberation xdg-utils menu libvulkan1 libu2f-udev \ 24 | && dpkg --install /home/node/brave.deb \ 25 | && rm -f /home/node/brave.deb /home/node/brave-keyring.deb 26 | 27 | USER node 28 | 29 | COPY --chown=node:node package.json /app/ 30 | COPY --chown=node:node package-lock.json /app/ 31 | COPY --chown=node:node patches/ /app/patches/ 32 | 33 | WORKDIR /app/ 34 | 35 | RUN npm ci 36 | -------------------------------------------------------------------------------- /broccoli/Brocfile.brave.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const Funnel = require("broccoli-funnel"); 6 | const MergeTrees = require("broccoli-merge-trees"); 7 | const broccoliSource = require("broccoli-source"); 8 | 9 | const WatchedDir = broccoliSource.WatchedDir; 10 | 11 | const modules = require("./modules-tree"); 12 | 13 | const specific = new WatchedDir("specific/node"); 14 | 15 | const srcTree = new MergeTrees( 16 | [specific, modules.modules, modules.bundles, modules.wasm], 17 | { overwrite: true }, 18 | ); 19 | 20 | const outputTree = new MergeTrees([srcTree], { overwrite: true }); 21 | 22 | // Output 23 | module.exports = new Funnel(outputTree, { 24 | exclude: ["**/vendor/!(react.js|react-dom.js)"], 25 | }); 26 | -------------------------------------------------------------------------------- /broccoli/Brocfile.node.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const Funnel = require("broccoli-funnel"); 6 | const MergeTrees = require("broccoli-merge-trees"); 7 | const broccoliSource = require("broccoli-source"); 8 | 9 | const WatchedDir = broccoliSource.WatchedDir; 10 | 11 | const modules = require("./modules-tree"); 12 | 13 | const specific = new WatchedDir("specific/node"); 14 | 15 | const sourceTree = modules.bundles; 16 | 17 | const assets = new MergeTrees([sourceTree, modules.static]); 18 | 19 | const srcTree = new MergeTrees( 20 | [ 21 | specific, 22 | modules.modules, 23 | modules.static, 24 | modules.bundles, 25 | new Funnel(assets, { destDir: "assets" }), 26 | ], 27 | { overwrite: true }, 28 | ); 29 | 30 | const outputTree = new MergeTrees([srcTree], { 31 | overwrite: true, 32 | }); 33 | 34 | // Output 35 | module.exports = outputTree; 36 | -------------------------------------------------------------------------------- /broccoli/Brocfile.webextension.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const Funnel = require("broccoli-funnel"); 6 | const MergeTrees = require("broccoli-merge-trees"); 7 | 8 | const specificTree = require("./specific-tree"); 9 | const modules = require("./modules-tree"); 10 | 11 | const modulesTree = new MergeTrees([ 12 | new Funnel(new MergeTrees([modules.static, modules.bundles]), { 13 | destDir: "modules", 14 | }), 15 | new Funnel(modules.wasm), 16 | ]); 17 | 18 | module.exports = new MergeTrees([modulesTree, specificTree]); 19 | -------------------------------------------------------------------------------- /broccoli/config.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const fs = require("fs"); 6 | const path = require("path"); 7 | 8 | const configFilePath = process.env.CONFIG_PATH; 9 | console.log("Configuration file:", configFilePath); 10 | 11 | const buildConfig = require(path.resolve(configFilePath)); 12 | 13 | if (!buildConfig.modules) { 14 | buildConfig.modules = fs 15 | .readdirSync(path.join(".", "modules")) 16 | .filter((dir) => 17 | fs.lstatSync(path.join(".", "modules", dir)).isDirectory(), 18 | ); 19 | } 20 | 21 | buildConfig.environment = process.env.ENVIRONMENT || "development"; 22 | buildConfig.isBeta = process.env.BETA === "True"; 23 | 24 | buildConfig.EXTENSION_VERSION = process.env.EXTENSION_VERSION; 25 | buildConfig.VERSION = process.env.VERSION; 26 | 27 | if (process.env.EXTENSION_LOG) { 28 | buildConfig.EXTENSION_LOG = process.env.EXTENSION_LOG; 29 | } 30 | 31 | module.exports = buildConfig; 32 | -------------------------------------------------------------------------------- /broccoli/env.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const env = (process.env.ENVIRONMENT || "development").toUpperCase(); 6 | 7 | module.exports = { 8 | [env]: true, 9 | INCLUDE_TESTS: process.env.INCLUDE_TESTS, 10 | SOURCE_MAPS: !(process.env.SOURCE_MAPS === "false"), 11 | DEBUG_PAGES: !(process.env.SOURCE_DEBUG === "false"), 12 | }; 13 | -------------------------------------------------------------------------------- /broccoli/modules/bundles-tree.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const Funnel = require("broccoli-funnel"); 6 | 7 | const buildConfig = require("../config"); 8 | const env = require("../env"); 9 | const SystemBuilder = require("./broccoli-webpack"); 10 | 11 | const bundleFiles = buildConfig.bundles; 12 | const prefix = "modules"; 13 | 14 | function getBundlesTree(modulesTree) { 15 | const excludeBundles = new Set(); 16 | 17 | let excludedBundleFiles; 18 | if (typeof bundleFiles === "undefined") { 19 | excludedBundleFiles = []; 20 | } else if (bundleFiles.length === 0) { 21 | excludedBundleFiles = ["**/*"]; 22 | } else { 23 | excludedBundleFiles = Array.from(excludeBundles); 24 | } 25 | 26 | const input = new Funnel(modulesTree, { 27 | destDir: prefix, 28 | exclude: excludedBundleFiles, 29 | }); 30 | 31 | const buildConfigBundler = buildConfig.bundler || {}; 32 | 33 | const builderConfig = { 34 | externals: buildConfigBundler.externals || [], 35 | globalDeps: buildConfigBundler.globalDeps || {}, 36 | sourceMaps: env.SOURCE_MAPS, 37 | lowResSourceMaps: false, 38 | sourceMapContents: env.SOURCE_MAPS, 39 | // required in case source module format is not esmb 40 | globalName: "WebDiscoveryProjectGlobal", 41 | rollup: true, 42 | }; 43 | 44 | const bundles = new SystemBuilder(input, { 45 | builderConfig: buildConfig.builderDefault || builderConfig, 46 | bundleConfigs: buildConfig.bundleConfigs || {}, 47 | }); 48 | 49 | const bundlesTree = new Funnel(bundles, { 50 | srcDir: prefix, 51 | allowEmpty: true, 52 | }); 53 | 54 | const wasmTree = new Funnel(bundles, { 55 | exclude: ["modules/**/*"], 56 | }); 57 | 58 | return { bundlesTree, wasmTree }; 59 | } 60 | 61 | module.exports = getBundlesTree; 62 | -------------------------------------------------------------------------------- /broccoli/modules/content-script-imports.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const fs = require("fs"); 6 | const writeFile = require("broccoli-file-creator"); 7 | const config = require("../config"); 8 | 9 | let fileContents = ""; 10 | if (config.modules) { 11 | config.modules.forEach((moduleName) => { 12 | if (fs.existsSync(`modules/${moduleName}/sources/content.es`)) { 13 | fileContents += `import './${moduleName}/content';\n`; 14 | } 15 | }); 16 | } 17 | 18 | module.exports = writeFile("module-content-script.es", fileContents); 19 | -------------------------------------------------------------------------------- /broccoli/modules/content-tests-imports.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const fs = require("fs"); 6 | const writeFile = require("broccoli-file-creator"); 7 | const config = require("../config"); 8 | 9 | let fileContents = ""; 10 | if (config.modules) { 11 | config.modules.forEach((moduleName) => { 12 | if (fs.existsSync(`modules/${moduleName}/tests/content-tests.es`)) { 13 | fileContents += `import './tests/${moduleName}/content-tests';\n`; 14 | } 15 | }); 16 | } 17 | 18 | module.exports = writeFile("module-content-tests.es", fileContents); 19 | -------------------------------------------------------------------------------- /broccoli/modules/dist-tree.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const path = require("path"); 6 | const Funnel = require("broccoli-funnel"); 7 | const MergeTrees = require("broccoli-merge-trees"); 8 | const broccoliSource = require("broccoli-source"); 9 | 10 | const buildConfig = require("../config"); 11 | const env = require("../env"); 12 | 13 | const subprojects = require( 14 | path.resolve(__dirname, "../../configs/common/subprojects/bundles"), 15 | ); 16 | 17 | const UnwatchedDir = broccoliSource.UnwatchedDir; 18 | 19 | module.exports = function getDistTree(modulesTree) { 20 | const exclude = ["**/dist/locale/**/*"]; // remove translations; 21 | 22 | if (!env.DEBUG_PAGES) { 23 | exclude.push("**/dist/debug/**/*"); // remove debug pages 24 | } 25 | 26 | const modulesTrees = [ 27 | new Funnel(modulesTree, { 28 | include: buildConfig.modules.map((name) => `${name}/dist/**/*`), 29 | exclude, 30 | getDestinationPath(_path) { 31 | return _path.replace("/dist", ""); 32 | }, 33 | }), 34 | ]; 35 | 36 | const suprojectsSet = new Set(); 37 | const getSubprojects = (moduleName) => { 38 | try { 39 | const { subprojects = [] } = require( 40 | path.resolve(__dirname, `../../modules/${moduleName}/build-config`), 41 | ); 42 | subprojects.forEach((project) => { 43 | suprojectsSet.add(project); 44 | }); 45 | } catch (error) { 46 | // this error is expected, because not all the modules have 'build-config.json' 47 | } 48 | }; 49 | 50 | buildConfig.modules.forEach((mod) => { 51 | getSubprojects(mod); 52 | }); 53 | 54 | buildConfig.subprojects = subprojects(Array.from(suprojectsSet)); 55 | 56 | const distTrees = modulesTrees.concat( 57 | (buildConfig.subprojects || []).map( 58 | (subproject) => 59 | new Funnel(new UnwatchedDir(subproject.src), { 60 | include: subproject.include || ["**/*"], 61 | destDir: subproject.dest, 62 | getDestinationPath(filename) { 63 | return filename 64 | .replace(".development", "") 65 | .replace(".production.min", ""); 66 | }, 67 | }), 68 | ), 69 | ); 70 | 71 | const distTree = new MergeTrees(distTrees); 72 | 73 | return distTree; 74 | }; 75 | -------------------------------------------------------------------------------- /broccoli/modules/integration-tests-imports.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const writeFile = require("broccoli-file-creator"); 6 | const glob = require("glob"); 7 | const camelCase = require("camelcase"); 8 | const config = require("../config"); 9 | 10 | function createDefaultExportName(parts) { 11 | return camelCase( 12 | parts 13 | .map((part) => part[0].toUpperCase() + part.substr(1).toLowerCase()) 14 | .join(""), 15 | ); 16 | } 17 | 18 | const imports = []; 19 | const defaults = []; 20 | 21 | if (config.modules && config.modules.indexOf("integration-tests") !== -1) { 22 | config.modules.forEach((moduleName) => { 23 | const prefix = `./modules/${moduleName}/tests/integration`; 24 | glob.sync(`${prefix}/**/*-test.es`).forEach((f) => { 25 | const filename = f.substring(prefix.length + 1, f.lastIndexOf(".")); 26 | const defaultExport = createDefaultExportName([ 27 | moduleName, 28 | ...filename.substring(0, filename.length - 5).split(/[^\w]+/g), 29 | ]); 30 | defaults.push(`${defaultExport}();`); 31 | imports.push( 32 | `import ${defaultExport} from './tests/${moduleName}/integration/${filename}';`, 33 | ); 34 | }); 35 | }); 36 | } 37 | 38 | module.exports = writeFile( 39 | "module-integration-tests.es", 40 | ` 41 | ${imports.join("\n")} 42 | 43 | TESTS.IntegrationTests = function () { 44 | describe('integration', function () { 45 | ${defaults.join("\n ")} 46 | }); 47 | }; 48 | `, 49 | ); 50 | -------------------------------------------------------------------------------- /broccoli/modules/modules-list.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const camelCase = require("camelcase"); 6 | const writeFile = require("broccoli-file-creator"); 7 | const config = require("../config"); 8 | 9 | let modulesList = ""; 10 | 11 | config.modules.forEach((module) => { 12 | const importStatement = [ 13 | "import", 14 | `${camelCase(module)}Module`, 15 | "from", 16 | `'../../${module}/background';`, 17 | ].join(" "); 18 | modulesList += importStatement; 19 | }); 20 | 21 | modulesList += "export default {"; 22 | config.modules.forEach((module, i) => { 23 | modulesList += `'${module}': ${camelCase(module)}Module`; 24 | if (i < config.modules.length - 1) { 25 | modulesList += ","; 26 | } 27 | }); 28 | modulesList += "};"; 29 | 30 | module.exports = writeFile("modules.es", modulesList); 31 | -------------------------------------------------------------------------------- /broccoli/specific-tree.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const fs = require("fs"); 6 | const path = require("path"); 7 | const Funnel = require("broccoli-funnel"); 8 | const Source = require("broccoli-source"); 9 | const MergeTrees = require("broccoli-merge-trees"); 10 | const writeFile = require("broccoli-file-creator"); 11 | const buildConfig = require("./config"); 12 | 13 | // Public key for a non-existent private key used to form the 14 | // following static extension ID for development purposes: olooapeelpgjekinbmklbmhbkijanpic 15 | const devManifestKey = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy+raRuzyIAHCSgCFRNSDRMoKWZx8YY8b4BfepG90e2vWfhPtNTZiEmtWtwSMKwD7kc/0rBQEXa2LzAfoGwFU5FrhApb0CPpw96OE8FeU9L8b9Fws938IOoMHmc75z0GcUhk+njlZhauqdji6qu7Pq8dEuz+YeCFp2tL6ax3TwAuBatKPDsFpsNivl6Y1Ll+eRV7Ss6DQLVQHBextMu03TQYAdotWvoige/9oLLz385oBiCnsxCrevbmqAJB3Hx37Cya3UX71pJJw4gEUsmdzNKRcCsJwwmt3eV0Bg1GPPGgb24HIHRKAUjyXRAbSx8cpSLlt7aKCcrX5Vp5ZMij34QIDAQAB"; 16 | 17 | function getSpecificsTree() { 18 | const specificTree = new Funnel( 19 | new Source.WatchedDir(`specific/${buildConfig.specific}`), 20 | { 21 | exclude: ["**/locale"], 22 | } 23 | ); 24 | 25 | if (buildConfig.environment === "development") { 26 | const manifestPath = path.join("specific", buildConfig.specific, "manifest.json"); 27 | 28 | if (fs.existsSync(manifestPath)) { 29 | const rawContent = fs.readFileSync(manifestPath, 'utf8'); 30 | const manifestContent = JSON.parse(rawContent); 31 | 32 | manifestContent.key = devManifestKey; 33 | 34 | const manifestFile = writeFile( 35 | "manifest.json", 36 | JSON.stringify(manifestContent, null, 2) 37 | ); 38 | 39 | return new MergeTrees([specificTree, manifestFile], { 40 | overwrite: true, 41 | }); 42 | } 43 | } 44 | return specificTree; 45 | } 46 | 47 | module.exports = getSpecificsTree(); 48 | -------------------------------------------------------------------------------- /configs/ci/integration-tests.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const urls = require("../common/urls"); 6 | 7 | module.exports = { 8 | platform: "webextension", 9 | specific: "web-discovery-project", 10 | baseURL: "/modules/", 11 | testsBasePath: "./build/modules", 12 | settings: { 13 | ...urls("sandbox"), 14 | channel: "99", 15 | ALLOWED_COUNTRY_CODES: [ 16 | "de", 17 | "at", 18 | "ch", 19 | "es", 20 | "us", 21 | "fr", 22 | "nl", 23 | "gb", 24 | "it", 25 | "se", 26 | ], 27 | WDP_CHANNEL: "test", 28 | }, 29 | default_prefs: {}, 30 | modules: [ 31 | "content-script-tests", 32 | "core", 33 | "hpnv2", 34 | "web-discovery-project", 35 | "integration-tests", 36 | "webextension-specific", 37 | "webrequest-pipeline", 38 | ], 39 | bundles: [ 40 | "core/content-script.bundle.js", 41 | "core/content-tests.bundle.js", 42 | "hpnv2/worker.asmjs.bundle.js", 43 | "hpnv2/worker.wasm.bundle.js", 44 | "integration-tests/run.bundle.js", 45 | "webextension-specific/app.bundle.js", 46 | ], 47 | builderDefault: { 48 | globalDeps: { 49 | chai: "chai", 50 | }, 51 | }, 52 | }; 53 | -------------------------------------------------------------------------------- /configs/ci/unit-tests.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const urls = require("../common/urls"); 6 | 7 | module.exports = { 8 | platform: "webextension", 9 | format: "system", 10 | brocfile: "Brocfile.node.js", 11 | baseURL: "/", 12 | testsBasePath: "./build/", 13 | testem_launchers: ["unit-node"], 14 | testem_launchers_ci: ["unit-node"], 15 | settings: { 16 | ...urls("brave.com"), 17 | id: "brave@brave.com", 18 | name: "Brave", 19 | }, 20 | default_prefs: {}, 21 | bundles: [], 22 | }; 23 | -------------------------------------------------------------------------------- /configs/common/subprojects/bundles.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const subprojects = { 6 | // Define bundles 7 | chai: { 8 | src: "node_modules/chai", 9 | include: ["chai.js"], 10 | dest: "vendor", 11 | }, 12 | "chai-dom": { 13 | src: "node_modules/chai-dom", 14 | include: ["chai-dom.js"], 15 | dest: "vendor", 16 | }, 17 | mocha: { 18 | src: "node_modules/mocha", 19 | include: ["mocha.css", "mocha.js"], 20 | dest: "vendor", 21 | }, 22 | sinon: { 23 | src: "node_modules/sinon/pkg", 24 | include: ["sinon.js"], 25 | dest: "vendor", 26 | }, 27 | "sinon-chai": { 28 | src: "node_modules/sinon-chai/lib", 29 | include: ["sinon-chai.js"], 30 | dest: "vendor", 31 | }, 32 | }; 33 | 34 | module.exports = (modules) => { 35 | const result = []; 36 | modules.forEach((m) => { 37 | if (subprojects[m] === undefined) { 38 | throw new Error(`Could not find subproject: ${m}`); 39 | } 40 | result.push(subprojects[m]); 41 | }); 42 | return result; 43 | }; 44 | -------------------------------------------------------------------------------- /configs/common/urls.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* please keep keys in this object sorted */ 6 | module.exports = (env = "sandbox") => { 7 | if (env === "sandbox") { 8 | return { 9 | ENDPOINT_HPNV2_ANONYMOUS: "https://collector.wdp.brave.software", // hpnv2/sources/endpoints.es 10 | ENDPOINT_HPNV2_DIRECT: "https://collector.wdp.brave.software", // hpnv2/sources/endpoints.es 11 | ENDPOINT_PATTERNS: "https://patterns.hpn.brave.software/patterns.gz", 12 | ENDPOINT_SAFE_QUORUM_PROVIDER: "https://safe-browsing-quorum.hpn.brave.software/config", 13 | ENDPOINT_STAR: "https://star.wdp.brave.software/", 14 | FETCHER_GATEWAY: "https://fg.search.brave.com", 15 | }; 16 | } 17 | 18 | if (env === "production") { 19 | return { 20 | ENDPOINT_HPNV2_ANONYMOUS: "https://collector.wdp.brave.com", // hpnv2/sources/endpoints.es 21 | ENDPOINT_HPNV2_DIRECT: "https://collector.wdp.brave.com", // hpnv2/sources/endpoints.es 22 | ENDPOINT_PATTERNS: "https://patterns.wdp.brave.com/patterns.gz", 23 | ENDPOINT_SAFE_QUORUM_PROVIDER: "https://quorum.wdp.brave.com/config", 24 | ENDPOINT_STAR: "https://star.wdp.brave.com/", 25 | FETCHER_GATEWAY: "https://fg.search.brave.com", 26 | }; 27 | } 28 | 29 | if (env === "local") { 30 | return { 31 | ENDPOINT_HPNV2_ANONYMOUS: "http://127.0.0.1:3001", // hpnv2/sources/endpoints.es 32 | ENDPOINT_HPNV2_DIRECT: "http://127.0.0.1:3001", // hpnv2/sources/endpoints.es 33 | ENDPOINT_PATTERNS: "http://127.0.0.1:8000/hw-patterns.gz", 34 | ENDPOINT_SAFE_QUORUM_PROVIDER: "http://127.0.0.1:7100/config", 35 | FETCHER_GATEWAY: "http://localhost:8083", 36 | }; 37 | } 38 | }; 39 | -------------------------------------------------------------------------------- /configs/extension.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const urls = require("./common/urls"); 6 | 7 | module.exports = { 8 | specific: "web-discovery-project", 9 | platform: "webextension", 10 | brocfile: "Brocfile.webextension.js", 11 | baseURL: "/modules/", 12 | pack: "web-ext build --source-dir build --artifacts-dir .", 13 | sourceMaps: false, 14 | format: "module", 15 | settings: { 16 | ...urls(), 17 | WDP_CHANNEL: "brave", 18 | WDP_PATTERNS_SIGNING: true, 19 | WDP_ENV: "production", 20 | ALLOWED_COUNTRY_CODES: [ 21 | "de", 22 | "at", 23 | "ch", 24 | "es", 25 | "us", 26 | "fr", 27 | "nl", 28 | "gb", 29 | "it", 30 | "be", 31 | "se", 32 | "dk", 33 | "fi", 34 | "cz", 35 | "gr", 36 | "hu", 37 | "ro", 38 | "no", 39 | "ca", 40 | "au", 41 | "ru", 42 | "ua", 43 | "in", 44 | "pl", 45 | "jp", 46 | "br", 47 | "mx", 48 | "cn", 49 | "ar", 50 | ], 51 | }, 52 | default_prefs: { 53 | "modules.web-discovery-project.enabled": true, 54 | "modules.hpnv2.enabled": true, 55 | "modules.fetcher.enabled": false, 56 | }, 57 | bundles: [ 58 | "core/content-script.bundle.js", 59 | "hpnv2/worker.wasm.bundle.js", 60 | "hpnv2/worker.asmjs.bundle.js", 61 | "webextension-specific/app.bundle.js", 62 | ], 63 | modules: [ 64 | "core", 65 | "web-discovery-project", 66 | "hpnv2", 67 | "fetcher", 68 | "webrequest-pipeline", 69 | "webextension-specific", 70 | ], 71 | }; 72 | -------------------------------------------------------------------------------- /configs/module.js: -------------------------------------------------------------------------------- 1 | const base = require("./extension"); 2 | const urls = require("./common/urls"); 3 | 4 | module.exports = { 5 | ...base, 6 | specific: "node", 7 | format: "common", 8 | brocfile: "Brocfile.brave.js", 9 | pack: "npm pack", 10 | settings: { 11 | ...base.settings, 12 | ...urls("production"), 13 | WDP_PATTERNS_SIGNING: true, 14 | WDP_ENV: "production", 15 | }, 16 | }; 17 | -------------------------------------------------------------------------------- /configs/production.js: -------------------------------------------------------------------------------- 1 | const base = require("./extension"); 2 | const urls = require("./common/urls"); 3 | 4 | module.exports = { 5 | ...base, 6 | settings: { 7 | ...base.settings, 8 | ...urls("production"), 9 | WDP_PATTERNS_SIGNING: true, 10 | WDP_ENV: "production", 11 | }, 12 | }; 13 | -------------------------------------------------------------------------------- /configs/sandbox.js: -------------------------------------------------------------------------------- 1 | const base = require("./extension"); 2 | const urls = require("./common/urls"); 3 | 4 | module.exports = { 5 | ...base, 6 | settings: { 7 | ...base.settings, 8 | ...urls("sandbox"), 9 | WDP_PATTERNS_SIGNING: true, 10 | WDP_ENV: "sandbox", 11 | }, 12 | default_prefs: { 13 | ...base.default_prefs, 14 | "logger.hpnv2.level": "debug", 15 | "logger.web-discovery-project.level": "debug", 16 | }, 17 | }; 18 | -------------------------------------------------------------------------------- /fern.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const program = require("commander"); 4 | const colors = require("colors"); 5 | 6 | // register sub-commands 7 | require("./fern/commands/build")(program); 8 | require("./fern/commands/lint")(program); 9 | require("./fern/commands/pack")(program); 10 | require("./fern/commands/serve")(program); 11 | require("./fern/commands/test")(program); 12 | 13 | (() => { 14 | colors.setTheme({ 15 | silly: "rainbow", 16 | input: "grey", 17 | verbose: "cyan", 18 | prompt: "grey", 19 | info: "green", 20 | data: "grey", 21 | help: "cyan", 22 | warn: "yellow", 23 | debug: "blue", 24 | error: "red", 25 | }); 26 | 27 | program.parse(process.argv); 28 | })(); 29 | -------------------------------------------------------------------------------- /fern/commands/lint.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | function checkIfValidOption(option, validOptions) { 6 | if (!validOptions.includes(option)) { 7 | throw new Error("Wrong / non-existent option!"); 8 | } 9 | } 10 | 11 | module.exports = (program) => { 12 | program 13 | .command("lint") 14 | .option("--fix", "fix style errors if possible") 15 | .option("--tests-only", "check only test files") 16 | .option("--sources-only", "check only source files") 17 | .option("--platform-only", 'check only "platforms/**" files') 18 | .option("-f --filetype ", 'specify file extension ("es", "jsx")') 19 | .option("-m --module ", "specify module folder") 20 | .action((options) => { 21 | const { CLIEngine } = require("eslint"); 22 | const cli = new CLIEngine({ 23 | cache: true, 24 | fix: options.fix, 25 | }); 26 | const formatter = cli.getFormatter(); 27 | 28 | const module = options.module ? `${options.module}/**` : "**"; 29 | let validSources = ["sources", "tests"]; 30 | let validFileTypes = ["es", "jsx", "js", "ts"]; 31 | 32 | if (options.testsOnly) { 33 | validSources = ["tests"]; 34 | } else if (options.sourcesOnly) { 35 | validSources = ["sources"]; 36 | } 37 | 38 | if (options.filetype) { 39 | checkIfValidOption(options.filetype, validFileTypes); 40 | validFileTypes = [options.filetype]; 41 | } 42 | 43 | let modulesPath = 44 | validSources.length === 1 45 | ? `modules/${module}/${validSources}/**/*.` 46 | : `modules/${module}/{${validSources.join(",")}}/**/*.`; 47 | 48 | modulesPath = 49 | validFileTypes.length === 1 50 | ? `${modulesPath}${validFileTypes}` 51 | : `${modulesPath}{${validFileTypes.join(",")}}`; 52 | 53 | const platformsPath = 54 | validFileTypes.length === 1 55 | ? `platforms/**/*.${validFileTypes}` 56 | : `platforms/**/*.{${validFileTypes.join(",")}}`; 57 | 58 | let pathsToLint = [modulesPath, platformsPath]; 59 | 60 | if (options.testsOnly || options.sourcesOnly) { 61 | pathsToLint = [modulesPath]; 62 | } else if (options.platformOnly) { 63 | pathsToLint = [platformsPath]; 64 | } 65 | 66 | console.log("Linting the following paths: ", pathsToLint); 67 | const report = cli.executeOnFiles(pathsToLint); 68 | 69 | if (options.fix) { 70 | CLIEngine.outputFixes(report); 71 | } 72 | 73 | const logs = formatter(report.results); 74 | 75 | if (logs.replace(/\s/g, "").length === 0) { 76 | console.log("No errors were found!"); 77 | } else { 78 | console.log(logs); 79 | } 80 | }); 81 | }; -------------------------------------------------------------------------------- /fern/commands/pack.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const { execSync } = require("child_process"); 6 | const { join } = require("path"); 7 | 8 | const { 9 | setConfigPath, 10 | getExtensionVersion, 11 | configParameter, 12 | } = require("../common"); 13 | 14 | function run(command) { 15 | // Make sure that binaries from `node_modules` are available in PATH. 16 | let PATH = join(__dirname, "..", "..", "node_modules", ".bin"); 17 | if (process.env.PATH) { 18 | PATH += `:${process.env.PATH}`; 19 | } 20 | 21 | return execSync(command, { 22 | encoding: "utf-8", 23 | env: { 24 | ...process.env, 25 | PATH, 26 | }, 27 | }).trim(); 28 | } 29 | 30 | module.exports = (program) => { 31 | program.command(`pack ${configParameter}`).action((configPath) => { 32 | const cfg = setConfigPath(configPath); 33 | const CONFIG = cfg.CONFIG; 34 | 35 | getExtensionVersion("package", CONFIG) 36 | .then((version) => { 37 | process.env.PACKAGE_VERSION = version; 38 | process.env.EXTENSION_VERSION = version; 39 | 40 | if (!process.env.VERSION) { 41 | process.env.VERSION = version; 42 | } 43 | 44 | if (!CONFIG.pack) { 45 | throw new Error("Pack not defined in config file"); 46 | } 47 | 48 | console.log(run(`bash -c "${CONFIG.pack}"`)); 49 | }) 50 | .catch((e) => { 51 | console.error("Something went wrong", e); 52 | process.exit(1); 53 | }); 54 | }); 55 | }; -------------------------------------------------------------------------------- /modules/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/.gitkeep -------------------------------------------------------------------------------- /modules/content-script-tests/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "subprojects": ["chai", "chai-dom", "mocha", "sinon", "sinon-chai"] 3 | } 4 | -------------------------------------------------------------------------------- /modules/content-script-tests/sources/background.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import background from "../core/base/background"; 6 | 7 | export default background({ 8 | init() { 9 | this.listener = null; 10 | }, 11 | 12 | unload() {}, 13 | 14 | getState() { 15 | return { 16 | a: 42, 17 | test: true, 18 | }; 19 | }, 20 | 21 | actions: { 22 | getSomeValue(...args) { 23 | args.pop(); // ignore `sender` 24 | return args; 25 | }, 26 | 27 | contentScriptRan(state) { 28 | if (this.listener) { 29 | this.listener(state); 30 | } 31 | return true; 32 | }, 33 | }, 34 | }); 35 | -------------------------------------------------------------------------------- /modules/content-script-tests/sources/content.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { registerContentScript } from "../core/content/register"; 6 | 7 | registerContentScript({ 8 | module: "content-script-tests", 9 | matches: ["http://example.com/"], 10 | excludeMatches: ["http://example.com/*?foo=42"], 11 | // TODO - check all_frames 12 | // TODO - check match_about_blank 13 | js: [ 14 | (window, _, WDP) => { 15 | const testModule = WDP.app.modules["content-script-tests"]; 16 | testModule.action("contentScriptRan", testModule.state); 17 | 18 | return { 19 | action1: (...args) => 20 | WDP.app.modules["content-script-tests"].action( 21 | "getSomeValue", 22 | ...args, 23 | ), 24 | }; 25 | }, 26 | ], 27 | }); 28 | -------------------------------------------------------------------------------- /modules/content-script-tests/tests/integration/messaging-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { expect, sleep, Events } from "../../../tests/core/integration/helpers"; 6 | 7 | export default function () { 8 | describe.skip("content-script", () => { 9 | let mdSubscription = null; 10 | 11 | afterEach(() => { 12 | mdSubscription.unsubscribe(); 13 | }); 14 | 15 | function onMousedown(callback) { 16 | mdSubscription = Events.subscribe("core:mouse-down", () => { 17 | callback(); 18 | }); 19 | } 20 | 21 | function triggerMousedown() { 22 | document.body.dispatchEvent( 23 | new MouseEvent("mousedown", { 24 | bubbles: true, 25 | }), 26 | ); 27 | } 28 | 29 | it("message can be received", (done) => { 30 | onMousedown(() => { 31 | done(); 32 | }); 33 | triggerMousedown(); 34 | }); 35 | 36 | it("single message should not arrive more than once after extension restart", (done) => { 37 | let messageCount = 0; 38 | onMousedown(() => { 39 | messageCount += 1; 40 | }); 41 | 42 | sleep(500).then(() => { 43 | try { 44 | expect(messageCount).to.equal(1); 45 | done(); 46 | } catch (e) { 47 | done(e); 48 | } 49 | }); 50 | 51 | triggerMousedown(); 52 | }); 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /modules/core/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "subprojects": [] 3 | } 4 | -------------------------------------------------------------------------------- /modules/core/sources/app/module-errors.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export class ModuleMissingError extends Error { 6 | constructor(moduleName) { 7 | super(`module '${moduleName}' is missing`); 8 | this.name = "ModuleMissingError"; 9 | } 10 | } 11 | 12 | export class ModuleDisabledError extends Error { 13 | constructor(moduleName) { 14 | super(`module '${moduleName}' is disabled`); 15 | this.name = "ModuleDisabledError"; 16 | } 17 | } 18 | 19 | export class ActionMissingError extends Error { 20 | constructor(moduleName, actionName) { 21 | super(`action '${actionName}' is not defined on module '${moduleName}'`); 22 | this.name = "ActionMissingError"; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /modules/core/sources/app/module.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import inject from "../kord/inject"; 6 | 7 | import modules from "./modules"; 8 | import Service from "./service"; 9 | import LifeCycle from "./life-cycle"; 10 | import Logger from "../logger"; 11 | 12 | export { lifecycleEvents } from "./life-cycle"; 13 | 14 | export default class Module extends LifeCycle { 15 | constructor(name) { 16 | super( 17 | name, 18 | Logger.get("life-cycle", { 19 | level: "log", 20 | prefix: `[lifecycle:${name}]`, 21 | }), 22 | ); 23 | 24 | this._init = (...args) => this.background.init(...args); 25 | this._unload = () => this.background.unload(); 26 | } 27 | 28 | get providedServices() { 29 | if (this._services) { 30 | return this._services; 31 | } 32 | 33 | this._services = Object.create(null); 34 | 35 | Object.keys(this.background.providesServices || {}).forEach( 36 | (serviceName) => { 37 | const initializer = this.background.providesServices[serviceName]; 38 | this._services[serviceName] = new Service(initializer); 39 | }, 40 | ); 41 | 42 | return this._services; 43 | } 44 | 45 | get requiredServices() { 46 | return this.background.requiresServices || []; 47 | } 48 | 49 | get _module() { 50 | return modules[this.name] || {}; 51 | } 52 | 53 | get background() { 54 | return this._module; 55 | } 56 | 57 | status() { 58 | return { 59 | name: this.name, 60 | isEnabled: this.isEnabled || this.isEnabling, 61 | loadingTime: this.loadingTime, 62 | loadingTimeSync: this.loadingTimeSync, 63 | state: 64 | this.isEnabled && 65 | this.background.getState && 66 | this.background.getState(), 67 | }; 68 | } 69 | 70 | action(name, ...args) { 71 | return inject.module(this.name).action(name, ...args); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /modules/core/sources/app/service.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import Defer from "../helpers/defer"; 6 | 7 | export default class Service { 8 | constructor(initializer) { 9 | this._initializer = initializer; 10 | } 11 | 12 | /* 13 | * Service is initialized only once 14 | * Multiple calls to init return same promise 15 | */ 16 | init(app, browser) { 17 | if (this._readyDefer) { 18 | return this._readyDefer.promise; 19 | } 20 | 21 | this._readyDefer = new Defer(); 22 | const startLoading = Date.now(); 23 | 24 | // wrap in promise to catch exceptions 25 | Promise.resolve() 26 | .then(() => this._initializer(app, browser)) 27 | .then( 28 | (service) => { 29 | this.loadingTime = Date.now() - startLoading; 30 | this._readyDefer.resolve(); 31 | 32 | this.api = service; 33 | this._moduleFactory = this._initializer.moduleFactory; 34 | }, 35 | (e) => { 36 | this._readyDefer.reject(e); 37 | }, 38 | ); 39 | 40 | return this._readyDefer.promise; 41 | } 42 | 43 | async moduleFactory(moduleName) { 44 | if (!this._moduleFactory) { 45 | return this.api; 46 | } 47 | return this._moduleFactory(moduleName); 48 | } 49 | 50 | isReady() { 51 | return this._readyDefer.promise; 52 | } 53 | 54 | unload() { 55 | // if was initialised (_readyDefer exists), and has an unload method on the initializer 56 | if (this._readyDefer && this._initializer.unload) { 57 | this._initializer.unload(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /modules/core/sources/array-buffer-utils.ts: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export function isUint8ArrayEqual(x: Uint8Array, y: Uint8Array) { 6 | if (x.length !== y.length) { 7 | return false; 8 | } 9 | for (let i = 0; i < x.length; i += 1) { 10 | if (x[i] !== y[i]) { 11 | return false; 12 | } 13 | } 14 | return true; 15 | } 16 | -------------------------------------------------------------------------------- /modules/core/sources/base/background.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* eslint no-param-reassign: 'off' */ 6 | 7 | import events from "../events"; 8 | 9 | // source: https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/assign 10 | function completeAssign(target, ...sources) { 11 | sources.forEach((source) => { 12 | const descriptors = Object.keys(source).reduce((_descriptors, key) => { 13 | _descriptors[key] = Object.getOwnPropertyDescriptor(source, key); 14 | return _descriptors; 15 | }, {}); 16 | // by default, Object.assign copies enumerable Symbols too 17 | if (typeof Symbol !== "undefined") { 18 | Object.getOwnPropertySymbols(source).forEach((sym) => { 19 | const descriptor = Object.getOwnPropertyDescriptor(source, sym); 20 | if (descriptor.enumerable) { 21 | descriptors[sym] = descriptor; 22 | } 23 | }); 24 | } 25 | Object.defineProperties(target, descriptors); 26 | }); 27 | return target; 28 | } 29 | 30 | export default function (originalBackground) { 31 | const background = completeAssign({}, originalBackground); 32 | const bgInit = background.init; 33 | const bgUnload = background.unload; 34 | const bgEvents = background.events; 35 | 36 | // bind actions to background object 37 | Object.keys(background.actions || {}).forEach((action) => { 38 | background.actions[action] = background.actions[action].bind(background); 39 | }); 40 | 41 | background.init = function init(...args) { 42 | const promise = Promise.resolve(bgInit.apply(background, args)); 43 | 44 | Object.keys(bgEvents || {}).forEach((event) => { 45 | bgEvents[event] = bgEvents[event].bind(background); 46 | events.sub(event, bgEvents[event]); 47 | }); 48 | return promise; 49 | }; 50 | 51 | background.unload = function unload(...args) { 52 | Object.keys(bgEvents || {}).forEach((event) => { 53 | events.un_sub(event, bgEvents[event]); 54 | }); 55 | 56 | bgUnload.apply(background, args); 57 | }; 58 | 59 | return background; 60 | } 61 | -------------------------------------------------------------------------------- /modules/core/sources/browser.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export * from "../platform/browser"; 6 | -------------------------------------------------------------------------------- /modules/core/sources/console.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import console from "../platform/console"; 6 | import prefs from "./prefs"; 7 | import config from "./config"; 8 | 9 | function noop() {} 10 | 11 | export function isLoggingEnabled() { 12 | try { 13 | // Detect dev flag on react-native 14 | if (typeof global !== "undefined" && global.__DEV__ === true) { 15 | return true; 16 | } 17 | 18 | // Extension built in development mode 19 | if (config.environment === "development") { 20 | return true; 21 | } 22 | 23 | // 'developer' pref set 24 | if (prefs.get("developer", false)) { 25 | return true; 26 | } 27 | 28 | // Fall-back to value of 'showConsoleLogs' pref 29 | return prefs.get("showConsoleLogs", false); 30 | } catch (ee) { 31 | return false; 32 | } 33 | } 34 | 35 | const _console = {}; 36 | 37 | export function enable() { 38 | _console.debug = console.log.bind(console, "[wdp] [debug]"); 39 | _console.log = console.log.bind(console, "[wdp]"); 40 | _console.error = console.error.bind(console, "[wdp] [error]"); 41 | _console.warn = console.warn.bind(console, "[wdp] [warning]"); 42 | _console.warning = _console.warn; 43 | 44 | _console.time = (console.time || noop).bind(console); 45 | _console.timeEnd = (console.timeEnd || noop).bind(console); 46 | } 47 | 48 | export function disable() { 49 | _console.debug = noop; 50 | _console.log = noop; 51 | _console.warn = noop; 52 | _console.warning = noop; 53 | _console.error = noop; 54 | 55 | _console.time = noop; 56 | _console.timeEnd = noop; 57 | } 58 | 59 | if (isLoggingEnabled()) { 60 | enable(); 61 | } else { 62 | disable(); 63 | } 64 | 65 | export default _console; 66 | -------------------------------------------------------------------------------- /modules/core/sources/content-script.bundle.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import "./content-script"; 6 | -------------------------------------------------------------------------------- /modules/core/sources/content-tests.bundle.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import "../module-content-tests"; 6 | -------------------------------------------------------------------------------- /modules/core/sources/content/console.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { window } from "../../platform/content/globals"; 6 | import config from "../config"; 7 | 8 | function noop() {} 9 | 10 | /** 11 | * In content context we only enable logging in developement build to not 12 | * pollute logs from visited web pages. 13 | */ 14 | export default config.environment === "development" 15 | ? window.console 16 | : new Proxy( 17 | {}, 18 | { 19 | get() { 20 | return noop; 21 | }, 22 | }, 23 | ); 24 | -------------------------------------------------------------------------------- /modules/core/sources/content/helpers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /** 6 | * Check if `window` is the top-level window. 7 | */ 8 | export function isTopWindow(window) { 9 | return window.self === window.top; 10 | } 11 | 12 | /** 13 | * Return a `Promise` which resolves once `window.document.body` exists. If it's 14 | * already the case when the function is invoked, then the promise resolves 15 | * immediately, otherwise it waits for the `DOMContentLoaded` event to trigger. 16 | */ 17 | export function documentBodyReady() { 18 | if (window.document && window.document.body) { 19 | return Promise.resolve(); 20 | } 21 | 22 | return new Promise((resolve) => { 23 | window.addEventListener("DOMContentLoaded", resolve, { once: true }); 24 | }); 25 | } 26 | -------------------------------------------------------------------------------- /modules/core/sources/content/i18n.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { chrome } from "../../platform/content/globals"; 6 | 7 | export default { 8 | getMessage(key, ...subs) { 9 | let substitutions = []; 10 | // Firefox accepts number as substitutions but chrome does not 11 | if (Array.isArray(subs[0])) { 12 | substitutions = subs[0].map(String); 13 | } else { 14 | substitutions = subs.map(String); 15 | } 16 | return chrome.i18n.getMessage(key, substitutions) || key; 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /modules/core/sources/content/ready-promise.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { waitFor } from "../helpers/wait"; 6 | import runtime from "../../platform/runtime"; 7 | 8 | const isChromeReady = async () => { 9 | const isWebextension = chrome.extension; 10 | 11 | // on firefox platform, we wait for chrome object as the extension is ready at that time 12 | if (!isWebextension) { 13 | if (typeof chrome !== "object") { 14 | throw new Error("chrome object not there yet"); 15 | } 16 | 17 | // if non extension chrome is there, we are most likely in content tests 18 | return true; 19 | } 20 | 21 | return new Promise((resolve, reject) => { 22 | runtime 23 | .sendMessage({ name: "appReady" }) 24 | .then(({ ready }) => { 25 | if (ready) { 26 | resolve(true); 27 | } else { 28 | reject(); 29 | } 30 | }) 31 | .catch(reject); 32 | setTimeout(reject, 300); 33 | }); 34 | }; 35 | 36 | export default function checkIfChromeReady() { 37 | return waitFor(isChromeReady).catch((e) => { 38 | window.console.error("failed to access background page", e); 39 | throw e; 40 | }); 41 | } 42 | -------------------------------------------------------------------------------- /modules/core/sources/crypto/random.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* eslint-disable no-bitwise */ 6 | 7 | import crypto from "../../platform/crypto"; 8 | 9 | // This function generates better numbers than Math.random but *should not* be 10 | // used for cryptographic operations. This is only for code that requires a 11 | // small amount of randomness. 12 | // 13 | // Doing the same as Firefox Math.random does, but with a crypto secure 64 bit number instead. 14 | // The equivalent in C++ is: double(uint64val & 0x1FFFFFFFFFFFFF) / (1 << 53); 15 | // WARNING: In tests (Linux), considerably slower than Math.random (5-10 times) 16 | export default function random() { 17 | const values = crypto.getRandomValues(new Uint32Array(2)); 18 | return (2 ** 32 * (values[0] & 0x1fffff) + values[1]) / 2 ** 53; 19 | } 20 | 21 | export function randomInt() { 22 | return Math.floor(random() * Number.MAX_SAFE_INTEGER); 23 | } 24 | -------------------------------------------------------------------------------- /modules/core/sources/decorators.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import sleep from "./helpers/sleep"; 6 | import { clearTimeout, setTimeout } from "../core/timers"; 7 | 8 | export function throttle(window, fn, threshhold) { 9 | let last; 10 | let timer; 11 | return (...args) => { 12 | const now = Date.now(); 13 | if (last && now < last + threshhold) { 14 | // reset timeout 15 | window.clearTimeout(timer); 16 | timer = window.setTimeout(() => { 17 | last = now; 18 | fn(...args); 19 | }, threshhold); 20 | } else { 21 | last = now; 22 | fn(...args); 23 | } 24 | }; 25 | } 26 | 27 | /** 28 | * simple version of https://davidwalsh.name/javascript-debounce-function, 29 | * without options to cancel or execute immediately. 30 | * 31 | * @param {Function} fn 32 | * @param {number} [delay] in ms 33 | * @returns {Function} the debounced function, 34 | * or the given function when delay is falsy. 35 | */ 36 | export function debounce(fn, delay) { 37 | let timeout = null; 38 | return !delay 39 | ? fn 40 | : function debounced(...args) { 41 | // https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/clearTimeout 42 | // Passing an invalid ID to clearTimeout() silently does nothing; no exception is thrown. 43 | clearTimeout(timeout); 44 | 45 | const delayed = () => { 46 | timeout = null; 47 | fn(...args); 48 | }; 49 | timeout = setTimeout(delayed, delay); 50 | }; 51 | } 52 | 53 | export function deadline(promise, timeout) { 54 | return Promise.race([sleep(timeout), promise]); 55 | } 56 | 57 | export function nextTick(fn, ...args) { 58 | return Promise.resolve().then(() => fn(...args)); 59 | } 60 | 61 | export function nextIdle() { 62 | if (typeof window === "undefined" || !window.requestIdleCallback) { 63 | return nextTick(() => {}); 64 | } 65 | 66 | return new Promise((resolve) => { 67 | window.requestIdleCallback(resolve); 68 | }); 69 | } 70 | 71 | export async function withTimeout(promise, timeoutInMs) { 72 | let timer; 73 | try { 74 | timer = new Promise((_, reject) => { 75 | setTimeout( 76 | () => 77 | reject(new Error(`Timed out after ${timeoutInMs / 1000} seconds`)), 78 | timeoutInMs, 79 | ); 80 | }); 81 | return await Promise.race([promise, timer]); 82 | } finally { 83 | clearTimeout(timer); 84 | } 85 | } 86 | 87 | // Helper if you only want to print the result of a promise. 88 | export async function tryPromise(promise, timeoutInMs = 1000) { 89 | try { 90 | return await withTimeout(promise, timeoutInMs); 91 | } catch (e) { 92 | return `Failed: ${e}`; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /modules/core/sources/globals-window.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import window from "../platform/globals-window"; 6 | 7 | export default window; 8 | -------------------------------------------------------------------------------- /modules/core/sources/gzip.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import * as gzip from "../platform/gzip"; 6 | 7 | /** 8 | * Compress a string 9 | * 10 | * @param {string} string to compress 11 | * @returns {UInt8Array} compressed data 12 | */ 13 | export const compress = gzip.compress || false; 14 | 15 | /** 16 | * Decompress a Gzip compressed string 17 | * 18 | * @param {UInt8Array} gzipped data 19 | * @returns {string} decompressed string 20 | */ 21 | export const decompress = gzip.decompress || false; 22 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/default-map.ts: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /** 6 | * Equivalent to Python's default dict, but in Javascript with a Map! 7 | * It behaves exactly like a map, but allows you to specify a callback to be 8 | * used when a `key` does not exist in the Map yet. 9 | * 10 | * >>> const myMap = new DefaultMap(() => []) 11 | * >>> myMap.get('foo') 12 | * [] 13 | * >> myMap.update('bar', v => v.push(42)) 14 | * >> myMap 15 | * DefaultMap { 'foo' => [], 'bar' => [ 42 ] } 16 | */ 17 | export default class DefaultMap { 18 | private map: Map; 19 | private valueCtr: () => V; 20 | 21 | constructor(valueCtr: () => V, ...args: any[]) { 22 | this.map = new Map(...args); 23 | this.valueCtr = valueCtr; 24 | } 25 | 26 | toMap() { 27 | return this.map; 28 | } 29 | 30 | toObj() { 31 | const obj = Object.create(null); 32 | this.forEach((v: V, k: K) => { 33 | obj[k] = v; 34 | }); 35 | return obj; 36 | } 37 | 38 | get size() { 39 | return this.map.size; 40 | } 41 | 42 | clear() { 43 | return this.map.clear(); 44 | } 45 | 46 | delete(key: K): boolean { 47 | return this.map.delete(key); 48 | } 49 | 50 | entries() { 51 | return this.map.entries(); 52 | } 53 | 54 | forEach(cb: (value: V, key: K, map: Map) => void, thisArg?: any): void { 55 | return this.map.forEach(cb, thisArg); 56 | } 57 | 58 | get(key: K): V { 59 | let value: V | undefined = this.map.get(key); 60 | 61 | if (value === undefined) { 62 | value = this.valueCtr(); 63 | this.set(key, value); 64 | } 65 | 66 | return value; 67 | } 68 | 69 | has(key: K): boolean { 70 | return this.map.has(key); 71 | } 72 | 73 | keys() { 74 | return this.map.keys(); 75 | } 76 | 77 | set(key: K, value: V) { 78 | this.map.set(key, value); 79 | return this; 80 | } 81 | 82 | values() { 83 | return this.map.values(); 84 | } 85 | 86 | // Extra API 87 | 88 | update(key: K, updateFn: (v: V) => V | undefined): void { 89 | const value = this.get(key); 90 | const result = updateFn(value); 91 | this.set(key, result === undefined ? value : result); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/defer.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default class Defer { 6 | static PENDING = Symbol("PENDING"); 7 | 8 | static RESOLVED = Symbol("RESOLVED"); 9 | 10 | static REJECTED = Symbol("REJECTED"); 11 | 12 | constructor() { 13 | this.promise = new Promise((resolve, reject) => { 14 | this.resolve = (...args) => { 15 | this.state = Defer.RESOLVED; 16 | return resolve(...args); 17 | }; 18 | this.reject = (...args) => { 19 | this.state = Defer.REJECTED; 20 | return reject(...args); // eslint-disable-line prefer-promise-reject-errors 21 | }; 22 | this.state = Defer.PENDING; 23 | }); 24 | 25 | this.promise 26 | // This will silence error logs in case the defered promise 27 | // gets rejected before it is consumed 28 | .catch(() => {}); 29 | } 30 | 31 | get isSettled() { 32 | return this.state !== Defer.PENDING; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/fixed-size-cache.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import LRU from "../LRU"; 6 | /* Fixed length lookup cache. Allows expensive operations to be cached for later lookup. Once 7 | * the cache limit is exceeded, least recently used values are removed. 8 | */ 9 | export default class FixedSizeCache { 10 | /* @param {function} buildValue - used to build a new value from key in case of cache miss. 11 | * @param {number} size - maximum elements stored in cache. 12 | * @param {function} buildKey - [Optional] used to extract key from argument. 13 | */ 14 | constructor(buildValue, size, buildKey) { 15 | this._buildValue = buildValue; 16 | this._buildKey = buildKey; 17 | this._maxKeySize = 1000; 18 | 19 | // Statistics 20 | this._hitCounter = 0; 21 | this._missCounter = 0; 22 | 23 | this.lru = new LRU(size); 24 | } 25 | 26 | /* Try to retrieve the value associated with `key` from the cache. If it's 27 | * not present, build it using `buildValue` and store it in the cache. 28 | * 29 | * This method always returns a value either from the LRU cache, or from a 30 | * direct call to `buildValue`. 31 | */ 32 | get(argument) { 33 | const key = this._buildKey ? this._buildKey(argument) : argument; 34 | let value = this.lru.get(key); 35 | 36 | if (value !== undefined) { 37 | // Cache hit 38 | this._hitCounter += 1; 39 | return value; 40 | } 41 | // Cache miss 42 | this._missCounter += 1; 43 | 44 | // Compute value 45 | value = this._buildValue(argument); 46 | 47 | // if key is large, don't cache 48 | if (!key || key.length > this._maxKeySize) { 49 | return value; 50 | } 51 | 52 | this.lru.set(key, value); 53 | return value; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/remote-action-provider.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import runtime from "../../platform/runtime"; 6 | 7 | /** 8 | * Abstract away exposing actions triggered through `sendMessage`. This is done 9 | * in the extension in different contexts: content scripts, popup windows, etc. 10 | */ 11 | export default class RemoteActionProvider { 12 | constructor(module, actions) { 13 | this.onMessage = (message) => { 14 | if (message.module === module) { 15 | const handler = actions[message.action]; 16 | if (handler !== undefined) { 17 | // Prepare array of arguments which will be given to `handler` 18 | let args = message.args || []; 19 | 20 | // Backward compatible "hack" since some users of this API still 21 | // provide a unique argument in the form of `{ message }`. If we 22 | // detect this scenario then we use it as argument for `handler`. 23 | if (message.message !== undefined) { 24 | args = [message.message]; 25 | } 26 | 27 | return Promise.resolve(handler(...args)); 28 | } 29 | } 30 | 31 | return undefined; // not handled by us 32 | }; 33 | } 34 | 35 | init() { 36 | runtime.onMessage.addListener(this.onMessage); 37 | } 38 | 39 | unload() { 40 | runtime.onMessage.removeListener(this.onMessage); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/sleep.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { setTimeout } from "../timers"; 6 | 7 | export default function sleep(time) { 8 | return new Promise((resolve) => setTimeout(resolve, time)); 9 | } 10 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/string-cache.ts: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default class Cache { 6 | public readonly size: number; 7 | public readonly cache: Array<{ 8 | key: string | undefined; 9 | value: Value | undefined; 10 | }> = []; 11 | private bitMask: number; 12 | 13 | constructor(size: number) { 14 | if (size <= 0 || size & (size - 1 !== 0)) { 15 | throw new Error("size must be a power of 2"); 16 | } 17 | this.size = size; 18 | this.bitMask = size - 1; 19 | this.clear(); 20 | } 21 | 22 | clear() { 23 | this.cache.length = 0; 24 | for (let i = 0; i < this.size; i += 1) { 25 | this.cache.push({ key: undefined, value: undefined }); 26 | } 27 | } 28 | 29 | get(key: string): Value | undefined { 30 | const val = this.cache[key.length & this.bitMask]; 31 | 32 | if (val.key !== key) { 33 | return undefined; 34 | } 35 | 36 | return val.value; 37 | } 38 | 39 | set(key: string, value: Value): Value { 40 | const current = this.cache[key.length & this.bitMask]; 41 | current.key = key; 42 | current.value = value; 43 | return value; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/strip-api-headers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | // Ends with *.wdp.brave.software or *.wdp.brave.com (and wdp-public) 6 | const regexpMatcher = /(?:(?:wdp|wdp-public)[.]brave[.](?:software|com))$/; 7 | 8 | // Do not remove headers for these 9 | const whitelist = new Set(); // Not need for exceptions 10 | 11 | export function isSafeToRemoveHeaders(hostname) { 12 | return regexpMatcher.test(hostname) && !whitelist.has(hostname); 13 | } 14 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/timeout.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import * as timers from "../timers"; 6 | 7 | /** 8 | * setInterval implementation using setTimeout as base method. This helper was 9 | * created with the intention of solving the multiple calls that can happen 10 | * using the setInterval function (if computer goes to sleep for example). 11 | * 12 | * To create an interval: 13 | * const timer = setTimeoutInterval(() => console.log('x'), 1000); 14 | * 15 | * To stop it: 16 | * timer.stop(); 17 | * 18 | */ 19 | function setTimeoutIntervalImpl(f, timeoutMS, args, instantStart) { 20 | let enabled = true; 21 | let timeout = null; 22 | 23 | const runTimeout = async () => { 24 | if (enabled) { 25 | // Call `f` and make sure we wait for it to terminate before we trigger 26 | // the next timeout (even if the function is async). 27 | try { 28 | await f(...args); 29 | } catch (ex) { 30 | /* Ignore */ 31 | } 32 | } 33 | 34 | if (enabled) { 35 | timeout = timers.setTimeout(runTimeout, timeoutMS); 36 | } 37 | }; 38 | 39 | const stop = () => { 40 | enabled = false; 41 | timers.clearTimeout(timeout); 42 | }; 43 | 44 | // Start running 45 | timeout = timers.setTimeout(runTimeout, instantStart ? 0 : timeoutMS); 46 | 47 | return { 48 | stop, 49 | }; 50 | } 51 | 52 | export default function setTimeoutInterval(f, timeoutMS, ...args) { 53 | return setTimeoutIntervalImpl(f, timeoutMS, args, false); 54 | } 55 | 56 | export function setTimeoutIntervalInstant(f, timeoutMS, ...args) { 57 | return setTimeoutIntervalImpl(f, timeoutMS, args, true); 58 | } 59 | -------------------------------------------------------------------------------- /modules/core/sources/helpers/wait.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { setTimeoutIntervalInstant } from "./timeout"; 6 | import * as timers from "../timers"; 7 | 8 | function waitForImpl(fn, timeout) { 9 | let error; 10 | let resolve; 11 | let reject; 12 | 13 | let timeoutId; 14 | let interval; 15 | 16 | // Clean-up resources 17 | const stop = () => { 18 | timers.clearTimeout(timeoutId); 19 | interval.stop(); 20 | }; 21 | 22 | const promise = new Promise((res, rej) => { 23 | resolve = res; 24 | reject = rej; 25 | }); 26 | 27 | const check = async () => { 28 | try { 29 | const result = await fn(); 30 | // In case of truthy result, we stop 31 | if (result) { 32 | stop(); 33 | resolve(result); 34 | return; 35 | } 36 | } catch (ex) { 37 | error = ex; 38 | } 39 | }; 40 | 41 | // Start check at regular interval until one of the following conditions is met: 42 | // * `timeout` ms elapsed 43 | // * `fn` eventually results into a truthy values 44 | interval = setTimeoutIntervalInstant(check, 10); 45 | 46 | // Reject after `timeout` ms 47 | timeoutId = timers.setTimeout(() => { 48 | stop(); 49 | reject(error || new Error("waitFor timeout")); 50 | }, timeout); 51 | 52 | return promise; 53 | } 54 | 55 | export function waitFor(fn, timeout = 15000) { 56 | return waitForImpl(fn, timeout); 57 | } 58 | 59 | export function wait(time) { 60 | return new Promise((resolve) => timers.setTimeout(resolve, time)); 61 | } 62 | -------------------------------------------------------------------------------- /modules/core/sources/history-service.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import historyService from "../platform/history-service"; 6 | 7 | // implements https://developer.chrome.com/extensions/history 8 | export default historyService; 9 | -------------------------------------------------------------------------------- /modules/core/sources/kord.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import inject, { 6 | getModuleList, 7 | setGlobal as injectSetGlobal, 8 | } from "./kord/inject"; 9 | 10 | export default { 11 | inject, 12 | get modules() { 13 | return getModuleList(); 14 | }, 15 | }; 16 | 17 | export function setApp(app) { 18 | injectSetGlobal(app); 19 | } 20 | -------------------------------------------------------------------------------- /modules/core/sources/lint.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* eslint-disable */ 6 | 7 | /* Empty module used to integrate eslint into unit tests */ 8 | export default class Lint {} 9 | -------------------------------------------------------------------------------- /modules/core/sources/platform.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import platform from "../platform/platform"; 6 | 7 | export { getResourceUrl } from "../platform/platform"; 8 | 9 | export function notImplemented() { 10 | throw new Error("Not implemented"); 11 | } 12 | 13 | export const isFirefox = platform.isFirefox; 14 | export const isChromium = platform.isChromium; 15 | export const isEdge = platform.isEdge; 16 | export const platformName = platform.platformName; 17 | export const isWebExtension = platformName === "webextension"; 18 | export const product = "BRAVE"; 19 | -------------------------------------------------------------------------------- /modules/core/sources/prefs.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { getPref, setPref, hasPref, clearPref, init } from "../platform/prefs"; 6 | import config from "./config"; 7 | 8 | export default { 9 | /** 10 | * Get a value from preferences db 11 | * @param {string} pref - preference identifier 12 | * @param {*=} defautlValue - returned value in case pref is not defined 13 | */ 14 | get(pref, defaultValue) { 15 | let value = defaultValue; 16 | if (config.default_prefs) { 17 | value = 18 | typeof config.default_prefs[pref] === "undefined" 19 | ? defaultValue 20 | : config.default_prefs[pref]; 21 | } 22 | return getPref(pref, value); 23 | }, 24 | /** 25 | * Set a value in preferences db 26 | * @param {string} pref - preference identifier 27 | * @param {*=} value 28 | */ 29 | set: setPref, 30 | 31 | /** 32 | * Check if there is a value in preferences db 33 | * @param {string} pref - preference identifier 34 | */ 35 | has: hasPref, 36 | /** 37 | * Clear value in preferences db 38 | * @param {string} pref - preference identifier 39 | */ 40 | clear: clearPref, 41 | 42 | /** 43 | * Set a value of type object in preferences db 44 | * @param {string} pref - preference identifier 45 | */ 46 | getObject(key, ...rest) { 47 | return JSON.parse(this.get(key, "{}", ...rest)); 48 | }, 49 | 50 | /** 51 | * Set a value in preferences db 52 | * @param {string} pref - preference identifier 53 | * @param {object|function} 54 | */ 55 | setObject(key, value, ...rest) { 56 | if (value instanceof Function) { 57 | const prevValue = this.getObject(key); 58 | const newValue = value(prevValue); 59 | this.setObject(key, newValue, ...rest); 60 | } else if (typeof value === "object") { 61 | this.set(key, JSON.stringify(value), ...rest); 62 | } else { 63 | throw new TypeError(); 64 | } 65 | }, 66 | 67 | init, 68 | }; 69 | -------------------------------------------------------------------------------- /modules/core/sources/request-sanitizer.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import webRequest from "./webrequest"; 6 | import { isSafeToRemoveHeaders } from "./helpers/strip-api-headers"; 7 | import { isTrackableOriginHeaderFromOurExtension } from "../platform/fetch"; 8 | import console from "./console"; 9 | import { extractHostname } from "./tlds"; 10 | import PermissionManager from "../platform/permission-manager"; 11 | 12 | function findHeader(requestHeaders, header) { 13 | const x = requestHeaders.find(({ name }) => name.toLowerCase() === header); 14 | return x ? x.value : ""; 15 | } 16 | 17 | function safeFilter(req) { 18 | const { requestHeaders, type, url, tabId } = req; 19 | 20 | if (tabId !== -1) { 21 | return undefined; 22 | } 23 | 24 | const hostname = extractHostname(url); 25 | if (!hostname) { 26 | console.warn("Could not extract hostname", url); 27 | return undefined; 28 | } 29 | 30 | const headersToRemove = []; 31 | if (isSafeToRemoveHeaders(hostname)) { 32 | headersToRemove.push("origin", "cookie"); 33 | if (type === "xmlhttprequest") { 34 | headersToRemove.push("accept-language", "user-agent"); 35 | } 36 | } else if ( 37 | isTrackableOriginHeaderFromOurExtension( 38 | findHeader(requestHeaders, "origin"), 39 | ) 40 | ) { 41 | headersToRemove.push("origin"); // Prevent extension origin leaks 42 | } 43 | 44 | if (headersToRemove.length > 0) { 45 | return { 46 | requestHeaders: requestHeaders.filter( 47 | (h) => !headersToRemove.includes(h.name.toLowerCase()), 48 | ), 49 | }; 50 | } 51 | 52 | return undefined; 53 | } 54 | 55 | let HAS_RIGHT_PERMISSIONS = false; 56 | 57 | export async function enableRequestSanitizer() { 58 | // WebExtension might not have webRequest API permission 59 | HAS_RIGHT_PERMISSIONS = await PermissionManager.contains([ 60 | PermissionManager.PERMISSIONS.WEB_REQUEST, 61 | PermissionManager.PERMISSIONS.WEB_REQUEST_BLOCKING, 62 | ]); 63 | 64 | if (!HAS_RIGHT_PERMISSIONS) { 65 | return; 66 | } 67 | 68 | const networkFilters = { 69 | urls: [""], 70 | tabId: -1, 71 | }; 72 | 73 | webRequest.onBeforeSendHeaders.addListener(safeFilter, networkFilters, [ 74 | "blocking", 75 | "requestHeaders", 76 | ]); 77 | } 78 | 79 | export function disableRequestSanitizer() { 80 | if (!HAS_RIGHT_PERMISSIONS) { 81 | return; 82 | } 83 | webRequest.onBeforeSendHeaders.removeListener(safeFilter); 84 | } 85 | -------------------------------------------------------------------------------- /modules/core/sources/services.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { service as pacemaker } from "./services/pacemaker"; 6 | 7 | export default { 8 | pacemaker, 9 | }; 10 | -------------------------------------------------------------------------------- /modules/core/sources/settings.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default (settings, other = {}) => ({ 6 | ...settings, 7 | ...other, 8 | }); 9 | -------------------------------------------------------------------------------- /modules/core/sources/time.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | function timestamp() { 6 | return Date.now(); 7 | } 8 | 9 | export { timestamp }; 10 | -------------------------------------------------------------------------------- /modules/core/sources/timers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export { 6 | setInterval, 7 | setTimeout, 8 | clearTimeout, 9 | clearInterval, 10 | } from "../platform/timers"; 11 | -------------------------------------------------------------------------------- /modules/core/sources/tlds.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export { 6 | getHostname as extractHostname, 7 | getDomainWithoutSuffix, 8 | } from "../platform/lib/tldts"; 9 | -------------------------------------------------------------------------------- /modules/core/sources/webrequest.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import webrequest, { VALID_RESPONSE_PROPERTIES } from "../platform/webrequest"; 6 | import { isEdge } from "./platform"; 7 | 8 | export default webrequest; 9 | 10 | export { VALID_RESPONSE_PROPERTIES }; 11 | 12 | function getOptionArray(options) { 13 | if (!options) { 14 | return []; 15 | } 16 | // get subset of options which are defined 17 | return [ 18 | options.BLOCKING, 19 | // firefox and chrome disagree on how to name these 20 | options.REQUEST_HEADERS || options.REQUESTHEADERS, 21 | options.RESPONSE_HEADERS || options.RESPONSEHEADERS, 22 | // extra headers: Chrome 72: 23 | // https://groups.google.com/a/chromium.org/forum/#!topic/chromium-extensions/vYIaeezZwfQ 24 | options.EXTRA_HEADERS, 25 | // request body disabled to avoid the overhead when not needed 26 | // options.REQUEST_BODY, 27 | ].filter((o) => !!o); 28 | } 29 | 30 | // build allowed extraInfo options from Options objects. 31 | export const EXTRA_INFO_SPEC = !isEdge 32 | ? { 33 | onBeforeRequest: getOptionArray(webrequest.OnBeforeRequestOptions), 34 | onBeforeSendHeaders: getOptionArray( 35 | webrequest.OnBeforeSendHeadersOptions, 36 | ), 37 | onSendHeaders: getOptionArray(webrequest.OnSendHeadersOptions), 38 | onHeadersReceived: getOptionArray(webrequest.OnHeadersReceivedOptions), 39 | onAuthRequired: getOptionArray(webrequest.OnAuthRequiredOptions), 40 | onResponseStarted: getOptionArray(webrequest.OnResponseStartedOptions), 41 | onBeforeRedirect: getOptionArray(webrequest.OnBeforeRedirectOptions), 42 | onCompleted: getOptionArray(webrequest.OnCompletedOptions), 43 | onErrorOccurred: undefined, 44 | } 45 | : { 46 | onBeforeRequest: ["blocking"], 47 | onBeforeSendHeaders: ["blocking", "requestHeaders"], 48 | onSendHeaders: ["requestHeaders"], 49 | onHeadersReceived: ["blocking", "requestHeaders"], 50 | onAuthRequired: ["responseHeaders", "blocking"], 51 | onResponseStarted: ["responseHeaders"], 52 | onBeforeRedirect: ["responseHeaders"], 53 | onCompleted: ["responseHeaders"], 54 | onErrorOccurred: undefined, 55 | }; 56 | -------------------------------------------------------------------------------- /modules/core/sources/zlib.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export { inflate, deflate } from "../platform/lib/zlib"; 6 | -------------------------------------------------------------------------------- /modules/core/tests/integration/helpers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* eslint-disable */ 6 | 7 | export * from "../test-helpers"; 8 | export { getResourceUrl } from "../../../core/platform"; 9 | -------------------------------------------------------------------------------- /modules/core/tests/integration/http-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { testServer, expect } from "../test-helpers"; 6 | import { promiseHttpHandler } from "../../../core/http"; 7 | 8 | export default function () { 9 | const url = testServer.getBaseUrl(); 10 | const responseTest = "hello world"; 11 | 12 | beforeEach(() => 13 | testServer.registerPathHandler("/", { result: responseTest }), 14 | ); 15 | describe("fetch", () => { 16 | it("doesn't send origin header", async () => { 17 | await testServer.fetch(); 18 | const hits = await testServer.getHitsForPath("/"); 19 | const origin = hits[0].headers.origin; 20 | // Header was either removed or it didn't exist 21 | expect(origin === "null" || origin === undefined).to.be.true; 22 | }); 23 | }); 24 | 25 | describe("promiseHttpHandler", () => { 26 | context("get request", () => { 27 | it("gets response of request", async () => { 28 | const resp = await promiseHttpHandler("GET", url); 29 | expect(resp.response).to.eql(responseTest); 30 | const hitCtr = await testServer.getHitCtr("/"); 31 | expect(hitCtr).to.equal(1); 32 | }); 33 | 34 | it("does not fulfill for a 404", () => 35 | promiseHttpHandler("GET", `${url}404`).then( 36 | () => { 37 | throw new Error("promise unexpectedly fulfilled"); 38 | }, 39 | () => {}, 40 | )); 41 | }); 42 | 43 | context("post request", () => { 44 | const postDataSent = '{ "data": "testdata" }'; 45 | 46 | it("posts provided data", async () => { 47 | const resp = await promiseHttpHandler("POST", url, postDataSent); 48 | expect(resp.response).to.eql(responseTest); 49 | const hits = (await testServer.getHits()).get("/"); 50 | expect(hits.length).to.equal(1); 51 | expect(hits[0].body).to.eql(JSON.parse(postDataSent)); 52 | }); 53 | 54 | it.skip("can compress sent post data", function () { 55 | // TODO - this is currently not possible to test using the testServer we 56 | // have on chromium-tests. So it is currently disabled. 57 | // return promiseHttpHandler('POST', url, postDataSent, undefined, true) 58 | // .then(function (resp) { 59 | // expect(hitCtr).to.eql(1); 60 | // expect(resp.response).to.eql(responseTest); 61 | // expect(contentEncodingHeader).to.eql('gzip'); 62 | // const postData = gzip.decompress(binaryStringToUint8Array(requestData)); 63 | // expect(postData).to.eql(postDataSent); 64 | // }); 65 | }); 66 | }); 67 | }); 68 | } 69 | -------------------------------------------------------------------------------- /modules/core/tests/test-helpers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import chai from "chai"; 6 | import { app, wrap } from "../../platform/test-helpers/helpers"; 7 | import { wait } from "../../core/helpers/wait"; 8 | 9 | export * from "../../platform/test-helpers/helpers"; 10 | export { wait, waitFor } from "../../core/helpers/wait"; 11 | 12 | export { testPageSources } from "../../platform/integration-tests/test-page-sources"; 13 | 14 | // Re-export some browser utils 15 | export { closeTab, newTab, updateTab, getTab } from "../../platform/tabs"; 16 | 17 | /** 18 | * The following helpers are platform independents and can be implemented 19 | * directly in core. 20 | */ 21 | 22 | export const Events = wrap(() => app.events); 23 | 24 | export const prefs = wrap(() => app.prefs); 25 | 26 | export const sleep = wait; 27 | 28 | export function clone(obj) { 29 | return JSON.parse(JSON.stringify(obj)); 30 | } 31 | 32 | export const expect = chai.expect; 33 | export { chai }; 34 | -------------------------------------------------------------------------------- /modules/core/tests/unit/app-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global chai, sinon, describeModule */ 6 | 7 | const tests = () => { 8 | describe("#disableModule", function () { 9 | let App; 10 | 11 | beforeEach(function () { 12 | App = this.module().default; 13 | }); 14 | 15 | it("sets a pref", async function () { 16 | const setPref = sinon.spy(); 17 | const app = new App(); 18 | app.isRunning = true; 19 | 20 | app.modules.test = { 21 | isDisabled: false, 22 | disable: () => {}, 23 | }; 24 | 25 | this.deps("core/prefs").default.set = setPref; 26 | 27 | await app.disableModule("test"); 28 | 29 | chai.expect(setPref).to.be.calledWith("modules.test.enabled", false); 30 | }); 31 | 32 | it("returns a promise for modules that are enabling", async function () { 33 | const app = new App(); 34 | const disableSpy = sinon.spy(); 35 | app.isRunning = true; 36 | 37 | app.modules.test = { 38 | isDisabled: false, 39 | isEnabling: true, 40 | disable: disableSpy, 41 | isReady: () => Promise.resolve(), 42 | }; 43 | 44 | const disablePromise = app.disableModule("test"); 45 | 46 | chai.expect(disableSpy).to.not.be.called; 47 | chai.expect(disablePromise).to.be.instanceof(Promise); 48 | 49 | await chai.expect(disablePromise).to.be.fulfilled; 50 | 51 | chai.expect(disableSpy).to.be.called; 52 | }); 53 | }); 54 | }; 55 | 56 | export default describeModule( 57 | "core/app", 58 | function () { 59 | return { 60 | "webextension-polyfill": { 61 | default: {}, 62 | }, 63 | "core/events": { 64 | default: {}, 65 | }, 66 | "core/prefs": { 67 | default: { 68 | set() {}, 69 | has: () => false, 70 | get() {}, 71 | }, 72 | }, 73 | "core/console": { 74 | isLoggingEnabled: () => false, 75 | default: {}, 76 | }, 77 | "core/kord": { 78 | setApp() {}, 79 | }, 80 | "core/app/module": { 81 | default: class {}, 82 | }, 83 | "platform/globals": { 84 | chrome: {}, 85 | }, 86 | "platform/browser": { 87 | forEachWindow() {}, 88 | }, 89 | "core/platform": { 90 | default: {}, 91 | }, 92 | "platform/sqlite": { 93 | openDBHome: () => {}, 94 | close: () => {}, 95 | }, 96 | }; 97 | }, 98 | tests, 99 | ); 100 | -------------------------------------------------------------------------------- /modules/core/tests/unit/bloom-filter-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global chai, describeModule */ 6 | 7 | const expect = chai.expect; 8 | 9 | export default describeModule( 10 | "core/bloom-filter", 11 | () => ({}), 12 | () => { 13 | describe("BloomFilter", function () { 14 | let BloomFilter; 15 | 16 | beforeEach(function () { 17 | BloomFilter = this.module().default; 18 | }); 19 | 20 | it("should pass simple testSingle tests", function () { 21 | const uut = new BloomFilter(new Array(101), 7); 22 | 23 | expect(uut.testSingle("x")).to.be.false; 24 | expect(uut.testSingle("y")).to.be.false; 25 | 26 | uut.addSingle("x"); 27 | expect(uut.testSingle("x")).to.be.true; 28 | expect(uut.testSingle("y")).to.be.false; 29 | 30 | uut.addSingle("y"); 31 | expect(uut.testSingle("x")).to.be.true; 32 | expect(uut.testSingle("y")).to.be.true; 33 | }); 34 | }); 35 | }, 36 | ); 37 | -------------------------------------------------------------------------------- /modules/core/tests/unit/helpers-md5-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global chai, describeModule */ 6 | 7 | const expect = chai.expect; 8 | 9 | export default describeModule( 10 | "core/helpers/md5", 11 | function () { 12 | return { 13 | "./console": { 14 | default: {}, 15 | }, 16 | }; 17 | }, 18 | function () { 19 | context("MD5 Hex-encoding tests", function () { 20 | let md5; 21 | 22 | beforeEach(function () { 23 | md5 = this.module().default; 24 | }); 25 | 26 | it("should create a hex-encoded MD5 hash of an ASCII value", function () { 27 | expect(md5("brave")).to.equal("d18eabd76bdc6fc6ccdfdd20d40ab20b"); 28 | }); 29 | 30 | it("should create a hex-encoded MD5 hash of an ASCII value", function () { 31 | expect(md5("value")).to.equal("2063c1608d6e0baf80249c42e2be5804"); 32 | }); 33 | 34 | it("should create a hex-encoded MD5 hash of an UTF-8 value", function () { 35 | expect(md5("日本")).to.equal("4dbed2e657457884e67137d3514119b3"); 36 | }); 37 | }); 38 | }, 39 | ); 40 | -------------------------------------------------------------------------------- /modules/core/tests/unit/helpers/strip-api-headers-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* globals describeModule, chai */ 6 | 7 | const expect = chai.expect; 8 | 9 | export default describeModule( 10 | "core/helpers/strip-api-headers", 11 | () => ({}), 12 | function () { 13 | describe("#isSafeToRemoveHeaders", function () { 14 | let isSafeToRemoveHeaders; 15 | 16 | beforeEach(function () { 17 | isSafeToRemoveHeaders = this.module().isSafeToRemoveHeaders; 18 | }); 19 | 20 | const safeToRemoveHosts = [ 21 | "wdp.brave.com", 22 | "star.wdp.brave.com", 23 | "star.wdp.brave.software", 24 | ]; 25 | for (const hostname of safeToRemoveHosts) { 26 | it(`should modify requests to whitelisted hosts: ${hostname}`, () => { 27 | expect(isSafeToRemoveHeaders(hostname)).to.be.true; 28 | }); 29 | } 30 | 31 | const protectedHosts = [ 32 | "www.google.com", 33 | "www.amazon.com", 34 | "localhost", 35 | "127.0.0.1", 36 | "search.brave.com", 37 | "search.brave.software", 38 | "anotherdomain.brave.com", 39 | "any-domain-ending-with.brave.software", 40 | ]; 41 | for (const hostname of protectedHosts) { 42 | it(`should not modify requests non-whitelisted hosts: ${hostname}`, () => { 43 | expect(isSafeToRemoveHeaders(hostname)).to.be.false; 44 | }); 45 | } 46 | }); 47 | }, 48 | ); 49 | -------------------------------------------------------------------------------- /modules/core/tests/unit/http-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global describeModule */ 6 | 7 | // TODO 8 | 9 | export default describeModule( 10 | "core/http", 11 | function () { 12 | return { 13 | "./console": { 14 | default: {}, 15 | }, 16 | "./gzip": { 17 | default: {}, 18 | }, 19 | "../platform/xmlhttprequest": { 20 | default: {}, 21 | }, 22 | "../platform/chrome-url-handler": { 23 | default: {}, 24 | }, 25 | "../platform/fetch": { 26 | default: {}, 27 | }, 28 | }; 29 | }, 30 | function () { 31 | context("promiseHttpHandler", function () {}); 32 | }, 33 | ); 34 | -------------------------------------------------------------------------------- /modules/core/tests/unit/prefs-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global chai, describeModule */ 6 | 7 | export default describeModule( 8 | "core/prefs", 9 | function () { 10 | return { 11 | "../platform/prefs": { 12 | getPref: "[dynamic]", 13 | }, 14 | "./config": { 15 | default: { 16 | default_prefs: {}, 17 | }, 18 | }, 19 | }; 20 | }, 21 | function () { 22 | let subject; 23 | 24 | describe("#get", function () { 25 | const key = "test"; 26 | 27 | beforeEach(function () { 28 | subject = this.module().default.get; 29 | this.deps("../platform/prefs").getPref = (_, defaultValue) => 30 | defaultValue; 31 | }); 32 | 33 | context("with not value stored", function () { 34 | const defaultValue = "defaultValue"; 35 | 36 | it("returns undefined", function () { 37 | chai.expect(subject(key)).to.equal(undefined); 38 | }); 39 | 40 | it("returns defaultValue if provided", function () { 41 | chai.expect(subject(key, defaultValue)).to.equal(defaultValue); 42 | }); 43 | 44 | context("with config default prefs", function () { 45 | it("return default prefs from config", function () { 46 | const defaultPrefValue = "defaultPrefValue"; 47 | this.deps("./config").default.default_prefs[key] = defaultPrefValue; 48 | chai.expect(subject(key)).to.equal(defaultPrefValue); 49 | chai.expect(subject(key, defaultValue)).to.equal(defaultPrefValue); 50 | }); 51 | }); 52 | }); 53 | }); 54 | }, 55 | ); 56 | -------------------------------------------------------------------------------- /modules/core/tests/unit/utils/dexie.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const IDBKeyRange = require("fake-indexeddb/lib/FDBKeyRange"); 6 | const indexedDB = require("fake-indexeddb"); 7 | 8 | global.indexedDB = indexedDB; 9 | global.IDBKeyRange = IDBKeyRange; 10 | 11 | async function getInitDexie() { 12 | const Dexie = await import("dexie"); 13 | 14 | function initDexie(name) { 15 | return new Dexie(name, { 16 | indexedDB, 17 | IDBKeyRange, 18 | }); 19 | } 20 | 21 | initDexie.__proto__.delete = Dexie.delete; 22 | initDexie.__proto__.exists = Dexie.exists; 23 | 24 | return; 25 | } 26 | module.exports = { 27 | "platform/lib/dexie": { 28 | default: () => getInitDexie, 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /modules/core/tests/unit/utils/url-parser.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const tldts = require("tldts-experimental"); 6 | const urlParser = require("@cliqz/url-parser"); 7 | const punycode = require("punycode"); 8 | 9 | module.exports = { 10 | "platform/lib/tldts": tldts, 11 | "@cliqz/url-parser": urlParser, 12 | punycode, 13 | }; 14 | -------------------------------------------------------------------------------- /modules/core/tests/unit/wait-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global describeModule, chai */ 6 | 7 | export default describeModule( 8 | "core/helpers/wait", 9 | () => ({ 10 | "core/console": { 11 | default: { 12 | error() {}, 13 | }, 14 | }, 15 | }), 16 | () => { 17 | describe("#waitFor", () => { 18 | let waitFor; 19 | beforeEach(function () { 20 | waitFor = this.module().waitFor; 21 | }); 22 | 23 | it("resolves immediatly", async () => 24 | chai.expect(await waitFor(() => true)).to.be.true); 25 | 26 | it("resolves with result of function (sync)", async () => 27 | chai.expect(await waitFor(() => "foo")).to.be.eql("foo")); 28 | 29 | it("resolves with result of function (async)", async () => 30 | chai 31 | .expect(await waitFor(() => Promise.resolve("foo"))) 32 | .to.be.eql("foo")); 33 | 34 | it("times out", () => 35 | chai 36 | .expect(waitFor(() => false, 100)) 37 | .to.eventually.be.rejectedWith("waitFor timeout")); 38 | 39 | it("succeeds after a few tries (sync)", () => { 40 | let i = 0; 41 | const fn = () => { 42 | i += 1; 43 | return i === 10; 44 | }; 45 | 46 | return waitFor(fn); 47 | }); 48 | 49 | it("succeeds after a few tries (async)", () => { 50 | let i = 0; 51 | const fn = async () => { 52 | i += 1; 53 | await new Promise((resolve) => setTimeout(resolve, 60)); 54 | return i === 10; 55 | }; 56 | 57 | return waitFor(fn); 58 | }); 59 | }); 60 | }, 61 | ); 62 | -------------------------------------------------------------------------------- /modules/fetcher/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "subprojects": [] 3 | } 4 | -------------------------------------------------------------------------------- /modules/fetcher/sources/background.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import background from "../core/base/background"; 6 | import inject from "../core/kord/inject"; 7 | import Manager from "./manager"; 8 | import logger from "./logger"; 9 | 10 | export default background({ 11 | requiresServices: ["pacemaker"], 12 | webDiscoveryProject: inject.module("web-discovery-project"), 13 | 14 | async init() { 15 | this.manager = new Manager(this.webDiscoveryProject); 16 | try { 17 | await this.manager.init(); 18 | } catch (ex) { 19 | logger.error("Unexpected error when trying to initialize fetcher", ex); 20 | } 21 | }, 22 | 23 | unload() { 24 | if (this.manager) { 25 | this.manager.unload(); 26 | this.manager = null; 27 | } 28 | }, 29 | 30 | status() { 31 | return { 32 | visible: true, 33 | }; 34 | }, 35 | 36 | actions: {}, 37 | }); 38 | -------------------------------------------------------------------------------- /modules/fetcher/sources/logger.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import Logger from "../core/logger"; 6 | 7 | export default Logger.get("fetcher", { level: "warning" }); 8 | -------------------------------------------------------------------------------- /modules/hpnv2/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "subprojects": [] 3 | } 4 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/README.md: -------------------------------------------------------------------------------- 1 | # Human Web Proxy Network (HPN) 2 | 3 | We published a blog post on [Human Web Proxy Network (HPN)](https://0x65.dev/blog/2019-12-04/human-web-proxy-network-hpn.html). 4 | It explains why we need HPN and gives an overview of how the protocol works. 5 | 6 | There are some aspects that did not made it into the blog post, 7 | but are relevant on a technical level. 8 | 9 | ## Detecting and Auto-Correcting Clock Drift 10 | 11 | When sending messages, we need an accurate clock; otherwise, users 12 | could potentially be de-anonymized based on their clock drift. 13 | 14 | The general idea on how to detect and auto-correct it in hpnv2 is as follows: 15 | 16 | - Every client fetches the clock time from the server (which each `/config` request). 17 | The server's clock is the source of truth and can be used to compute the clock drift. 18 | - In parallel, clients run a timer (`setInterval`) which fires every minute. 19 | As long as the system clock is in sync with the estimated 20 | time - based on server time plus elapsed minutes - the clock on the client 21 | can be trusted. 22 | 23 | The clock can get out-of-sync for two reasons: 24 | 25 | 1. The system clock on the user's machine is consistently off 26 | 2. The system clock is temporarily outdated after the machine awoke from suspend mode 27 | 28 | In case 1, it is easy to correct, as the clock drift remains constant. 29 | Case 2 will force the client to synchronize with the server again. 30 | 31 | The code can be found in `trusted-clock.es`. 32 | 33 | ## Client Side Burst Mitigation 34 | 35 | There are recurring situations when clients will send messages 36 | in a coordinated fashion. The effect can be like a 37 | distributed DoS attack on our infrastructure. 38 | As a mitigation, clients can randomly delay messages following 39 | recommendations that we can tuned from the server side. 40 | 41 | Ideally, clients should not send messages all at once and these 42 | throttling rules should not be necessary. Still, it can happen 43 | and the mechanism helps us to deal with spikes, without having 44 | to push new releases. 45 | 46 | More details can be found in the comments in the code (`message-throttler.es`). 47 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/constants.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const VERSION = 1; 6 | const ECDH_P256_AES_128_GCM = 0xea; 7 | 8 | export { VERSION, ECDH_P256_AES_128_GCM }; 9 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/errors.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | // https://stackoverflow.com/questions/31089801/extending-error-in-javascript-with-es6-syntax-babel 6 | 7 | class ExtendableError extends Error { 8 | constructor(message) { 9 | super(message); 10 | this.name = this.constructor.name; 11 | if (typeof Error.captureStackTrace === "function") { 12 | Error.captureStackTrace(this, this.constructor); 13 | } else { 14 | this.stack = new Error(message).stack; 15 | } 16 | } 17 | } 18 | 19 | /** 20 | * If you get an error in HPN, this function will give you 21 | * a hint whether retrying with the same message again later 22 | * is a reasonable strategy. 23 | * 24 | * That avoids futile attempts, as some types of errors are permanent. 25 | * For example, once you exceed the rate limit, retrying with the same 26 | * message (with the identical timestamp) will only trigger the same error. 27 | * 28 | * But note that even for recoverable errors, retrying immediately is 29 | * almost never a good idea. When you get an error, it means HPN already 30 | * exhausted the basic options (e.g., retrying failing HTTP requests). 31 | */ 32 | class RecoverableError extends ExtendableError { 33 | constructor(message) { 34 | super(message); 35 | this.isRecoverableHpnv2Error = true; 36 | this.isPermanentHpnv2Error = false; 37 | } 38 | } 39 | 40 | class PermanentError extends ExtendableError { 41 | constructor(message) { 42 | super(message); 43 | this.isRecoverableHpnv2Error = false; 44 | this.isPermanentHpnv2Error = true; 45 | } 46 | } 47 | 48 | export class MsgQuotaError extends PermanentError {} 49 | export class NotReadyError extends RecoverableError {} 50 | export class InvalidMsgError extends PermanentError {} 51 | export class NoCredentialsError extends RecoverableError {} 52 | export class BadCredentialsError extends RecoverableError {} 53 | export class SignMsgError extends PermanentError {} 54 | export class TooBigMsgError extends PermanentError {} 55 | export class TransportError extends RecoverableError {} 56 | export class MsgTimeoutError extends RecoverableError {} 57 | export class OldVersionError extends PermanentError {} 58 | export class FetchConfigError extends RecoverableError {} 59 | export class JoinGroupsError extends RecoverableError {} 60 | export class InitSignerError extends PermanentError {} 61 | export class ServerError extends RecoverableError {} 62 | export class ClockOutOfSyncWhileJoining extends RecoverableError {} 63 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/group-signer.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import config from "../core/config"; 6 | import console from "../core/console"; 7 | import Worker from "../platform/worker"; 8 | 9 | export default class GroupSigner { 10 | async _sendWorker(fn, ...args) { 11 | const id = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); 12 | return new Promise((resolve, reject) => { 13 | this.promises[id] = { resolve, reject }; 14 | this.worker.postMessage({ 15 | id, 16 | fn, 17 | args, 18 | }); 19 | }); 20 | } 21 | 22 | async seed(...args) { 23 | return this._sendWorker("seed", ...args); 24 | } 25 | 26 | async setGroupPubKey(...args) { 27 | return this._sendWorker("setGroupPubKey", ...args); 28 | } 29 | 30 | async setUserCredentials(...args) { 31 | return this._sendWorker("setUserCredentials", ...args); 32 | } 33 | 34 | async getUserCredentials(...args) { 35 | return this._sendWorker("getUserCredentials", ...args); 36 | } 37 | 38 | async startJoin(...args) { 39 | return this._sendWorker("startJoin", ...args); 40 | } 41 | 42 | async finishJoin(...args) { 43 | return this._sendWorker("finishJoin", ...args); 44 | } 45 | 46 | async sign(...args) { 47 | return this._sendWorker("sign", ...args); 48 | } 49 | 50 | constructor() { 51 | const build = typeof WebAssembly !== "undefined" ? "wasm" : "asmjs"; 52 | this.worker = new Worker( 53 | `${config.baseURL}hpnv2/worker.${build}.bundle.js`, 54 | ); 55 | this.promises = {}; 56 | this.worker.onmessage = (args) => { 57 | if (args.data.logMessage) { 58 | const msg = args.data.logMessage; 59 | console[msg.type](...msg.args); 60 | return; 61 | } 62 | 63 | const { 64 | data: { id, data, error }, 65 | } = args; 66 | const p = this.promises[id]; 67 | delete this.promises[id]; 68 | if (error) { 69 | p.reject(new Error(error)); 70 | } else { 71 | p.resolve(data); 72 | } 73 | }; 74 | } 75 | 76 | unload() { 77 | if (this.worker) { 78 | this.worker.terminate(); 79 | delete this.worker; 80 | this.promises = {}; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/logger.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import Logger from "../core/logger"; 6 | 7 | export default Logger.get("hpnv2", { level: "warning" }); 8 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/utils.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* eslint no-bitwise: 'off' */ 6 | 7 | import { inflate, deflate } from "../core/zlib"; 8 | import { TooBigMsgError } from "./errors"; 9 | 10 | export function formatDate(hours) { 11 | return new Date(hours * 3600 * 1000).toISOString().replace(/[^0-9]/g, ""); 12 | } 13 | 14 | /** 15 | * Returns a string, for example: "20190122" 16 | */ 17 | export function formatHoursAsYYYYMMDD(hoursSinceEpoch) { 18 | return formatDate(hoursSinceEpoch).slice(0, 8); 19 | } 20 | 21 | export function formatError(e, original = true) { 22 | const name = e && e.constructor && e.constructor.name; 23 | const msg = e && e.message; 24 | const stack = e && e.stack; 25 | const originalError = original 26 | ? e && e.originalError && formatError(e.originalError, false) 27 | : undefined; 28 | return { name, msg, stack, originalError }; 29 | } 30 | 31 | export async function reflectPromise(promise) { 32 | try { 33 | return { value: await promise }; 34 | } catch (error) { 35 | return { isError: true, error }; 36 | } 37 | } 38 | 39 | // https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 40 | function nextPow2(_v) { 41 | let v = _v | 0; 42 | v -= 1; 43 | v |= v >> 1; 44 | v |= v >> 2; 45 | v |= v >> 4; 46 | v |= v >> 8; 47 | v |= v >> 16; 48 | v += 1; 49 | return v; 50 | } 51 | 52 | function encodeLength(length) { 53 | // We could also encode length = 32767 = (1 << 15) - 1, 54 | // but since the message overhead is 2 bytes, in that case 55 | // the final message would go to the 64KB bucket, which 56 | // we don't want, so enforcing length < (1 << 15) - 1. 57 | if (length >= (1 << 15) - 1) { 58 | throw new TooBigMsgError("Message is too big"); 59 | } 60 | return (1 << 15) | length; 61 | } 62 | 63 | function decodeLength(data) { 64 | if (data < 1 << 15) { 65 | throw new Error("Wrong encoded length"); 66 | } 67 | return data & ((1 << 15) - 1); 68 | } 69 | 70 | export function encodeWithPadding(message) { 71 | const compressed = deflate(message); 72 | const paddedSize = Math.max(1 << 10, nextPow2(2 + compressed.length)); 73 | const data = new Uint8Array(paddedSize); 74 | new DataView(data.buffer).setUint16(0, encodeLength(compressed.length)); 75 | data.set(compressed, 2); 76 | return data; 77 | } 78 | 79 | export function decodeWithPadding(data) { 80 | const compressedLength = decodeLength(new DataView(data.buffer).getUint16()); 81 | return inflate(data.slice(2, 2 + compressedLength)); 82 | } 83 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/window.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default class { 6 | init() {} 7 | 8 | unload() {} 9 | } 10 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/worker-common.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default function makeWorker(_self, getCredentialManager) { 6 | const self = _self; 7 | const console = { 8 | log: (...args) => self.postMessage({ logMessage: { type: "log", args } }), 9 | error: (...args) => 10 | self.postMessage({ logMessage: { type: "error", args } }), 11 | }; 12 | 13 | function wrapError(e) { 14 | return `Worker error: '${e && e.message}', stack: <<< ${e && e.stack} >>>`; 15 | } 16 | 17 | // Assuming there are no two concurrent messages being handled (caller 18 | // waits for response before sending another message) 19 | let signer; 20 | self.onmessage = async ({ data: { id, fn, args } }) => { 21 | if (!signer) { 22 | const CredentialManager = await getCredentialManager(); 23 | signer = new CredentialManager(); 24 | } 25 | const now = performance.now(); 26 | try { 27 | const data = signer[fn](...args); 28 | console.log("[hpnv2-worker]", fn, performance.now() - now, "ms"); 29 | self.postMessage({ id, data }); 30 | } catch (e) { 31 | const error = wrapError(e); 32 | console.error("[hpnv2-worker]", error); 33 | self.postMessage({ id, error }); 34 | } 35 | }; 36 | } 37 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/worker.asmjs.bundle.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import getCredentialManager from "anonymous-credentials/lib/asmjs"; 6 | import makeWorker from "./worker-common"; 7 | 8 | makeWorker(self, getCredentialManager); 9 | -------------------------------------------------------------------------------- /modules/hpnv2/sources/worker.wasm.bundle.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import getCredentialManager from "anonymous-credentials/lib/wasm"; 6 | import makeWorker from "./worker-common"; 7 | 8 | makeWorker(self, getCredentialManager); 9 | -------------------------------------------------------------------------------- /modules/hpnv2/tests/unit/digest-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global chai */ 6 | /* global describeModule */ 7 | 8 | export default describeModule( 9 | "hpnv2/digest", 10 | function () { 11 | return {}; 12 | }, 13 | 14 | () => { 15 | const expect = chai.expect; 16 | let flatten; 17 | let digest; 18 | 19 | describe("flatten", () => { 20 | beforeEach(function () { 21 | flatten = this.module().flatten; 22 | digest = this.module().digest; 23 | }); 24 | 25 | it("simple digest", () => { 26 | expect(() => digest(["foo"], {})).to.throw( 27 | "Found undefined field when calculating digest", 28 | ); 29 | expect(digest(["foo"], { foo: 5, bar: 6 })).to.deep.equal([5]); 30 | }); 31 | 32 | it("simple flatten", () => { 33 | const msg = { 34 | hello: { 35 | good: { 36 | bye: 7, 37 | }, 38 | bad: 5, 39 | ugly: {}, 40 | }, 41 | abc: 1, 42 | def: { 43 | ghi: { 44 | jkl: {}, 45 | }, 46 | }, 47 | }; 48 | const flatMsg = [ 49 | [["abc"], 1], 50 | [["def", "ghi", "jkl"], {}], 51 | [["hello", "bad"], 5], 52 | [["hello", "good", "bye"], 7], 53 | [["hello", "ugly"], {}], 54 | ]; 55 | expect(JSON.stringify(flatten(msg))).to.equal(JSON.stringify(flatMsg)); 56 | }); 57 | }); 58 | }, 59 | ); 60 | -------------------------------------------------------------------------------- /modules/integration-tests/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "subprojects": ["chai", "chai-dom", "mocha", "sinon", "sinon-chai"] 3 | } 4 | -------------------------------------------------------------------------------- /modules/integration-tests/dist/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Integration Tests 6 | 7 | 8 | 9 |
10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /modules/integration-tests/dist/setup.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global TAP */ 6 | 7 | mocha.setup({ 8 | ui: "bdd", 9 | timeout: 20000, 10 | reporter: TAP, 11 | }); 12 | 13 | console.log(mocha.options); 14 | 15 | window.TESTS = {}; 16 | -------------------------------------------------------------------------------- /modules/integration-tests/dist/tap-reporter.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global Mocha */ 6 | 7 | function TAP(runner) { 8 | Mocha.reporters.HTML.call(this, runner); 9 | 10 | const title = this.title.bind(this); 11 | 12 | // Use websocket to transmit test results to runner 13 | const socket = new WebSocket("ws://localhost:3001"); 14 | let messages = []; 15 | 16 | const WS_OPEN = 1; 17 | const log = (message, mtype = "tap") => { 18 | if (socket.readyState === WS_OPEN) { 19 | socket.send(JSON.stringify({ [mtype]: message })); 20 | if (mtype === "tap") { 21 | log(`${message}\n`, "log"); 22 | } 23 | } else { 24 | messages.push(message); 25 | } 26 | }; 27 | 28 | // Send logging to `fern.js` 29 | this.logChannel = new BroadcastChannel("extlog"); 30 | this.logChannel.onmessage = (msg) => { 31 | const s = `${msg.data.level || "unk"} - ${msg.data.msg || ""}\n`; 32 | log(s, "log"); 33 | }; 34 | 35 | socket.addEventListener("open", () => { 36 | messages.forEach(log); 37 | messages = []; 38 | }); 39 | 40 | let n = 1; 41 | let passes = 0; 42 | let failures = 0; 43 | 44 | runner.on("start", () => { 45 | const total = runner.grepTotal(runner.suite); 46 | log(`1..${total}`); 47 | }); 48 | 49 | runner.on("test end", () => { 50 | n += 1; 51 | }); 52 | 53 | runner.on("pending", (test) => { 54 | log(`ok ${n} ${title(test)} # SKIP -`); 55 | }); 56 | 57 | runner.on("pass", (test) => { 58 | passes += 1; 59 | log(`ok ${n} ${title(test)}`); 60 | }); 61 | 62 | runner.on("fail", (test, err) => { 63 | failures += 1; 64 | log(`not ok ${n} ${title(test)}`); 65 | if (err.stack) { 66 | log(err.stack.replace(/^/gm, " ")); 67 | } 68 | if (err.message) { 69 | log(err.message.replace(/^/gm, " ")); 70 | } 71 | }); 72 | 73 | runner.on("end", () => { 74 | log(`# tests ${passes + failures}`); 75 | log(`# pass ${passes}`); 76 | log(`# fail ${failures}`); 77 | if (socket.readyState === WS_OPEN) { 78 | socket.send(JSON.stringify({ action: "END" })); 79 | } 80 | }); 81 | } 82 | 83 | TAP.prototype = { 84 | ...Mocha.reporters.HTML.prototype, 85 | title(test) { 86 | return test.fullTitle().replace(/#/g, ""); 87 | }, 88 | }; 89 | -------------------------------------------------------------------------------- /modules/integration-tests/sources/initialize-test-helpers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import * as browser from "../core/browser"; 6 | import * as gzip from "../core/gzip"; 7 | import webrequest from "../core/webrequest"; 8 | import * as http from "../core/http"; 9 | import testServer from "../tests/core/http-server"; 10 | import events from "../core/events"; 11 | 12 | export default function (window) { 13 | Object.assign(window.WDP, { 14 | TestHelpers: { 15 | events, 16 | testServer, 17 | browser, 18 | gzip, 19 | http, 20 | webrequest, 21 | }, 22 | }); 23 | } 24 | -------------------------------------------------------------------------------- /modules/integration-tests/sources/run.bundle.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { window, chrome } from "../platform/globals"; 6 | import * as tabs from "../platform/tabs"; 7 | 8 | import { setGlobal } from "../core/kord/inject"; 9 | 10 | import { app, testServer, win } from "../tests/core/integration/helpers"; 11 | 12 | // Include all integration tests in the same bundle 13 | import "../module-integration-tests"; 14 | 15 | setGlobal(app); 16 | 17 | const TEST_URL = window.location.href; 18 | 19 | (function start() { 20 | const tabsListener = new Map(); 21 | 22 | chrome.tabs.onUpdated.addListener((tabId, _, tab) => { 23 | tabsListener.set(tabId, tab.url); 24 | }); 25 | 26 | chrome.tabs.onCreated.addListener((tab) => { 27 | tabsListener.set(tab.id, tab.url); 28 | }); 29 | 30 | chrome.tabs.onRemoved.addListener((tabId) => { 31 | tabsListener.delete(tabId); 32 | }); 33 | 34 | const closeAllTabs = async () => { 35 | const onTabCloseException = () => {}; 36 | 37 | const promises = []; 38 | for (const [tabId, url] of tabsListener) { 39 | // Don't close about pages as they are commonly used to debug state of the browser 40 | if (url !== TEST_URL && url.startsWith("about:") === false) { 41 | promises.push(tabs.closeTab(tabId).catch(onTabCloseException)); 42 | } 43 | } 44 | 45 | if (promises.length === 0) { 46 | return; 47 | } 48 | 49 | await Promise.all(promises); 50 | }; 51 | 52 | // Inject all tests 53 | Object.keys(window.TESTS).forEach((testName) => { 54 | const testFunction = window.TESTS[testName]; 55 | testFunction(); 56 | }); 57 | 58 | // Intercept all telemetry 59 | win.allTelemetry = []; 60 | 61 | before(async () => { 62 | for (const tab of await tabs.query({})) { 63 | tabsListener.set(tab.id, tab.url); 64 | } 65 | 66 | await closeAllTabs(); 67 | await testServer.reset(); 68 | }); 69 | 70 | after(async () => { 71 | await closeAllTabs(); 72 | await testServer.reset(); 73 | }); 74 | 75 | beforeEach(async () => { 76 | win.allTelemetry = []; 77 | }); 78 | 79 | afterEach(async () => { 80 | // Reset global state 81 | win.WDP.TestHelpers.http.resetFetchHandler(); 82 | await testServer.reset(); 83 | if (!win.preventRestarts) { 84 | await closeAllTabs(); 85 | } 86 | }); 87 | 88 | // Check if we should autostart the tests 89 | const searchParams = new window.URLSearchParams(window.location.search); 90 | const autostartParams = searchParams.getAll("autostart"); 91 | const autostart = autostartParams[autostartParams.length - 1]; 92 | 93 | if (autostart === "true") { 94 | mocha.run(); 95 | } 96 | })(); 97 | -------------------------------------------------------------------------------- /modules/web-discovery-project/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "subprojects": [] 3 | } 4 | -------------------------------------------------------------------------------- /modules/web-discovery-project/dist/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/dist/.gitkeep -------------------------------------------------------------------------------- /modules/web-discovery-project/sources/ad-detection.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /** 6 | * Google pagead aclk look like this: 7 | * https://www.googleadservices.com/pagead/aclk?sa=L&ai=DChcSEwjNi5bcsbPWAhUW4BsKHUePBAwYABARGgJ3bA&ohost=www.google.de&cid=CAASEuRo7v8yDlI1j5_Xe3oAtyANqQ&sig=AOD64_0I3As2z06whZRtfqOC3PGdhk9SIQ&ctype=5&q=&ved=0ahUKEwjc7JLcsbPWAhVLuhQKHQWpCRcQ9aACCKIB&adurl= 8 | * 9 | * This function takes such an url and returns a normalized string 10 | * (which is no longer an url). Links to identical ads should be 11 | * normalized to the same string while links to different ads 12 | * should be mapped to different keys. 13 | */ 14 | export function normalizeAclkUrl(url) { 15 | const parts = url.split("aclk?"); 16 | if (parts.length !== 2) { 17 | throw new Error(`Expected Google pagead "aclk" URL. Instead got: ${url}`); 18 | } 19 | 20 | // Ignore the "ved" code, as it seems to change between clicks. 21 | // 22 | // For background information about the "ved" code, see 23 | // https://deedpolloffice.com/blog/articles/decoding-ved-parameter 24 | const noVed = parts[1].replace(/ved=.*&/, ""); 25 | 26 | // TODO: hack, needs to be replaced by a more robust solution 27 | return noVed.replace(/&q=&adurl=$/, "").replace(/&adurl=&q=$/, ""); 28 | } 29 | -------------------------------------------------------------------------------- /modules/web-discovery-project/sources/fallback-dns.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /** 6 | * As WebExtensions do not provide an API to do DNS resolution, 7 | * we can only resolve domains that we have recently seen 8 | * in requests. 9 | */ 10 | export default class { 11 | constructor() { 12 | this._domain2IP = new Map(); 13 | 14 | // By default, cached entries will expire after 5 minutes. 15 | // Note: Expired entries will not immediately be purged, 16 | // but only when "flushExpiredCacheEntries" is called. 17 | this.ttlInMs = 5 * 60 * 1000; 18 | } 19 | 20 | resolveHost(hostname) { 21 | const entry = this._domain2IP.get(hostname); 22 | if (entry) { 23 | return Promise.resolve(entry.ip); 24 | } 25 | return Promise.reject(); 26 | } 27 | 28 | cacheDnsResolution(hostname, ip) { 29 | const now = new Date(); 30 | const ttl = new Date(+now + this.ttlInMs); 31 | this._domain2IP.set(hostname, { ip, ttl }); 32 | } 33 | 34 | flushExpiredCacheEntries() { 35 | const now = new Date(); 36 | 37 | for (const [k, v] of this._domain2IP) { 38 | if (v.ttl <= now) { 39 | this._domain2IP.delete(k); 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /modules/web-discovery-project/sources/html-helpers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import inject from "../core/kord/inject"; 6 | import { DOMParser } from "linkedom"; 7 | 8 | const core = inject.module("core"); 9 | 10 | function getHTML(originalURL) { 11 | return core.action("getHTML", originalURL).then((html) => { 12 | if (html) { 13 | return html; 14 | } 15 | throw new Error(`Failed to get content for tab with url=${originalURL}`); 16 | }); 17 | } 18 | 19 | export function parseHtml(html) { 20 | if (!parseHtml.domParser) { 21 | parseHtml.domParser = new DOMParser(); 22 | } 23 | 24 | return parseHtml.domParser.parseFromString(html, "text/html"); 25 | } 26 | 27 | /** 28 | * @param originalURL URL as seen by the browser 29 | */ 30 | export function getContentDocument(originalURL) { 31 | return getHTML(originalURL).then(parseHtml); 32 | } 33 | -------------------------------------------------------------------------------- /modules/web-discovery-project/sources/logger.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import Logger from "../core/logger"; 6 | 7 | export default Logger.get("wdp", { level: "warning" }); 8 | -------------------------------------------------------------------------------- /modules/web-discovery-project/sources/styles/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/sources/styles/.gitkeep -------------------------------------------------------------------------------- /modules/web-discovery-project/sources/window.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default class Win { 6 | init() {} 7 | 8 | unload() {} 9 | } 10 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/integration/doublefetch-test.es: -------------------------------------------------------------------------------- 1 | import { 2 | app, 3 | expect, 4 | testServer, 5 | } from "../../../tests/core/integration/helpers"; 6 | 7 | export default function () { 8 | // these tests are only for configs with cookies permissions 9 | if (chrome.cookies) { 10 | describe("WebDiscoveryProject doublefetch tests", function () { 11 | const testPath = "/cookie"; 12 | const testCookieName = "COOKIE"; 13 | const testCookieValue = "1234"; 14 | 15 | const getCookie = (testUrl) => 16 | new Promise((resolve) => 17 | chrome.cookies.get( 18 | { 19 | name: testCookieName, 20 | url: testUrl, 21 | }, 22 | resolve, 23 | ), 24 | ); 25 | 26 | beforeEach(async () => { 27 | await testServer.registerPathHandler(testPath, { 28 | result: "

Hello world

40 | new Promise((resolve) => 41 | chrome.cookies.remove( 42 | { 43 | name: testCookieName, 44 | url: testServer.getBaseUrl(testPath), 45 | }, 46 | resolve, 47 | ), 48 | ), 49 | ); 50 | 51 | it("does not cause cookies to be set", async () => { 52 | const wdp = 53 | app.modules["web-discovery-project"].background.webDiscoveryProject; 54 | const testUrl = testServer.getBaseUrl(testPath); 55 | await wdp.doublefetchHandler.anonymousHttpGet(testUrl); 56 | const cookie = await getCookie(testUrl); 57 | expect(cookie).to.be.null; 58 | }); 59 | }); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fallback-dns-test.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* global chai, describeModule */ 6 | 7 | const expect = chai.expect; 8 | const MockDate = require("mockdate"); 9 | 10 | function putClockForward(elapsedTimeInMs) { 11 | const now = new Date(); 12 | MockDate.set(new Date(+now + elapsedTimeInMs)); 13 | } 14 | 15 | export default describeModule( 16 | "web-discovery-project/fallback-dns", 17 | () => ({}), 18 | () => { 19 | describe("FallbackDns", function () { 20 | let uut; 21 | 22 | const someHost = "example.test"; 23 | const someIp = "127.0.0.1"; 24 | 25 | beforeEach(function () { 26 | const FallbackDns = this.module().default; 27 | uut = new FallbackDns(); 28 | 29 | MockDate.set(new Date("2000-01-01")); 30 | }); 31 | 32 | afterEach(function () { 33 | MockDate.reset(); 34 | }); 35 | 36 | it("should initially be empty", function () { 37 | return expect(uut.resolveHost(someHost)).to.be.rejected; 38 | }); 39 | 40 | it("should find cached value", function () { 41 | uut.cacheDnsResolution("example1.test", "127.0.0.1"); 42 | uut.cacheDnsResolution("example2.test", "127.0.0.2"); 43 | return Promise.all([ 44 | expect(uut.resolveHost("example1.test")).to.eventually.equal( 45 | "127.0.0.1", 46 | ), 47 | expect(uut.resolveHost("example2.test")).to.eventually.equal( 48 | "127.0.0.2", 49 | ), 50 | ]); 51 | }); 52 | 53 | it("should evict old entries", function () { 54 | uut.cacheDnsResolution(someHost, someIp); 55 | return expect(uut.resolveHost(someHost)).to.be.fulfilled.then(() => { 56 | // TTL is still not exceeded 57 | putClockForward(uut.ttlInMs - 1); 58 | uut.flushExpiredCacheEntries(); 59 | 60 | return expect(uut.resolveHost(someHost)).to.be.fulfilled.then(() => { 61 | // Now the TTL is exceeded and the cache should be cleared 62 | putClockForward(1); 63 | uut.flushExpiredCacheEntries(); 64 | return expect(uut.resolveHost(someHost)).to.be.rejected; 65 | }); 66 | }); 67 | }); 68 | }); 69 | }, 70 | ); 71 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/am/fussballschuh-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/am/fussballschuh-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/am/gardening-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/am/gardening-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/am/test-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/am/test-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/bing/donald-trump-2018-05-28/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/bing/donald-trump-2018-05-28/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/bing/donald-trump-2018-05-28/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://www.bing.com/search?q=donald+trump&qs=n&form=QBLH&sp=-1&pq=donald+trump&sc=0-0&sk=&cvid=264D041856E64CF482FCA2496A83960B", 3 | 4 | "mustContain": [ 5 | { 6 | "type": "wdp", 7 | "action": "query", 8 | "payload": { 9 | "r": { 10 | "0": { 11 | "t": "Donald Trump – Wikipedia", 12 | "u": "https://de.wikipedia.org/wiki/Donald_Trump" 13 | }, 14 | "1": { 15 | "t": "Donald Trump - SPIEGEL ONLINE", 16 | "u": "http://www.spiegel.de/thema/donald_trump/" 17 | }, 18 | "2": { 19 | "t": "Donald Trump: Präsident der USA | ZEIT ONLINE", 20 | "u": "https://www.zeit.de/thema/donald-trump" 21 | }, 22 | "3": { 23 | "t": "Donald Trump - Aktuelle Beiträge zum US …", 24 | "u": "https://www.tagesspiegel.de/themen/donald-trump/" 25 | }, 26 | "4": { 27 | "t": "Donald Trump | STERN.de", 28 | "u": "https://www.stern.de/lifestyle/leute/themen/donald-trump-4540962.html" 29 | }, 30 | "5": { 31 | "t": "Donald J. Trump (@realDonaldTrump) | Twitter", 32 | "u": "https://twitter.com/realdonaldtrump" 33 | }, 34 | "6": { 35 | "t": "Donald Trump - News von WELT", 36 | "u": "https://www.welt.de/themen/donald-trump/" 37 | }, 38 | "7": { 39 | "t": "Donald Trump | Us-news | The Guardian", 40 | "u": "https://www.theguardian.com/us-news/donaldtrump" 41 | }, 42 | "8": { 43 | "t": "Donald J. Trump - Official Site", 44 | "u": "https://www.donaldjtrump.com/" 45 | }, 46 | "9": { 47 | "t": "Donald Trump - Wikipedia", 48 | "u": "https://en.wikipedia.org/wiki/Donald_Trump" 49 | } 50 | }, 51 | "q": "donald trump", 52 | "qurl": "https://www.bing.com/search?q=donald+trump&qs=n&form=QBLH&sp=-1&pq=donald+trump&sc=0-0&sk=&cvid=264D041856E64CF482FCA2496A83960B", 53 | "ctry": "de" 54 | } 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/bing/img-search-2020-02-23/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/bing/img-search-2020-02-23/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/empty-page/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/empty-page/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/empty-page/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://some.empty.page.test", 3 | 4 | // Given a page without content, it should be safe to assume 5 | // that we want to send any messages for it. 6 | "mustNotContain": [{ "action": "*" }] 7 | } 8 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/angela-merkel-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/angela-merkel-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/cricket-icc-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/cricket-icc-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/cricket-icc-2023-10-10/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://www.google.com/search?q=cricket%20icc&gl=us&hl=en", 3 | "mustContain": [ 4 | { 5 | "type": "wdp", 6 | "action": "widgetTitle", 7 | "payload": { 8 | "q": "cricket icc", 9 | "widgetTitle": "ICC Cricket World Cup", 10 | "ctry": "de" 11 | } 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/elon-musk-twitter-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/elon-musk-twitter-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/f1-2023-09-27/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/f1-2023-09-27/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/f1-2023-09-27/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://www.google.com/search?q=f1&gl=us&hl=en", 3 | "mustContain": [ 4 | { 5 | "type": "wdp", 6 | "action": "widgetTitle", 7 | "payload": { 8 | "q": "f1", 9 | "widgetTitle": "Formula 1", 10 | "ctry": "de" 11 | } 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/goetze-wm-2014-tor-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/goetze-wm-2014-tor-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/goetze-wm-2014-tor-2023-10-10/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://www.google.com/search?q=goetze%20wm%202014%20tor&gl=us&hl=en", 3 | "mustContain": [ 4 | { 5 | "type": "wdp", 6 | "action": "query", 7 | "payload": { 8 | "r": { 9 | "0": { 10 | "t": "Mario Götze WM 2014 Finale Tor - YouTube", 11 | "u": "https://www.youtube.com/watch?v=TEWcZ-6l10U", 12 | "age": null, 13 | "m": null 14 | }, 15 | "1": { 16 | "t": "Mario Götze WM 2014 Finale Tor in 4K/UHD 13.07.2014", 17 | "u": "https://www.youtube.com/watch?v=YiWllGdMITE", 18 | "age": null, 19 | "m": null 20 | }, 21 | "2": { 22 | "t": "WM FINALE 2014 - Tor Götze ! ( 5 TV Kommentatoren )", 23 | "u": "https://www.youtube.com/watch?v=t9xm_Ingtuc", 24 | "age": null, 25 | "m": null 26 | }, 27 | "3": { 28 | "t": "WM 2014 Finale Tor Mario Götze - YouTube", 29 | "u": "https://www.youtube.com/watch?v=j_guXT4I-AQ", 30 | "age": null, 31 | "m": null 32 | }, 33 | "4": { 34 | "t": "Gotze Goal vs Argentina | 2014 World Cup Final - YouTube", 35 | "u": "https://www.youtube.com/watch?v=lu_0zOUTdPE", 36 | "age": null, 37 | "m": null 38 | }, 39 | "5": { 40 | "t": "Mario Götze Germany vs Argentina 2014 FIFA World Cup Goal.", 41 | "u": "https://www.youtube.com/watch?v=1QJ6P8Yxil4", 42 | "age": null, 43 | "m": "tor" 44 | }, 45 | "6": { 46 | "t": "WM 2014 Finale Tor Mario Götze - YouTube", 47 | "u": "https://www.youtube.com/watch?v=AZ-AzW8Y06w", 48 | "age": null, 49 | "m": null 50 | }, 51 | "7": { 52 | "t": "WM-Finale 2014 - Tor von Mario Götze - YouTube", 53 | "u": "https://www.youtube.com/watch?v=g67hj4VB8z0", 54 | "age": null, 55 | "m": "goetze" 56 | }, 57 | "8": { 58 | "t": "Mario Götzes WM-Siegtreffer ist das \"Tor des Jahrzehnts\"", 59 | "u": "https://www.dfb.de/news/detail/mario-goetzes-wm-siegtreffer-ist-das-tor-des-jahrzehnts-215608/", 60 | "age": null, 61 | "m": null 62 | }, 63 | "9": { 64 | "t": "WM 2014: Das Tor, das ewig bleibt - Sport", 65 | "u": "https://www.sueddeutsche.de/sport/goetze-schuerrle-wm-2014-tor-finale-1.4971888", 66 | "age": null, 67 | "m": null 68 | } 69 | }, 70 | "q": "goetze wm 2014 tor", 71 | "qurl": "https://www.google.com/search?q=goetze%20wm%202014%20tor&gl=us&hl=en", 72 | "ctry": "de" 73 | } 74 | } 75 | ] 76 | } -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/green-apple-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/green-apple-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/grossformat-laserdrucker-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/grossformat-laserdrucker-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/konrad-zuse-bilder-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/konrad-zuse-bilder-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/kuckucksuhr-kaufen-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/kuckucksuhr-kaufen-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/kuckucksuhr-kaufen-2023-10-10/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://www.google.com/search?q=kuckucksuhr%20kaufen&gl=us&hl=en", 3 | "mustContain": [ 4 | { 5 | "type": "wdp", 6 | "action": "query", 7 | "payload": { 8 | "r": { 9 | "0": { 10 | "t": "Kuckucksuhren » jetzt online kaufen | uhren-park.de", 11 | "u": "https://www.uhren-park.de/index.php/cat/c1_Kuckucksuhren.html", 12 | "age": null, 13 | "m": null 14 | }, 15 | "1": { 16 | "t": "Original Kuckucksuhren aus dem Schwarzwald", 17 | "u": "https://www.kuckucksuhr.net/", 18 | "age": null, 19 | "m": null 20 | }, 21 | "2": { 22 | "t": "Kuckucksuhren aus dem Schwarzwald", 23 | "u": "https://www.schwarzwaldpalast.de/", 24 | "age": null, 25 | "m": null 26 | }, 27 | "3": { 28 | "t": "Original Kuckucksuhren aus dem Schwarzwald", 29 | "u": "https://www.hausder1000uhren.de/", 30 | "age": null, 31 | "m": null 32 | }, 33 | "4": { 34 | "t": "Kuckucksuhren / Spezialuhren: Küche, Haushalt & Wohnen", 35 | "u": "https://www.amazon.de/Kuckucksuhren/b?ie=UTF8&node=2970892031", 36 | "age": null, 37 | "m": null 38 | }, 39 | "5": { 40 | "t": "Kuckucksuhren bestellen | Große Auswahl an ...", 41 | "u": "https://www.haus-der-schwarzwalduhren.de/", 42 | "age": null, 43 | "m": null 44 | }, 45 | "6": { 46 | "t": "Kuckucksuhren online kaufen", 47 | "u": "https://www.ebay.de/b/Kuckucksuhren/79644/bn_2394739", 48 | "age": null, 49 | "m": null 50 | }, 51 | "7": { 52 | "t": "Exklusive Kuckucksuhren Unikate", 53 | "u": "https://www.kuckucksuhr.com/", 54 | "age": null, 55 | "m": null 56 | }, 57 | "8": { 58 | "t": "Kuckucksuhren günstig online kaufen", 59 | "u": "https://www.kaufland.de/wanduhren/kuckucksuhren/", 60 | "age": null, 61 | "m": null 62 | }, 63 | "9": { 64 | "t": "Kuckucksuhren aus dem Schwarzwald", 65 | "u": "https://kuckucksuhren.shop/", 66 | "age": null, 67 | "m": null 68 | } 69 | }, 70 | "q": "kuckucksuhr kaufen", 71 | "qurl": "https://www.google.com/search?q=kuckucksuhr%20kaufen&gl=us&hl=en", 72 | "ctry": "de" 73 | } 74 | } 75 | ] 76 | } -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/nba-2023-09-27/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/nba-2023-09-27/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/nba-2023-09-27/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://www.google.com/search?q=nba&gl=us&hl=en", 3 | "mustContain": [ 4 | { 5 | "type": "wdp", 6 | "action": "widgetTitle", 7 | "payload": { 8 | "q": "nba", 9 | "widgetTitle": "NBA", 10 | "ctry": "de" 11 | } 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/nirvana-polly-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/nirvana-polly-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/nirvana-polly-2023-10-10/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://www.google.com/search?q=nirvana%20polly&gl=us&hl=en", 3 | "mustContain": [ 4 | { 5 | "type": "wdp", 6 | "action": "video-p", 7 | "payload": { 8 | "q": "nirvana polly", 9 | "qurl": "https://www.google.com/search?q=nirvana%20polly&gl=us&hl=en", 10 | "r": { 11 | "0": { 12 | "t": "Nirvana - Polly (Audio)", 13 | "u": "https://www.youtube.com/watch?v=DrlaVYKWeLU" 14 | } 15 | }, 16 | "ctry": "de" 17 | } 18 | }, 19 | { 20 | "type": "wdp", 21 | "action": "videos-p", 22 | "payload": { 23 | "q": "nirvana polly", 24 | "qurl": "https://www.google.com/search?q=nirvana%20polly&gl=us&hl=en", 25 | "r": { 26 | "0": { 27 | "t": "Nirvana - Polly (Live On MTV Unplugged, 1993 / Unedited)", 28 | "u": "https://www.youtube.com/watch?v=3H0NHHKBemg" 29 | }, 30 | "1": { 31 | "t": "Nirvana - Polly (Live On MTV Unplugged, 1993 / Rehearsal)", 32 | "u": "https://www.youtube.com/watch?v=ZSEoYDA2txw" 33 | }, 34 | "2": { 35 | "t": "Nirvana - Polly (Live At The Paramount, Seattle / 1991)", 36 | "u": "https://www.youtube.com/watch?v=abBgsNx85mI" 37 | }, 38 | "3": { 39 | "t": "Polly", 40 | "u": "https://www.google.com/#" 41 | } 42 | }, 43 | "ctry": "de" 44 | } 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/passauer-dom-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/passauer-dom-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/sneezing-panda-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/sneezing-panda-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/sq-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/sq-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/trump-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/trump-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/von-der-leyen-2023-10-10/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/go/von-der-leyen-2023-10-10/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/yahoo/donald-trump-2018-05-28/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-extractor/yahoo/donald-trump-2018-05-28/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-extractor/yahoo/donald-trump-2018-05-28/scenario.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://de.search.yahoo.com/search?p=donald+trump&fr=yfp-t&fp=1&toggle=1&cop=mss&ei=UTF-8", 3 | 4 | "mustContain": [ 5 | { 6 | "type": "wdp", 7 | "action": "query", 8 | "payload": { 9 | "r": { 10 | "0": { 11 | "t": "Donald Trump – Wikipedia", 12 | "u": "https://de.wikipedia.org/wiki/Donald_Trump" 13 | }, 14 | "1": { 15 | "t": "Donald Trump - SPIEGEL ONLINE", 16 | "u": "http://www.spiegel.de/thema/donald_trump/" 17 | }, 18 | "2": { 19 | "t": "Donald Trump: Präsident der USA | ZEIT ONLINE", 20 | "u": "https://www.zeit.de/thema/donald-trump" 21 | }, 22 | "3": { 23 | "t": "Donald Trump - Aktuelle Beiträge zum US-Präsidenten", 24 | "u": "https://www.tagesspiegel.de/themen/donald-trump/" 25 | }, 26 | "4": { 27 | "t": "Donald J. Trump (@realDonaldTrump) | Twitter", 28 | "u": "https://twitter.com/realdonaldtrump" 29 | }, 30 | "5": { 31 | "t": "Donald Trump | STERN.de", 32 | "u": "https://www.stern.de/lifestyle/leute/themen/donald-trump-4540962.html" 33 | }, 34 | "6": { 35 | "t": "Donald Trump- Steckbrief, News, Bilder | GALA.de", 36 | "u": "https://www.gala.de/stars/starportraets/donald-trump-20526492.html" 37 | }, 38 | "7": { 39 | "t": "Donald Trump - News von WELT", 40 | "u": "https://www.welt.de/themen/donald-trump/" 41 | }, 42 | "8": { 43 | "t": "Donald Trump | Us-news | The Guardian", 44 | "u": "https://www.theguardian.com/us-news/donaldtrump" 45 | }, 46 | "9": { 47 | "t": "Donald Trump - Wikipedia", 48 | "u": "https://en.wikipedia.org/wiki/Donald_Trump" 49 | } 50 | }, 51 | "q": "donald trump", 52 | "qurl": "https://de.search.yahoo.com/search?p=donald+trump&fr=yfp-t&fp=1&toggle=1&cop=mss&ei=UTF-8", 53 | "ctry": "de" 54 | } 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/android-user-agent-page-with-ads-2019-07-24/expected-ads.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/android-user-agent-page-with-ads-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/android-user-agent-page-with-ads-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/android-user-agent-page-without-ads-2019-07-24/expected-ads.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/android-user-agent-page-without-ads-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/android-user-agent-page-without-ads-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/coffee-ads-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/coffee-ads-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/flight-page-2019-07-24/expected-ads.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": "sa=L&ai=DChcSEwiQ9_brpc3jAhVY4ncKHbErArwYABAAGgJlZg&ohost=www.google.com&cid=CAASEuRoYMeSA9WNOTUpnvFvCXWP4w&sig=AOD64_3CGt05X959rCr_MfeWNnKvOmi9dg", 4 | "url": "https://www.omio.com/lps/flight-paris-london?id=Y29ubmVjdGlvbl9wYWdlLmNvbS5mbGlnaHQuMzc5NzI3LjM4MDU1Mw==&utm_source=google&utm_medium=cpc&utm_campaign=SEM_EU_EN_Flight_All_All_I.III_%5BFlight_01_Rou%3APro_0%5D_P%3AFlight&utm_term=flight+paris+london&adgroup=SEM_EU_EN_Flight_All_All_I.III_%5BFlight_01_Rou%3APro_0%5D_P%3AFlight_(Paris_379727_FR)_(London_380553_UK)&content=basic&h=bHBzLmguc2VtLmkucm91LmZsaWdodF8wMS4u&def_sort=s&easy=1" 5 | }, 6 | { 7 | "key": "sa=L&ai=DChcSEwiQ9_brpc3jAhVY4ncKHbErArwYABABGgJlZg&ohost=www.google.com&cid=CAASEuRoYMeSA9WNOTUpnvFvCXWP4w&sig=AOD64_3UXQRLer4WvQPUyRKERDNtHo9CQw", 8 | "url": "https://www.edreams.com/flights/paris-london/PAR/LON/" 9 | }, 10 | { 11 | "key": "sa=L&ai=DChcSEwiQ9_brpc3jAhVY4ncKHbErArwYABAHGgJlZg&ohost=www.google.com&cid=CAASEuRoYMeSA9WNOTUpnvFvCXWP4w&sig=AOD64_2Figzklen1_PTJBby2rul3X8EzEQ", 12 | "url": "https://www.opodo.com/flights/paris-london/PAR/LON/" 13 | }, 14 | { 15 | "key": "sa=L&ai=DChcSEwiQ9_brpc3jAhVY4ncKHbErArwYABAIGgJlZg&ohost=www.google.com&cid=CAASEuRoYMeSA9WNOTUpnvFvCXWP4w&sig=AOD64_0fTy55m2vCCBszylANkQAxLBT9mQ", 16 | "url": "https://www.airfrance.de/DE/de/local/process/standardbooking/SearchAction.do?multidest=true" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/flight-page-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/flight-page-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/gardening-shoes-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/gardening-shoes-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/page-with-no-ads-2019-07-24/expected-ads.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/page-with-no-ads-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/page-with-no-ads-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/potato-ads-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/potato-ads-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/shoe-ads-2019-07-24/expected-ads.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABAAGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_3Eb2Zj5w6Z0hHBW--hAqJ_JTLWPg", 4 | "url": "https://www.mirapodo.de/schuhe/" 5 | }, 6 | { 7 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABAFGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_3tIw-RP_Zt9w07e5AXf61tgjY6vA", 8 | "url": "https://www.zalando.de/katalog/" 9 | }, 10 | { 11 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABAKGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_2r3fZn4Wz31k5QX5S2eRXGCexa3A", 12 | "url": "https://www.mytheresa.com/de-de/sale/shoes.html" 13 | }, 14 | { 15 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABAYGgJ3cw&sig=AOD64_1d6Fp4kr6nhlb_71RJEOAn7x0HnQ&ctype=5", 16 | "url": "https://www.amazon.de/adidas-Yeezy-Boost-Inertia-Runner/dp/B07NLK71KX?source=ps-sl-shoppingads-lpcontext&psc=1?source=ps-sl-shoppingads-lpcontext&psc=1" 17 | }, 18 | { 19 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABAUGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_2Z1YsBC_TUIsbybs-9vB4qLxKkUw&ctype=5", 20 | "url": "https://www.eurotops.de/luxus-lederhausschuh-braun-40-40-braun-38453-40.html" 21 | }, 22 | { 23 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABARGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_1xfWbWOJ01kwDpgRexkwdSYmS_ng&ctype=5", 24 | "url": "https://www.zalando.de/zalando-essentials-schnuerer-black-za812m004-q11.html?size=46" 25 | }, 26 | { 27 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABASGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_0_QL0NuA0LeUQjmEYpeR6S08tluQ&ctype=5", 28 | "url": "https://www.mytheresa.com/de-de/givenchy-urban-street-leather-sneakers-1129197.html" 29 | }, 30 | { 31 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABAZGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_0LbYAKYm9-JnjCnyrIivOi978t9Q&ctype=5", 32 | "url": "https://www.mytheresa.com/de-de/stella-mccartney-eclypse-sneakers-1126288.html" 33 | }, 34 | { 35 | "key": "sa=L&ai=DChcSEwig_su1n83jAhWT4VEKHXbHAe4YABATGgJ3cw&ohost=www.google.de&cid=CAASE-RoeXiYS2-c-wDcPHiDfPhulyE&sig=AOD64_0OkI3an4Uy4ow60XE3zGwh2d6yuQ&ctype=5", 36 | "url": "https://www.brooksrunning.com/de_de/brooks-running-shoes-ricochet-le-womens/1202921B030.110.html" 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/fixtures/content-test/shoe-ads-2019-07-24/page.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/modules/web-discovery-project/tests/unit/fixtures/content-test/shoe-ads-2019-07-24/page.html.gz -------------------------------------------------------------------------------- /modules/web-discovery-project/tests/unit/generate-fixtures.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | yarn build-module 4 | # Remove "webextension-polyfill" references which require an app to be run as a web-extension 5 | find ../../../../build/platform -name "*.js" -type f \ 6 | | xargs -I {} sed -i "" "s/require(\"webextension-polyfill\")/null/g" {} 7 | ./generate-fixtures.js 8 | -------------------------------------------------------------------------------- /modules/webextension-specific/sources/background.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import runtime from "../platform/runtime"; 6 | import { chrome } from "../platform/globals"; 7 | import events from "../core/events"; 8 | import background from "../core/base/background"; 9 | 10 | /** 11 | @namespace webextension-specific 12 | @module webextension-specific 13 | @class Background 14 | */ 15 | export default background({ 16 | init() { 17 | this.onMessage = (message) => { 18 | if (message.name === "appReady") { 19 | return Promise.resolve({ ready: true }); 20 | } 21 | 22 | return undefined; 23 | }; 24 | runtime.onMessage.addListener(this.onMessage); 25 | 26 | this.onTabSelect = ({ tabId }) => { 27 | chrome.tabs.get(tabId, (tabInfo) => 28 | events.pub("core:tab_select", { ...tabInfo, tabId }), 29 | ); 30 | }; 31 | this.onTabClose = (tabId, removeInfo) => 32 | events.pub("core:tab_close", { tabId, ...removeInfo }); 33 | this.onTabOpen = (tabInfo) => events.pub("core:tab_open", tabInfo); 34 | 35 | chrome.tabs.onActivated.addListener(this.onTabSelect); 36 | chrome.tabs.onRemoved.addListener(this.onTabClose); 37 | chrome.tabs.onCreated.addListener(this.onTabOpen); 38 | }, 39 | 40 | unload() { 41 | runtime.onMessage.removeListener(this.onMessage); 42 | chrome.tabs.onActivated.removeListener(this.onTabSelect); 43 | chrome.tabs.onRemoved.removeListener(this.onTabClose); 44 | chrome.tabs.onCreated.removeListener(this.onTabOpen); 45 | }, 46 | 47 | events: {}, 48 | 49 | actions: {}, 50 | }); 51 | -------------------------------------------------------------------------------- /modules/webrequest-pipeline/sources/logger.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import Logger from "../core/logger"; 6 | 7 | export default Logger.get("webrequest-pipeline", { level: "warning" }); 8 | -------------------------------------------------------------------------------- /modules/webrequest-pipeline/sources/page.ts: -------------------------------------------------------------------------------- 1 | export const PAGE_LOADING_STATE = { 2 | CREATED: "created", 3 | NAVIGATING: "navigating", 4 | COMMITTED: "committed", 5 | LOADED: "loaded", 6 | COMPLETE: "complete", 7 | }; 8 | 9 | type Frame = { 10 | parentFrameId: number; 11 | url?: string; 12 | }; 13 | 14 | interface FirefoxWebRequestDetails extends chrome.webRequest.WebRequestDetails { 15 | originUrl?: string; 16 | documentUrl?: string; 17 | } 18 | 19 | export default class Page { 20 | id: number; 21 | url?: string; 22 | isRedirect: boolean; 23 | isPrivate: boolean; 24 | created: number; 25 | destroyed: number | null; 26 | lastRequestId: number | null; 27 | frames: Map; 28 | state: string; 29 | activeTime: number; 30 | activeFrom: number; 31 | counter: number; 32 | 33 | previous?: Page; 34 | 35 | constructor({ id, active, url, incognito }: chrome.tabs.Tab) { 36 | this.id = id || 0; 37 | this.url = url; 38 | this.isRedirect = false; 39 | this.isPrivate = incognito; 40 | this.created = Date.now(); 41 | this.destroyed = null; 42 | this.lastRequestId = null; 43 | this.frames = new Map([ 44 | [ 45 | 0, 46 | { 47 | parentFrameId: -1, 48 | url, 49 | }, 50 | ], 51 | ]); 52 | this.state = PAGE_LOADING_STATE.CREATED; 53 | 54 | this.activeTime = 0; 55 | this.activeFrom = active ? Date.now() : 0; 56 | 57 | this.counter = 0; 58 | } 59 | 60 | setActive(active: boolean) { 61 | if (active && this.activeFrom === 0) { 62 | this.activeFrom = Date.now(); 63 | } else if (!active && this.activeFrom > 0) { 64 | this.activeTime += Date.now() - this.activeFrom; 65 | this.activeFrom = 0; 66 | } 67 | } 68 | 69 | updateState(newState: string) { 70 | this.state = newState; 71 | } 72 | 73 | stage() { 74 | this.setActive(false); 75 | this.destroyed = Date.now(); 76 | // unset previous (to prevent history chain memory leak) 77 | this.previous = undefined; 78 | } 79 | 80 | /** 81 | * Return the URL of top-level document (i.e.: tab URL). 82 | */ 83 | getTabUrl(): string | undefined { 84 | return this.url; 85 | } 86 | 87 | /** 88 | * Return the URL of the frame. 89 | */ 90 | getFrameUrl(context: FirefoxWebRequestDetails): string | undefined { 91 | const { frameId } = context; 92 | 93 | const frame = this.frames.get(frameId); 94 | 95 | // In some cases, frame creation does not trigger a webRequest event (e.g.: 96 | // if the iframe is specified in the HTML of the page directly). In this 97 | // case we try to fall-back to something else: documentUrl, originUrl, 98 | // initiator. 99 | if (frame === undefined) { 100 | return context.documentUrl || context.originUrl || context.initiator; 101 | } 102 | 103 | return frame.url; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /patches/broccoli+3.5.2.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/broccoli/dist/load_brocfile.js b/node_modules/broccoli/dist/load_brocfile.js 2 | index 6af31e5..76bae29 100644 3 | --- a/node_modules/broccoli/dist/load_brocfile.js 4 | +++ b/node_modules/broccoli/dist/load_brocfile.js 5 | @@ -4,8 +4,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) { 6 | }; 7 | const path_1 = __importDefault(require("path")); 8 | const findup_sync_1 = __importDefault(require("findup-sync")); 9 | -const esm_1 = __importDefault(require("esm")); 10 | -const esmRequire = esm_1.default(module); 11 | /** 12 | * Require a brocfile via either ESM or TypeScript 13 | * 14 | @@ -34,7 +32,7 @@ function requireBrocfile(brocfilePath) { 15 | } 16 | else { 17 | // Load brocfile via esm shim 18 | - brocfile = esmRequire(brocfilePath); 19 | + brocfile = require(brocfilePath); 20 | } 21 | // ESM `export default X` is represented as module.exports = { default: X } 22 | // eslint-disable-next-line no-prototype-builtins 23 | -------------------------------------------------------------------------------- /platforms/webextension/console.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { window } from "./globals"; 6 | 7 | export default (window && window.console) || console; 8 | -------------------------------------------------------------------------------- /platforms/webextension/content/globals.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const newChrome = typeof browser !== "undefined" ? browser : chrome; 6 | const newWindow = window; 7 | 8 | export { newChrome as chrome, newWindow as window }; 9 | -------------------------------------------------------------------------------- /platforms/webextension/crypto.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default crypto; 6 | -------------------------------------------------------------------------------- /platforms/webextension/fetch.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import "abortcontroller-polyfill/dist/abortcontroller-polyfill-only"; 6 | import window from "./globals-window"; 7 | import chrome from "./globals-chrome"; 8 | 9 | const { fetch, Headers, Request, Response, AbortController } = window; 10 | 11 | /** 12 | * Note: On Chrome, chrome.runtime.id is the same, but 13 | * on Firefox, they distinguish between the extension ID 14 | * (e.g., "firefox@brave.com") and the internal UUID 15 | * (e.g., "0b3696ed-9250-44a9-93d3-0c43caea7b3b"). 16 | * 17 | * On Firefox, the UUID is by user. That means different users 18 | * of the same extension will have different (but fixed) UUIDs. 19 | */ 20 | function getExtensionUuid() { 21 | try { 22 | const id = chrome.i18n.getMessage("@@extension_id"); 23 | if (id) { 24 | return id; 25 | } 26 | } catch (e) { 27 | // fallback 28 | } 29 | 30 | return chrome.runtime.id; 31 | } 32 | const extensionUuid = getExtensionUuid(); 33 | 34 | const blacklistedOriginHeaders = [ 35 | `moz-extension://${extensionUuid}`, 36 | `chrome-extension://${extensionUuid}`, 37 | ]; 38 | 39 | /** 40 | * Detects origin headers automatically added by the browser. 41 | * We have to detect them, so we can later remove them using the 42 | * webrequestAPI (see webrequest-pipeline/fetch-sanitizer.es). 43 | * 44 | * Warning: We have to be sure only to detect requests made from our 45 | * extension. Otherwise, we break other extensions that are relying on 46 | * the origin header. 47 | */ 48 | function isTrackableOriginHeaderFromOurExtension(value) { 49 | return blacklistedOriginHeaders.some((x) => value === x); 50 | } 51 | 52 | export { 53 | fetch as default, 54 | fetch as fetchArrayBuffer, 55 | fetch, 56 | Headers, 57 | Request, 58 | Response, 59 | AbortController, 60 | isTrackableOriginHeaderFromOurExtension, 61 | }; 62 | -------------------------------------------------------------------------------- /platforms/webextension/globals-chrome.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import window from "./globals-window"; 6 | 7 | export default window !== undefined ? window.chrome || {} : {}; 8 | -------------------------------------------------------------------------------- /platforms/webextension/globals-window.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default typeof window !== "undefined" ? window : {}; 6 | -------------------------------------------------------------------------------- /platforms/webextension/globals.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import browser from "webextension-polyfill"; 6 | 7 | import window from "./globals-window"; 8 | import chrome from "./globals-chrome"; 9 | 10 | export { chrome, window, browser }; 11 | 12 | export function isContentScriptsSupported() { 13 | return ( 14 | window !== undefined && 15 | typeof window.browser !== "undefined" && 16 | window.browser.contentScripts 17 | ); 18 | } 19 | -------------------------------------------------------------------------------- /platforms/webextension/gzip.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import pako from "pako"; 6 | 7 | /** 8 | * @param {!string} uncompressed input 9 | * @returns {Uint8Array} gzipped output 10 | */ 11 | export function compress(str) { 12 | return pako.gzip(str); 13 | } 14 | 15 | /** 16 | * @param {!string} gzipped input 17 | * @returns {string} uncompressed output 18 | */ 19 | export function decompress(str) { 20 | return pako.gunzip(str, { to: "string" }); 21 | } 22 | -------------------------------------------------------------------------------- /platforms/webextension/history-service.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { browser } from "./globals"; 6 | 7 | export default browser.history; 8 | -------------------------------------------------------------------------------- /platforms/webextension/integration-tests/test-launcher.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { chrome } from "../globals"; 6 | 7 | export default function (indexFilePath, { grep, autostart, invert, retries }) { 8 | let url = `/modules/${indexFilePath}?autostart=${autostart}`; 9 | 10 | if (invert === "true") { 11 | url = `${url}&invert=true`; 12 | } 13 | 14 | if (grep) { 15 | url = `${url}&grep=${grep}`; 16 | } 17 | 18 | if (retries) { 19 | url = `${url}&retries=${grep}`; 20 | } 21 | 22 | return chrome.runtime.getURL(url); 23 | } 24 | -------------------------------------------------------------------------------- /platforms/webextension/integration-tests/test-page-sources.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export const testPageSources = { 6 | "pages": [ 7 | { 8 | "helloworld.com/hello": 9 | ` 10 | 11 | Hello 12 | 13 | 14 |

Hello World, hello page.

15 | 16 | ` 17 | }, 18 | { 19 | "helloworld.com/about": 20 | ` 21 | 22 | About 23 | 24 | 25 |

Hello World, about page.

26 | 27 | ` 28 | } 29 | ], 30 | } -------------------------------------------------------------------------------- /platforms/webextension/lib/dexie.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import Dexie from "dexie"; 6 | 7 | export default function () { 8 | return Promise.resolve(Dexie); 9 | } 10 | -------------------------------------------------------------------------------- /platforms/webextension/lib/tldts.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export * from "tldts-experimental"; 6 | -------------------------------------------------------------------------------- /platforms/webextension/lib/zlib.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import pako from "pako"; 6 | 7 | const inflate = pako.inflate.bind(pako); 8 | const deflate = pako.deflate.bind(pako); 9 | 10 | export { inflate, deflate }; 11 | -------------------------------------------------------------------------------- /platforms/webextension/location-change-observer.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import events from "../core/events"; 6 | import { chrome } from "./globals"; 7 | 8 | export default class LocationChangeObserver { 9 | constructor() { 10 | this.onLocationChangeHandler = ({ 11 | tabId, 12 | url, 13 | frameId, 14 | transitionType, 15 | }) => { 16 | // We should only forward main_document URLs for on-location change. 17 | if (frameId !== 0) { 18 | return; 19 | } 20 | 21 | // We need to check if the on-location change happened in a private tab. 22 | // Modules like web-discovery-project should not collect data about sites 23 | // visited in private tab. 24 | chrome.tabs.get(tabId, (tab) => { 25 | if (chrome.runtime.lastError) { 26 | return; 27 | } 28 | const { incognito: isPrivate, active, pinned } = tab; 29 | events.pub("content:location-change", { 30 | active, 31 | frameId, 32 | isPrivate, 33 | pinned, 34 | tabId, 35 | transitionType, 36 | url, 37 | windowId: tabId, 38 | windowTreeInformation: { tabId }, 39 | }); 40 | }); 41 | }; 42 | } 43 | 44 | init() { 45 | chrome.webNavigation.onCommitted.addListener(this.onLocationChangeHandler); 46 | chrome.webNavigation.onHistoryStateUpdated.addListener( 47 | this.onLocationChangeHandler 48 | ); 49 | } 50 | 51 | unload() { 52 | chrome.webNavigation.onCommitted.removeListener( 53 | this.onLocationChangeHandler 54 | ); 55 | chrome.webNavigation.onHistoryStateUpdated.removeListener( 56 | this.onLocationChangeHandler 57 | ); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /platforms/webextension/permission-manager.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { chrome } from "./globals"; 6 | 7 | const PERMISSIONS = { 8 | WEB_REQUEST: "webRequest", 9 | WEB_REQUEST_BLOCKING: "webRequestBlocking", 10 | }; 11 | 12 | export default { 13 | PERMISSIONS, 14 | contains: (permissions) => 15 | new Promise((resolve) => { 16 | chrome.permissions.contains({ permissions }, resolve); 17 | }), 18 | }; 19 | -------------------------------------------------------------------------------- /platforms/webextension/platform.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { chrome } from "./globals"; 6 | 7 | function checkUserAgent(pattern) { 8 | try { 9 | return navigator.userAgent.indexOf(pattern) !== -1; 10 | } catch (e) { 11 | return false; 12 | } 13 | } 14 | 15 | const def = { 16 | isFirefox: checkUserAgent("Firefox"), 17 | isChromium: checkUserAgent("Chrome"), 18 | isEdge: checkUserAgent("Edge"), 19 | platformName: "webextension", 20 | }; 21 | 22 | export default def; 23 | 24 | export function getResourceUrl(path) { 25 | return chrome.runtime.getURL(`modules/${path}`.replace(/\/+/g, "/")); 26 | } 27 | -------------------------------------------------------------------------------- /platforms/webextension/prefs.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import events from "../core/events"; 6 | import console from "../core/console"; 7 | import { chrome } from "../platform/globals"; 8 | 9 | const PREFS_KEY = "extension-prefs"; 10 | let initialised = false; 11 | const prefs = {}; 12 | 13 | function syncToStorage() { 14 | chrome.storage.local.set({ [PREFS_KEY]: prefs }); 15 | } 16 | 17 | export function init() { 18 | return new Promise((resolve) => { 19 | chrome.storage.local.get([PREFS_KEY], (result) => { 20 | Object.assign(prefs, result[PREFS_KEY] || {}); 21 | initialised = true; 22 | resolve(); 23 | }); 24 | }); 25 | } 26 | 27 | export function getAllPrefs() { 28 | return Object.keys(prefs); 29 | } 30 | 31 | export function getPref(pref, notFound) { 32 | if (!initialised) { 33 | console.log( 34 | `loading pref ${pref} before prefs were initialised, you will not get the correct result` 35 | ); 36 | return prefs[pref] || notFound; 37 | } 38 | 39 | if (prefs && prefs[pref] !== undefined) { 40 | return prefs[pref]; 41 | } 42 | return notFound; 43 | } 44 | 45 | export function setPref(pref, value) { 46 | const changed = prefs[pref] !== value; 47 | 48 | prefs[pref] = value; 49 | 50 | // trigger prefchange event 51 | if (changed) { 52 | events.pub("prefchange", pref, "set"); 53 | } 54 | 55 | if (!initialised) { 56 | console.log( 57 | `setting pref ${pref} before prefs were initialised, you will not get the correct result` 58 | ); 59 | return Promise.resolve(); 60 | } 61 | 62 | if (changed) { 63 | syncToStorage(); 64 | } 65 | return Promise.resolve(); 66 | } 67 | 68 | export function hasPref(pref) { 69 | return pref in prefs; 70 | } 71 | 72 | export function clearPref(pref) { 73 | delete prefs[pref]; 74 | 75 | // trigger prefchange event 76 | events.pub("prefchange", pref, "clear"); 77 | 78 | syncToStorage(); 79 | return true; 80 | } 81 | -------------------------------------------------------------------------------- /platforms/webextension/resource-loader-storage.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import KeyValueStore from "./kv-store"; 6 | 7 | export default class Storage { 8 | constructor(filePath) { 9 | this.key = ["resource-loader", ...filePath].join(":"); 10 | } 11 | 12 | load() { 13 | return KeyValueStore.get(this.key); 14 | } 15 | 16 | save(data) { 17 | return KeyValueStore.set(this.key, data); 18 | } 19 | 20 | delete() { 21 | return KeyValueStore.remove(this.key); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /platforms/webextension/runtime.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | // NOTE: this is an attempt at using `webextension-polyfill` to abstract away 6 | // API differences between different browser (e.g.: promisify chrome functions). 7 | // Eventually, it could be used globally from 'platforms/webextension/globals.es'. 8 | import browserPoly from "webextension-polyfill"; 9 | 10 | export default (typeof browser !== "undefined" ? browser : browserPoly).runtime; 11 | -------------------------------------------------------------------------------- /platforms/webextension/test-helpers/helpers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { browser } from "../globals"; 6 | 7 | export const wrap = (getObj) => 8 | new Proxy( 9 | {}, 10 | { 11 | get(target, name) { 12 | const obj = getObj(); 13 | let prop = obj[name]; 14 | 15 | if (typeof prop === "function") { 16 | prop = prop.bind(obj); 17 | } 18 | return prop; 19 | }, 20 | set(target, name, value) { 21 | const obj = getObj(); 22 | obj[name] = value; 23 | return true; 24 | }, 25 | } 26 | ); 27 | 28 | const bgWindow = wrap(() => browser.extension.getBackgroundPage().window); 29 | export const win = wrap(() => bgWindow); 30 | export const WDP = wrap(() => bgWindow.WDP); 31 | export const app = wrap(() => bgWindow.WDP.app); 32 | 33 | export function getUrl(path) { 34 | return browser.runtime.getURL(path); 35 | } 36 | 37 | export const testServer = wrap(() => win.WDP.TestHelpers.testServer); 38 | -------------------------------------------------------------------------------- /platforms/webextension/text-decoder.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default typeof TextDecoder !== "undefined" ? TextDecoder : undefined; 6 | -------------------------------------------------------------------------------- /platforms/webextension/text-encoder.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export default typeof TextEncoder !== "undefined" ? TextEncoder : undefined; 6 | -------------------------------------------------------------------------------- /platforms/webextension/timers.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | const _setTimeout = (...args) => 6 | (typeof setTimeout === "undefined" 7 | ? window.setTimeout.bind(window) 8 | : setTimeout)(...args); 9 | const _setInterval = (...args) => 10 | (typeof setInterval === "undefined" 11 | ? window.setInterval.bind(window) 12 | : setInterval)(...args); 13 | const _clearTimeout = (...args) => 14 | (typeof clearTimeout === "undefined" 15 | ? window.clearTimeout.bind(window) 16 | : clearTimeout)(...args); 17 | const _clearInterval = (...args) => 18 | (typeof clearInterval === "undefined" 19 | ? window.clearInterval.bind(window) 20 | : clearInterval)(...args); 21 | 22 | export { 23 | _setTimeout as setTimeout, 24 | _setInterval as setInterval, 25 | _clearTimeout as clearTimeout, 26 | _clearInterval as clearInterval, 27 | }; 28 | -------------------------------------------------------------------------------- /platforms/webextension/web-discovery-project/opentabs.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | export function getAllOpenPages() { 6 | return new Promise((resolve, reject) => { 7 | try { 8 | const res = []; 9 | chrome.windows.getAll({ populate: true }, (windows) => { 10 | windows.forEach((window) => { 11 | window.tabs.forEach((tab) => { 12 | res.push(tab.url); 13 | }); 14 | }); 15 | resolve(res); 16 | }); 17 | } catch (ee) { 18 | reject(ee); 19 | } 20 | }); 21 | } 22 | 23 | export default getAllOpenPages; 24 | -------------------------------------------------------------------------------- /platforms/webextension/webnavigation.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { browser } from "./globals"; 6 | 7 | export default browser.webNavigation; 8 | -------------------------------------------------------------------------------- /platforms/webextension/webrequest.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { chrome } from "./globals"; 6 | 7 | export const VALID_RESPONSE_PROPERTIES = { 8 | onBeforeRequest: ["cancel", "redirectUrl"], 9 | onBeforeSendHeaders: ["cancel", "requestHeaders"], 10 | onSendHeaders: [], 11 | onHeadersReceived: ["cancel", "redirectUrl", "responseHeaders"], 12 | onAuthRequired: ["cancel"], 13 | onResponseStarted: [], 14 | onBeforeRedirect: [], 15 | onCompleted: [], 16 | onErrorOccurred: [], 17 | }; 18 | 19 | export default chrome.webRequest || {}; 20 | -------------------------------------------------------------------------------- /platforms/webextension/windows.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | import { chrome } from "./globals"; 6 | 7 | export default (chrome && chrome.windows) || undefined; 8 | -------------------------------------------------------------------------------- /platforms/webextension/worker.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | /* eslint no-restricted-globals: 'off' */ 6 | 7 | export default typeof Worker !== "undefined" ? Worker : undefined; 8 | -------------------------------------------------------------------------------- /platforms/webextension/xmlhttprequest.es: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | function setPrivateFlags() {} 6 | function setBackgroundRequest() {} 7 | function XMLHttpRequestFactory() { 8 | return XMLHttpRequest; 9 | } 10 | 11 | export { XMLHttpRequestFactory, setPrivateFlags, setBackgroundRequest }; 12 | -------------------------------------------------------------------------------- /run_tests_in_docker.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | help() { 4 | echo 'Usage:' 5 | echo " $0 " 6 | echo 7 | echo 'Available arguments for fern.js:' 8 | awk '/def matrix/{flag=1;next}/^]/{flag=0}flag' Jenkinsfile | 9 | grep testParams | 10 | cut -d':' -f 2- | 11 | sed '/^\s*$/d' | 12 | sed s'/,*$//' 13 | } 14 | 15 | if [ -z "$1" ]; then 16 | help 17 | exit 1 18 | fi 19 | 20 | FERN_ARGS="$1" 21 | 22 | # Create temporary docker file 23 | DOCKER_FILE="Dockerfile.tmp" 24 | cp Dockerfile.ci "${DOCKER_FILE}" 25 | cat <>${DOCKER_FILE} 26 | # Copy extension directory inside of Docker as the User "node" 27 | COPY --chown=node:node . /app/ 28 | EOF 29 | 30 | # Build docker 31 | echo "Building docker image" 32 | docker build -f "${DOCKER_FILE}" -t docker-wdp-tests . 33 | rm -frv "${DOCKER_FILE}" 34 | 35 | case "$FERN_ARGS" in 36 | *--extension-log*) 37 | EXTLOG_MOUNT=" -v $(pwd):/app/report " 38 | ;; 39 | esac 40 | 41 | # Run docker 42 | echo "Running tests, you can connect using a vnc client to 'localhost:15900 with password vnc'" 43 | echo "Fern config: ${FERN_ARGS}" 44 | DOCKER_RUN="docker run --privileged --rm -t --user node --shm-size=512m -p 15900:5900 -v $(pwd)/modules:/app/modules -v $(pwd)/platforms:/app/platforms $EXTLOG_MOUNT -w /app docker-wdp-tests ./tests/run_tests.sh $FERN_ARGS" 45 | 46 | if type xtightvncviewer >/dev/null 2>&1; then 47 | ${DOCKER_RUN} & 48 | sleep 5 49 | echo vnc | xtightvncviewer -autopass localhost::15900 50 | else 51 | ${DOCKER_RUN} 52 | fi 53 | -------------------------------------------------------------------------------- /specific/node/index.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this 3 | * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ 4 | 5 | module.exports = { 6 | App: require("./core/app").default, 7 | config: require("./core/config").default, 8 | }; 9 | -------------------------------------------------------------------------------- /specific/web-discovery-project/assets/brave.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brave/web-discovery-project/4c48c42bbe813c4ccf2624fc274ca7014e7fcd00/specific/web-discovery-project/assets/brave.png -------------------------------------------------------------------------------- /specific/web-discovery-project/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "WebDiscoveryProject", 3 | "description": "WebDiscoveryProject standalone extension", 4 | "version": "1.1.0", 5 | "manifest_version": 2, 6 | "icons": { "256": "assets/brave.png" }, 7 | "content_security_policy": "default-src 'self'; connect-src * data: blob: filesystem:; style-src 'self' data: 'unsafe-inline'; img-src 'self' data:; frame-src 'self' data:; font-src 'self' data:; media-src * data: blob: filesystem:; script-src 'self' 'wasm-eval';", 8 | "permissions": [ 9 | "", 10 | "history", 11 | "storage", 12 | "tabs", 13 | "unlimitedStorage", 14 | "webRequest", 15 | "webNavigation", 16 | "webRequestBlocking" 17 | ], 18 | "background": { "scripts": ["modules/webextension-specific/app.bundle.js"] }, 19 | "content_scripts": [ 20 | { 21 | "matches": ["*://*/*"], 22 | "run_at": "document_start", 23 | "match_about_blank": true, 24 | "js": ["modules/core/content-script.bundle.js"] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /testem.json: -------------------------------------------------------------------------------- 1 | { 2 | "launchers": { 3 | "unit-node": { 4 | "exe": "node", 5 | "args": ["--icu-data-dir=node_modules/full-icu", "tests/runners/unit-node.js"], 6 | "protocol": "tap" 7 | }, 8 | "firefox-web-ext": { 9 | "exe": "node", 10 | "args": ["tests/runners/firefox-web-ext.js"], 11 | "protocol": "tap" 12 | }, 13 | "brave-web-ext": { 14 | "exe": "node", 15 | "args": ["tests/runners/brave-web-ext.js"], 16 | "protocol": "tap" 17 | } 18 | }, 19 | "browser_args": { 20 | "chromium": [ 21 | "--no-sandbox" 22 | ] 23 | }, 24 | "test_page": "tests/index.mustache" 25 | } 26 | -------------------------------------------------------------------------------- /tests/run_tests.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | 5 | export DISPLAY=:0 6 | Xvfb $DISPLAY -screen 0 1024x768x24 -ac & 7 | sleep 1 8 | openbox & 9 | 10 | x11vnc -storepasswd vnc /tmp/vncpass 11 | x11vnc -rfbport 5900 -rfbauth /tmp/vncpass -forever >/dev/null 2>&1 & 12 | 13 | # while running this script, you can add multiple paratemeters in quotes, e.g.: 14 | # './configs/ci/browser.js --firefox /home/node/firefox56/firefox/firefox' 15 | # ./fern.js serve "$@" --include-tests 16 | 17 | # to use test instead of serve comment previous line and uncomment the next one 18 | node --unhandled-rejections=strict fern.js test "$@" --ci report.xml 19 | -------------------------------------------------------------------------------- /tests/runners/brave-web-ext.js: -------------------------------------------------------------------------------- 1 | const runner = require("./test-runner-common"); 2 | const BraveBrowser = require("./launchers/brave-web-ext").Browser; 3 | 4 | runner.run(new BraveBrowser()); 5 | -------------------------------------------------------------------------------- /tests/runners/firefox-web-ext.js: -------------------------------------------------------------------------------- 1 | const runner = require("./test-runner-common"); 2 | const FirefoxBrowser = require("./launchers/firefox-web-ext").Browser; 3 | 4 | runner.run(new FirefoxBrowser()); 5 | -------------------------------------------------------------------------------- /tests/runners/launchers/brave-web-ext.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const getOptionsUrl = require("./test-options"); 3 | 4 | exports.Browser = class BraveBrowser { 5 | constructor() { 6 | this.brave = null; 7 | } 8 | 9 | async run({ 10 | configFilePath = process.env.CONFIG_PATH, 11 | config, 12 | outputPath = process.env.OUTPUT_PATH || "./build", 13 | bravePath = process.env.BRAVE_PATH, 14 | sourceDir, 15 | braveProfile, 16 | keepProfileChanges = false, 17 | } = {}) { 18 | if (config === undefined) { 19 | config = require(path.resolve(configFilePath)); 20 | } 21 | 22 | if (sourceDir === undefined) { 23 | sourceDir = 24 | config.platform === "firefox" 25 | ? path.resolve(outputPath, config.settings.id) 26 | : path.resolve(outputPath); 27 | } 28 | 29 | const options = { 30 | chromiumBinary: bravePath, 31 | chromiumProfile: braveProfile, 32 | noReload: true, 33 | sourceDir, 34 | artifactsDir: sourceDir, 35 | startUrl: getOptionsUrl(), 36 | keepProfileChanges, 37 | target: "chromium", 38 | }; 39 | 40 | const webExt = await import("web-ext"); 41 | const runner = await webExt.default.cmd.run(options, { 42 | getValidatedManifest() { 43 | return { 44 | name: "", 45 | version: "", 46 | }; 47 | }, 48 | }); 49 | 50 | this.brave = runner.extensionRunners[0]; 51 | this.reloadAllExtensions = runner.reloadAllExtensions.bind(runner); 52 | } 53 | 54 | async unload() { 55 | if (this.brave) { 56 | await this.brave.exit(); 57 | this.brave = null; 58 | } 59 | } 60 | }; 61 | -------------------------------------------------------------------------------- /tests/runners/launchers/test-options.js: -------------------------------------------------------------------------------- 1 | module.exports = function getOptionsUrl() { 2 | // Special data url to pass options 3 | return `data:text/plain,${JSON.stringify({ 4 | grep: process.env.MOCHA_GREP || "", 5 | autostart: "true", 6 | invert: process.env.MOCHA_INVERT || "false", 7 | retries: process.env.MOCHA_RETRIES || 1, 8 | })}`; 9 | }; 10 | -------------------------------------------------------------------------------- /tests/runners/tap-result-streamer.js: -------------------------------------------------------------------------------- 1 | const WebSocket = require("ws"); 2 | const fs = require("fs"); 3 | 4 | exports.TapStreamer = class TapStreamer { 5 | constructor(host = "127.0.0.1", port = 3001) { 6 | this.logFile = process.env.EXTENSION_LOG; 7 | // Listener to close event 8 | this.onclose = () => {}; 9 | 10 | // Start listening for client 11 | this.wss = new WebSocket.Server({ host, port }); 12 | this.sockets = new Set(); 13 | 14 | this.wss.on("error", (err) => { 15 | // eslint-disable-next-line no-console 16 | console.error("TAP result streamer error", err); 17 | }); 18 | 19 | this.wss.on("connection", async (ws) => { 20 | // Keep track of opened sockets so that they can be terminated properly on `unload` 21 | this.sockets.add(ws); 22 | 23 | ws.on("message", (message) => { 24 | const parsed = JSON.parse(message); 25 | if (parsed.tap) { 26 | // Forward logs from browser to stdin 27 | // eslint-disable-next-line no-console 28 | console.log(parsed.tap); 29 | } else if (this.logFile && parsed.log) { 30 | const s = `${new Date().toISOString()} ${parsed.log}`; 31 | fs.appendFileSync(this.logFile, s); 32 | } else if (parsed.action === "END") { 33 | this.onclose(); 34 | } 35 | }); 36 | 37 | ws.on("close", () => { 38 | this.onclose(); 39 | this.sockets.delete(ws); 40 | }); 41 | }); 42 | } 43 | 44 | unload() { 45 | this.wss.close(); 46 | this.sockets.forEach((ws) => { 47 | ws.close(); 48 | }); 49 | } 50 | }; 51 | -------------------------------------------------------------------------------- /tests/runners/test-runner-common.js: -------------------------------------------------------------------------------- 1 | const spawn = require("child_process").spawn; 2 | const TapStreamer = require("./tap-result-streamer").TapStreamer; 3 | 4 | exports.run = async function run(browser) { 5 | // Test-server is used as a mock server from the tests. It can be useful 6 | // whenever you expect the extension to make request http requests to a 7 | // server, while being able to mock the response beforehand, as well as 8 | // getting information about requests made to the test server afterward. 9 | const server = spawn( 10 | "node", 11 | ["./tests/test-server.js"] /* , { stdio: 'inherit' } */ 12 | ); 13 | 14 | // This WebSocket server is used to stream the TAP test results from the 15 | // extension to the runner. It will forward all messages received to STDOUT so 16 | // that the results can be handled by testem. 17 | const tapStreamer = new TapStreamer(); 18 | 19 | // Close test server and browser on exit. 20 | const tearDown = () => { 21 | // Try to kill TAP result streamer 22 | try { 23 | tapStreamer.unload(); 24 | } catch (ex) { 25 | /* Ignore */ 26 | } 27 | 28 | // Try to kill test http server 29 | try { 30 | server.kill(); 31 | } catch (ex) { 32 | /* Ignore */ 33 | } 34 | 35 | // Try to kill browser 36 | try { 37 | return browser.unload(); 38 | } catch (ex) { 39 | /* Ignore */ 40 | } 41 | 42 | return Promise.resolve(); 43 | }; 44 | 45 | // If test runner is killed via CTRL-C, then still close test server 46 | process.on("SIGTERM", tearDown); 47 | 48 | /** 49 | * TAP Streamer will emit the `close` event whenever the browser signals that 50 | * tests finished running or if the remote socket is closed for whatever 51 | * reason. 52 | */ 53 | tapStreamer.onclose = async () => { 54 | // Do not tear down the test runner if specified 55 | if (process.env.KEEP_OPEN !== undefined) { 56 | return; 57 | } 58 | 59 | try { 60 | await tearDown(); 61 | } catch (ex) { 62 | /* Ignore */ 63 | } 64 | 65 | process.exit(0); 66 | }; 67 | 68 | // Start browser 69 | await browser.run(); 70 | }; 71 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": true, 4 | "target": "es6", 5 | "noEmit": true, 6 | "composite": true, 7 | "jsx": "react", 8 | "strict": true, 9 | "esModuleInterop": true, 10 | "moduleResolution": "node", 11 | "allowSyntheticDefaultImports": true, 12 | "allowUnreachableCode": false, 13 | "allowUnusedLabels": false, 14 | "forceConsistentCasingInFileNames": true, 15 | "keyofStringsOnly": true, 16 | "noFallthroughCasesInSwitch": true, 17 | "noImplicitAny": true, 18 | "noImplicitReturns": true, 19 | "noImplicitThis": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "strictFunctionTypes": true, 23 | "strictNullChecks": true, 24 | "strictPropertyInitialization": true 25 | }, 26 | "include": [ 27 | "./modules/**/*.ts" 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /update-brave.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | rm -fr brave 4 | mkdir brave 5 | cd brave && wget 'https://github.com/brave/brave-browser/releases/download/v1.60.114/brave-browser-1.60.114-linux-amd64.zip' -O brave.zip && unzip brave.zip && rm brave.zip 6 | sudo chown root:root ./brave/chrome-sandbox 7 | sudo chmod 4755 ./brave/chrome-sandbox 8 | export BRAVE_PATH="./brave/brave" 9 | --------------------------------------------------------------------------------