├── .dockerignore ├── .editorconfig ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug.yml │ └── config.yml ├── PULL_REQUEST_TEMPLATE ├── dependabot.yml ├── settings.yml └── workflows │ ├── auto-merge.yml │ ├── build-and-test.yml │ ├── codeql.yml │ ├── idle.yml │ └── release-please.yml ├── .gitignore ├── .nvmrc ├── .prettierignore ├── .prettierrc.json ├── .release-please-manifest.json ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── REVIEWING.md ├── SECURITY.md ├── bin └── wrapper.js ├── conf ├── config-example.json ├── config-test.json └── hsts-preload.json ├── jsconfig.json ├── mdn-observatory-webext ├── .eslintrc.json ├── README.md ├── _locales │ └── en │ │ └── messages.json ├── assets │ ├── A+.svg │ ├── A-.svg │ ├── A.svg │ ├── B+.svg │ ├── B-.svg │ ├── B.svg │ ├── C+.svg │ ├── C-.svg │ ├── C.svg │ ├── D+.svg │ ├── D-.svg │ ├── D.svg │ ├── E+.svg │ ├── E-.svg │ ├── E.svg │ ├── F+.svg │ ├── F.svg │ ├── error.svg │ ├── fonts │ │ ├── Inter-Bold.ttf │ │ └── Inter-Regular.ttf │ ├── icon.png │ ├── icon.svg │ ├── img │ │ ├── 128x128.png │ │ ├── 16x16.png │ │ ├── 24x24.png │ │ ├── 32x32.png │ │ ├── A+.512.png │ │ ├── A+.png │ │ ├── A-.512.png │ │ ├── A-.png │ │ ├── A.512.png │ │ ├── A.png │ │ ├── B+.512.png │ │ ├── B+.png │ │ ├── B-.512.png │ │ ├── B-.png │ │ ├── B.512.png │ │ ├── B.png │ │ ├── C+.512.png │ │ ├── C+.png │ │ ├── C-.512.png │ │ ├── C-.png │ │ ├── C.512.png │ │ ├── C.png │ │ ├── D+.512.png │ │ ├── D+.png │ │ ├── D-.512.png │ │ ├── D-.png │ │ ├── D.512.png │ │ ├── D.png │ │ ├── E+.512.png │ │ ├── E+.png │ │ ├── E-.512.png │ │ ├── E-.png │ │ ├── E.512.png │ │ ├── E.png │ │ ├── F+.512.png │ │ ├── F+.png │ │ ├── F.512.png │ │ ├── F.png │ │ ├── error.512.png │ │ ├── error.png │ │ ├── icon.512.png │ │ └── icon.png │ └── lib │ │ └── browser-polyfill.min.js ├── bundle.js ├── package-lock.json ├── package.json ├── src │ ├── background.js │ ├── manifest.json │ ├── popup.html │ └── popup.js └── test │ └── sample.js ├── migrations ├── 001.do.sites.sql ├── 001.undo.sites.sql ├── 002.do.expectations.sql ├── 002.undo.expectations.sql ├── 003.do.scans.sql ├── 003.undo.scans.sql ├── 004.do.tests.sql ├── 004.undo.tests.sql ├── 005.do.httpobs-user.sql ├── 005.undo.httpobs-user.sql ├── 006.do.mat-views.sql ├── 006.undo.mat-views.sql ├── 007.do.history-mat-view.sql ├── 007.undo.history-mat-view.sql ├── 008.do.unique_index_on_sites_domain.sql ├── 008.undo.unique_index_on_sites_domain.sql ├── 009.do.remove_sites_fields.sql ├── 009.undo.remove_sites_fields.sql ├── 010.do.remove_scans_fields.sql └── 010.undo.remove_scans_fields.sql ├── package-lock.json ├── package.json ├── release-please-config.json ├── src ├── analyzer │ ├── cspParser.js │ ├── hsts.js │ ├── tests │ │ ├── cookies.js │ │ ├── cors.js │ │ ├── cross-origin-resource-policy.js │ │ ├── csp.js │ │ ├── redirection.js │ │ ├── referrer-policy.js │ │ ├── strict-transport-security.js │ │ ├── subresource-integrity.js │ │ ├── x-content-type-options.js │ │ └── x-frame-options.js │ └── utils.js ├── api │ ├── errors.js │ ├── global-error-handler.js │ ├── index.js │ ├── server.js │ ├── utils.js │ ├── v2 │ │ ├── analyze │ │ │ └── index.js │ │ ├── recommendations │ │ │ └── index.js │ │ ├── scan │ │ │ └── index.js │ │ ├── schemas.js │ │ ├── stats │ │ │ └── index.js │ │ └── utils.js │ └── version │ │ └── index.js ├── config.js ├── constants.js ├── database │ ├── migrate.js │ └── repository.js ├── grader │ ├── charts.js │ └── grader.js ├── headers.js ├── index.js ├── maintenance │ └── index.js ├── retrieve-hsts.js ├── retriever │ ├── retriever.js │ ├── session.js │ ├── url.js │ └── utils.js ├── scan.js ├── scanner │ └── index.js └── types.js └── test ├── analyzer-utils.test.js ├── apiv2.test.js ├── cookies.test.js ├── cors.test.js ├── cross-origin-resource-policy.test.js ├── csp-parser.test.js ├── csp.test.js ├── database.test.js ├── files ├── domains.txt ├── test_content_sri_impl_external_http.html ├── test_content_sri_impl_external_https1.html ├── test_content_sri_impl_external_https2.html ├── test_content_sri_impl_external_noproto.html ├── test_content_sri_impl_sameorigin.html ├── test_content_sri_no_scripts.html ├── test_content_sri_notimpl_external_http.html ├── test_content_sri_notimpl_external_https.html ├── test_content_sri_notimpl_external_noproto.html ├── test_content_sri_sameorigin1.html ├── test_content_sri_sameorigin2.html ├── test_content_sri_sameorigin3.html ├── test_parse_http_equiv_headers_case_insensitivity.html ├── test_parse_http_equiv_headers_csp1.html ├── test_parse_http_equiv_headers_csp2.html ├── test_parse_http_equiv_headers_csp_multiple_http_equiv1.html ├── test_parse_http_equiv_headers_not_allowed.html ├── test_parse_http_equiv_headers_referrer1.html └── test_parse_http_equiv_headers_x_frame_options.html ├── grader.test.js ├── helpers.js ├── helpers └── db.js ├── redirection.test.js ├── referrer-policy.test.js ├── retriever.test.js ├── scanner.test.js ├── strict-transport-security.test.js ├── subresource-integrity.test.js ├── x-content-type-options.test.js └── x-frame-options.test.js /.dockerignore: -------------------------------------------------------------------------------- 1 | .github 2 | .git 3 | .vscode 4 | .idea 5 | mdn-observatory-webext 6 | test 7 | node_modules 8 | npm-debug.log 9 | Dockerfile 10 | .dockerignore 11 | .env 12 | conf/config.json 13 | api-examples.http 14 | .DS_Store 15 | .editorconfig 16 | .gitignore 17 | .prettierrc.json 18 | 19 | 20 | # Ignore generated credentials from google-github-actions/auth 21 | gha-creds-*.json 22 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | [*] 7 | indent_style = space 8 | indent_size = 2 9 | end_of_line = lf 10 | charset = utf-8 11 | trim_trailing_whitespace = true 12 | insert_final_newline = true 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false 16 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Order is important. The last matching pattern takes precedence. See: 2 | # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners 3 | 4 | # DEFAULT OWNERS 5 | * @mdn/core-dev 6 | 7 | # These are @mdn-bot because the auto-merge GHA workflow uses the PAT of this account. 8 | # If another reviewer is specified, update the PAT token or auto-merge will cease to be automatic. 9 | /package.json @mdn-bot 10 | /package-lock.json @mdn-bot 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.yml: -------------------------------------------------------------------------------- 1 | name: "Issue report" 2 | description: Report an unexpected problem or unintended behavior. 3 | labels: ["needs triage"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | ### Before you start 9 | 10 | **Want to fix the problem yourself?** This project is open source and we welcome fixes and improvements from the community! 11 | 12 | ↩ Check the project [CONTRIBUTING.md](../blob/main/CONTRIBUTING.md) guide to see how to get started. 13 | 14 | --- 15 | - type: textarea 16 | id: problem 17 | attributes: 18 | label: What information was incorrect, unhelpful, or incomplete? 19 | validations: 20 | required: true 21 | - type: textarea 22 | id: expected 23 | attributes: 24 | label: What did you expect to see? 25 | validations: 26 | required: true 27 | - type: textarea 28 | id: references 29 | attributes: 30 | label: Do you have any supporting links, references, or citations? 31 | description: Link to information that helps us confirm your issue. 32 | - type: textarea 33 | id: more-info 34 | attributes: 35 | label: Do you have anything more you want to share? 36 | description: For example, steps to reproduce, screenshots, screen recordings, or sample code. 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Content or feature request 4 | url: https://github.com/mdn/mdn/issues/new/choose 5 | about: Propose new content for MDN Web Docs or submit a feature request using this link. 6 | - name: MDN GitHub Discussions 7 | url: https://github.com/orgs/mdn/discussions 8 | about: Does the issue involve a lot of changes, or is it hard to split it into actionable tasks? Start a discussion before opening an issue. 9 | - name: MDN Web Docs on Discourse 10 | url: https://discourse.mozilla.org/c/mdn/learn/250 11 | about: Need help with assessments on MDN Web Docs? We have a support community for this purpose on Discourse. 12 | - name: Help with code 13 | url: https://stackoverflow.com/ 14 | about: If you are stuck and need help with code, StackOverflow is a great resource. 15 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE: -------------------------------------------------------------------------------- 1 | 2 | 3 | ### Description 4 | 5 | 6 | 7 | ### Motivation 8 | 9 | 10 | 11 | ### Additional details 12 | 13 | 14 | 15 | ### Related issues and pull requests 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | 9 | - package-ecosystem: npm 10 | directory: "/" 11 | schedule: 12 | interval: weekly 13 | 14 | - package-ecosystem: npm 15 | directory: "/mdn-observatory-webext" 16 | schedule: 17 | interval: weekly 18 | -------------------------------------------------------------------------------- /.github/settings.yml: -------------------------------------------------------------------------------- 1 | repository: 2 | # See https://github.com/apps/settings for all available settings. 3 | 4 | # The name of the repository. Changing this will rename the repository 5 | name: project-template 6 | 7 | # A short description of the repository that will show up on GitHub 8 | description: MDN Web Docs project template 9 | 10 | # A URL with more information about the repository 11 | homepage: https://github.com/mdn/project-template 12 | 13 | # The branch used by default for pull requests and when the repository is cloned/viewed. 14 | default_branch: main 15 | 16 | # This repository is a template that others can use to start a new repository. 17 | is_template: true 18 | 19 | branches: 20 | - name: main 21 | protection: 22 | # Required. Require at least one approving review on a pull request, before merging. Set to null to disable. 23 | required_pull_request_reviews: 24 | # The number of approvals required. (1-6) 25 | required_approving_review_count: 1 26 | # Dismiss approved reviews automatically when a new commit is pushed. 27 | dismiss_stale_reviews: true 28 | # Blocks merge until code owners have reviewed. 29 | require_code_owner_reviews: true 30 | 31 | collaborators: 32 | - username: Rumyra 33 | permission: admin 34 | 35 | - username: fiji-flo 36 | permission: admin 37 | 38 | labels: 39 | - name: bug 40 | color: D41130 41 | description: "Something that's wrong or not working as expected" 42 | - name: chore 43 | color: 258CD3 44 | description: "A routine task" 45 | - name: "good first issue" 46 | color: 48B71D 47 | description: "Great for newcomers to start contributing" 48 | - name: "help wanted" 49 | color: 2E7A10 50 | description: "Contributions welcome" 51 | 52 | teams: 53 | - name: core 54 | permission: admin 55 | -------------------------------------------------------------------------------- /.github/workflows/auto-merge.yml: -------------------------------------------------------------------------------- 1 | name: auto-merge 2 | 3 | on: 4 | pull_request_target: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | auto-merge: 10 | runs-on: ubuntu-latest 11 | if: github.event.pull_request.user.login == 'dependabot[bot]' 12 | 13 | steps: 14 | - name: Dependabot metadata 15 | id: dependabot-metadata 16 | uses: dependabot/fetch-metadata@v2 17 | with: 18 | github-token: ${{ secrets.AUTOMERGE_TOKEN }} 19 | 20 | - name: Squash and merge 21 | if: ${{ steps.dependabot-metadata.outputs.update-type == 'version-update:semver-minor' || steps.dependabot-metadata.outputs.update-type == 'version-update:semver-patch' }} 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} 24 | run: | 25 | gh pr review ${{ github.event.pull_request.html_url }} --approve 26 | gh pr comment ${{ github.event.pull_request.html_url }} --body "@dependabot squash and merge" 27 | -------------------------------------------------------------------------------- /.github/workflows/build-and-test.yml: -------------------------------------------------------------------------------- 1 | # run unit tests 2 | name: build-and-test 3 | 4 | on: 5 | workflow_dispatch: 6 | push: 7 | branches: [main, wip-andi] 8 | pull_request: 9 | branches: [main] 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | services: 15 | postgres: 16 | image: postgres:16 17 | env: 18 | POSTGRES_USER: observatory 19 | POSTGRES_PASSWORD: observatory 20 | POSTGRES_DB: observatory 21 | ports: 22 | - 5432:5432 23 | options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 24 | 25 | steps: 26 | - uses: actions/checkout@v4 27 | 28 | - name: Install Node 29 | uses: actions/setup-node@v4 30 | with: 31 | node-version-file: ".nvmrc" 32 | 33 | - name: Install dependencies 34 | run: "npm install" 35 | 36 | - name: Run type checks 37 | run: "npm run tsc" 38 | 39 | - name: Run tests 40 | run: npm test 41 | env: 42 | PGDATABASE: observatory 43 | PGHOST: localhost 44 | PGUSER: observatory 45 | PGPASSWORD: observatory 46 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [] 6 | paths-ignore: 7 | - "**.md" 8 | # pull_request: 9 | # # The branches below must be a subset of the branches above 10 | # branches: [] 11 | # paths-ignore: 12 | # - "**.md" 13 | 14 | jobs: 15 | analyze: 16 | name: Analyze 17 | runs-on: ubuntu-latest 18 | permissions: 19 | actions: read 20 | contents: read 21 | security-events: write 22 | 23 | strategy: 24 | matrix: 25 | # Add the language(s) you want to analyze here as an array of strings 26 | # for example: ['javascript'] or ['python', 'javascript'] 27 | language: ["javascript"] 28 | 29 | steps: 30 | - name: Checkout repository 31 | uses: actions/checkout@v4 32 | 33 | # Initializes the CodeQL tools for scanning. 34 | - name: Initialize CodeQL 35 | uses: github/codeql-action/init@v3 36 | with: 37 | languages: ${{ matrix.language }} 38 | 39 | - name: Perform CodeQL Analysis 40 | uses: github/codeql-action/analyze@v3 41 | with: 42 | category: "/language:${{matrix.language}}" 43 | -------------------------------------------------------------------------------- /.github/workflows/idle.yml: -------------------------------------------------------------------------------- 1 | # This workflow is hosted at: https://github.com/mdn/workflows/blob/main/.github/workflows/idle.yml 2 | # Docs for this workflow: https://github.com/mdn/workflows/blob/main/README.md#idle 3 | name: "Label idle issues" 4 | 5 | on: 6 | schedule: 7 | - cron: "0 8 * * *" 8 | 9 | jobs: 10 | mark-as-idle: 11 | uses: mdn/workflows/.github/workflows/idle.yml@main 12 | with: 13 | target-repo: "mdn/mdn-http-observatory" 14 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yml: -------------------------------------------------------------------------------- 1 | name: release-please 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | pull-requests: write 11 | 12 | jobs: 13 | release-please: 14 | if: github.repository == 'mdn/mdn-http-observatory' 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Release 18 | uses: GoogleCloudPlatform/release-please-action@v4 19 | id: release 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .env 3 | conf/config.json 4 | api-examples.http 5 | mdn-observatory-webext/dist 6 | .DS_Store 7 | compare_output.txt 8 | load-script.js 9 | .vscode/ -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v20 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | CHANGELOG.md 2 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5" 3 | } 4 | -------------------------------------------------------------------------------- /.release-please-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "1.4.0" 3 | } 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of conduct 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, read [Mozilla's Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 5 | 6 | ## Reporting violations 7 | 8 | For more information on how to report violations of the Community Participation Guidelines, read the [How to report](https://www.mozilla.org/about/governance/policies/participation/reporting/) page. 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20 2 | 3 | RUN apt-get -y update && \ 4 | apt-get install -y git && \ 5 | mkdir -p /home/node/app/node_modules && \ 6 | chown -R node:node /home/node/app 7 | 8 | WORKDIR /home/node/app 9 | USER node 10 | COPY --chown=node:node . . 11 | RUN npm install 12 | 13 | ARG GIT_SHA=dev 14 | ARG RUN_ID=unknown 15 | # Get the current HSTS list 16 | RUN npm run updateHsts 17 | 18 | RUN env 19 | 20 | ENV RUN_ID=${RUN_ID} 21 | ENV GIT_SHA=${GIT_SHA} 22 | ENV NODE_EXTRA_CA_CERTS=node_modules/extra_certs/ca_bundle/ca_intermediate_bundle.pem 23 | EXPOSE 8080 24 | CMD [ "node", "src/api/index.js" ] 25 | -------------------------------------------------------------------------------- /REVIEWING.md: -------------------------------------------------------------------------------- 1 | # Reviewing guide 2 | 3 | All reviewers must abide by the [code of conduct](CODE_OF_CONDUCT.md); they are also protected by the code of conduct. 4 | A reviewer should not tolerate poor behavior and is encouraged to [report any behavior](CODE_OF_CONDUCT.md#Reporting_violations) that violates the code of conduct. 5 | 6 | ## Review process 7 | 8 | The MDN Web Docs team has a well-defined review process that must be followed by reviewers in all repositories under the GitHub MDN organization. 9 | This process is described in detail on the [Pull request guidelines](https://developer.mozilla.org/en-US/docs/MDN/Community/Pull_requests) page. 10 | 11 | 18 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | If you've discovered a security issue, please report it through the form linked 6 | below, which will create a secure, private ticket. 7 | https://bugzilla.mozilla.org/form.web.bounty 8 | 9 | MDN may be eligible for 10 | [Mozilla's Security Bug Bounty Program](https://www.mozilla.org/en-US/security/bug-bounty/). 11 | You can find more information about the bounty program in the 12 | [Mozilla Web Bug Bounty FAQ](https://www.mozilla.org/en-US/security/bug-bounty/faq-webapp/). 13 | You can use the above form even if you are not interested in a bounty reward. 14 | -------------------------------------------------------------------------------- /bin/wrapper.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | "use strict"; 3 | 4 | import { spawnSync } from "node:child_process"; 5 | import { createRequire } from "node:module"; 6 | import path from "node:path"; 7 | import { fileURLToPath } from "node:url"; 8 | 9 | // Resolve __dirname from ESM environment 10 | const __filename = fileURLToPath(import.meta.url); 11 | const __dirname = path.dirname(__filename); 12 | 13 | // Set the environment variable for extra CA certificates 14 | let caCertPath = import.meta.resolve("node_extra_ca_certs_mozilla_bundle"); 15 | caCertPath = new URL(caCertPath).pathname; 16 | caCertPath = path.dirname(caCertPath); 17 | caCertPath = path.join(caCertPath, "ca_bundle", "ca_intermediate_bundle.pem"); 18 | process.env.NODE_EXTRA_CA_CERTS = caCertPath; 19 | 20 | // The target script you want to run (relative to this script's directory) 21 | const targetScript = path.join(__dirname, "..", "src", "scan.js"); 22 | 23 | // Forward any arguments passed to this script 24 | const args = process.argv.slice(2); 25 | 26 | // Spawn a new Node process to run the target script with inherited stdio 27 | const result = spawnSync(process.execPath, [targetScript, ...args], { 28 | stdio: "inherit", 29 | env: process.env, 30 | }); 31 | 32 | // Exit with the same code the spawned script returned 33 | process.exit(result.status ?? 1); 34 | -------------------------------------------------------------------------------- /conf/config-example.json: -------------------------------------------------------------------------------- 1 | { 2 | "database": { 3 | "database": "observatory", 4 | "user": "postgres" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /conf/config-test.json: -------------------------------------------------------------------------------- 1 | { 2 | "database": { 3 | "database": "observatory_test", 4 | "user": "postgres" 5 | }, 6 | "api": { 7 | "enableLogging": false, 8 | "cooldown": 3 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "checkJs": true, 4 | "target": "es2022", 5 | "maxNodeModuleJsDepth": 0, 6 | "module": "NodeNext", 7 | "moduleResolution": "nodenext", 8 | "strict": true 9 | }, 10 | "exclude": ["node_modules", "mdn-observatory-webext"] 11 | } -------------------------------------------------------------------------------- /mdn-observatory-webext/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true, 5 | "node": true 6 | }, 7 | "extends": [ 8 | "eslint:recommended" 9 | ], 10 | "globals": { 11 | "document": false, 12 | "escape": false, 13 | "navigator": false, 14 | "unescape": false, 15 | "window": false, 16 | "describe": true, 17 | "before": true, 18 | "it": true, 19 | "expect": true, 20 | "sinon": true, 21 | "chrome": true, 22 | "browser": true 23 | }, 24 | "plugins": [], 25 | "parserOptions": { 26 | "ecmaVersion": 2020, 27 | "sourceType": "module" 28 | }, 29 | "rules": { 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /mdn-observatory-webext/README.md: -------------------------------------------------------------------------------- 1 | # MDN Observatory 2 | 3 | Checking http-level security and grading a site 4 | 5 | ## Development 6 | 7 | This extension was created with [Extension CLI](https://oss.mobilefirst.me/extension-cli/)! 8 | 9 | If you find this software helpful [star](https://github.com/MobileFirstLLC/extension-cli/) or [sponsor](https://github.com/sponsors/MobileFirstLLC) this project. 10 | 11 | 12 | ### Available Commands 13 | 14 | | Commands | Description | 15 | | --- | --- | 16 | | `npm run start` | build extension, watch file changes | 17 | | `npm run build` | generate release version | 18 | | `npm run docs` | generate source code docs | 19 | | `npm run clean` | remove temporary files | 20 | | `npm run test` | run unit tests | 21 | | `npm run sync` | update config files | 22 | 23 | For CLI instructions see [User Guide →](https://oss.mobilefirst.me/extension-cli/) 24 | 25 | ### Learn More 26 | 27 | **Extension Developer guides** 28 | 29 | - [Getting started with extension development](https://developer.chrome.com/extensions/getstarted) 30 | - Manifest configuration: [version 2](https://developer.chrome.com/extensions/manifest) - [version 3](https://developer.chrome.com/docs/extensions/mv3/intro/) 31 | - [Permissions reference](https://developer.chrome.com/extensions/declare_permissions) 32 | - [Chrome API reference](https://developer.chrome.com/docs/extensions/reference/) 33 | 34 | **Extension Publishing Guides** 35 | 36 | - [Publishing for Chrome](https://developer.chrome.com/webstore/publish) 37 | - [Publishing for Edge](https://docs.microsoft.com/en-us/microsoft-edge/extensions-chromium/publish/publish-extension) 38 | - [Publishing for Opera addons](https://dev.opera.com/extensions/publishing-guidelines/) 39 | - [Publishing for Firefox](https://extensionworkshop.com/documentation/publish/submitting-an-add-on/) 40 | -------------------------------------------------------------------------------- /mdn-observatory-webext/_locales/en/messages.json: -------------------------------------------------------------------------------- 1 | { 2 | "appName": { 3 | "message": "MDN Observatory" 4 | }, 5 | "appShortName": { 6 | "message": "MDN Observatory" 7 | }, 8 | "appDescription": { 9 | "message": "Checking http-level security and grading a site" 10 | } 11 | } -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/A+.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | A 52 | + 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/A-.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 40 | 45 | 50 | 55 | 56 | 58 | 62 | A 73 | - 84 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/A.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | A 52 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/B+.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | B 52 | + 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/B-.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | B 52 | - 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/B.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | B 52 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/C+.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | C 52 | + 64 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/C-.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | C 52 | - 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/C.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | C 52 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/D+.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | D 52 | + 64 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/D-.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | D 52 | - 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/D.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | D 52 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/E+.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | E 52 | + 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/E-.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | E 52 | - 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/E.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | E 52 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/F+.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | F 52 | + 63 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/F.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | F 52 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/error.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 35 | 37 | 41 | ? 52 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/fonts/Inter-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/fonts/Inter-Bold.ttf -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/fonts/Inter-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/fonts/Inter-Regular.ttf -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/icon.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/icon.svg: -------------------------------------------------------------------------------- 1 | 2 | Satellite AntennaSatellite Antenna 50 | -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/128x128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/128x128.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/16x16.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/24x24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/24x24.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/32x32.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/A+.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/A+.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/A+.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/A+.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/A-.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/A-.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/A-.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/A-.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/A.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/A.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/A.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/A.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/B+.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/B+.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/B+.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/B+.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/B-.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/B-.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/B-.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/B-.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/B.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/B.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/B.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/B.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/C+.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/C+.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/C+.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/C+.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/C-.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/C-.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/C-.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/C-.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/C.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/C.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/C.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/C.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/D+.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/D+.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/D+.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/D+.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/D-.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/D-.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/D-.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/D-.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/D.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/D.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/D.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/D.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/E+.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/E+.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/E+.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/E+.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/E-.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/E-.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/E-.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/E-.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/E.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/E.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/E.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/E.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/F+.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/F+.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/F+.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/F+.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/F.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/F.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/F.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/F.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/error.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/error.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/error.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/icon.512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/icon.512.png -------------------------------------------------------------------------------- /mdn-observatory-webext/assets/img/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdn/mdn-http-observatory/1d5c12f34bc5705e142f2b937093f79da95dc814/mdn-observatory-webext/assets/img/icon.png -------------------------------------------------------------------------------- /mdn-observatory-webext/bundle.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import * as esbuild from "esbuild"; 3 | import { polyfillNode } from "esbuild-plugin-polyfill-node"; 4 | import path from "node:path"; 5 | import fs from "fs"; 6 | 7 | async function build() { 8 | const outputDir = "dist"; 9 | const publicDir = "assets"; 10 | const localeDir = "_locales"; 11 | const popupEntry = "popup.js"; 12 | const popupHtml = "popup.html"; 13 | const backgroundEntry = "background.js"; 14 | const manifest = "manifest.json"; 15 | 16 | const plugins = [polyfillNode({})]; 17 | const entryPoints = [ 18 | path.resolve("src", popupEntry), 19 | path.resolve("src", backgroundEntry), 20 | ]; 21 | 22 | await esbuild.build({ 23 | entryPoints: entryPoints, 24 | plugins: plugins, 25 | outdir: outputDir, 26 | bundle: true, 27 | minify: false, 28 | external: ["deasync"], 29 | format: "esm", 30 | sourcemap: true, 31 | }); 32 | await fs.promises.copyFile( 33 | path.resolve("src", popupHtml), 34 | path.resolve(outputDir, popupHtml) 35 | ); 36 | await fs.promises.copyFile( 37 | path.resolve("src", manifest), 38 | path.resolve(outputDir, manifest) 39 | ); 40 | await fs.promises.cp(publicDir, outputDir, { recursive: true }); 41 | await fs.promises.cp(localeDir, path.resolve(outputDir, localeDir), { 42 | recursive: true, 43 | }); 44 | } 45 | 46 | build().catch((err) => { 47 | console.log(err); 48 | process.exit(1); 49 | }); 50 | -------------------------------------------------------------------------------- /mdn-observatory-webext/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mdn-observatory", 3 | "description": "Checking http-level security and grading a site", 4 | "version": "0.0.1", 5 | "homepage": "http://chrome.google.com/webstore", 6 | "author": "ENTER YOUR NAME HERE", 7 | "repository": { 8 | "type": "git", 9 | "url": "ENTER GIT REPO URL" 10 | }, 11 | "type": "module", 12 | "scripts": { 13 | "start": "xt-build -e dev -w", 14 | "start:firefox": "xt-build -e dev -p firefox -w", 15 | "build": "node bundle.js" 16 | }, 17 | "babel": { 18 | "presets": [ 19 | "@babel/preset-env" 20 | ] 21 | }, 22 | "eslintIgnore": [ 23 | "test/**/*" 24 | ], 25 | "devDependencies": { 26 | "esbuild": "0.25.5", 27 | "esbuild-plugin-polyfill-node": "^0.3.0", 28 | "extension-cli": "latest" 29 | }, 30 | "xtdocs": { 31 | "source": { 32 | "include": [ 33 | "README.md", 34 | "src" 35 | ] 36 | } 37 | }, 38 | "xtbuild": { 39 | "js_bundles": [ 40 | { 41 | "name": "obsi", 42 | "src": "./src/**/*.js" 43 | } 44 | ] 45 | }, 46 | "dependencies": { 47 | "deasync": "^0.1.30", 48 | "obsi": "file:.." 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /mdn-observatory-webext/src/background.js: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | browser.webNavigation.onCompleted.addListener(async (details) => { 3 | if (details.frameId === 0) { 4 | const tab = await browser.tabs.get(details.tabId); 5 | const url = new URL(tab.url); 6 | const hostname = url.hostname; 7 | 8 | let apiResult; 9 | 10 | // check for a cached version 11 | const cachedResult = await browser.storage.local.get(hostname); 12 | if (cachedResult[hostname]) { 13 | // check timestamp 14 | const ts = new Date(cachedResult[hostname].end_time); 15 | const now = new Date(); 16 | // 24 hours 17 | const CACHE_PERIOD_MS = 1000 * 60 * 60 * 24; 18 | if (now.getTime() - ts.getTime() < CACHE_PERIOD_MS) { 19 | apiResult = cachedResult[hostname]; 20 | } 21 | } 22 | 23 | if (!apiResult) { 24 | try { 25 | apiResult = await fetchApiResponse(hostname); 26 | } catch (error) { 27 | console.error("Failed to fetch or process API response:", error); 28 | const iconPath = `img/error.png`; 29 | browser.action.setIcon({ path: iconPath, tabId: details.tabId }); 30 | return; 31 | } 32 | } 33 | 34 | const iconPath = `img/${apiResult.grade}.png`; 35 | browser.action.setIcon({ path: iconPath, tabId: details.tabId }); 36 | 37 | // store result in the cache 38 | if (apiResult) { 39 | try { 40 | const data = {}; 41 | data[hostname] = apiResult; 42 | await browser.storage.local.set(data); 43 | } catch (error) { 44 | console.error(error); 45 | } 46 | } 47 | } 48 | }); 49 | 50 | async function fetchApiResponse(host) { 51 | try { 52 | const apiUrl = `https://http-observatory.security.mozilla.org/api/v1/analyze?host=${encodeURIComponent( 53 | host 54 | )}`; 55 | const res = await fetch(apiUrl, { method: "POST" }); 56 | const result = await res.json(); 57 | return result; 58 | } catch (error) { 59 | console.error("Fetch error:", error); 60 | throw error; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /mdn-observatory-webext/src/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MDN Observatory", 3 | "short_name": "Obsi", 4 | "description": "MDN HTTP Observatory checks certain security relevant headers of the current web site.", 5 | "homepage_url": "https://github.com/mdn/mdn-http-observatory", 6 | "version": "1.0.0", 7 | "manifest_version": 3, 8 | "default_locale": "en", 9 | "permissions": ["activeTab", "webNavigation", "tabs", "storage"], 10 | "host_permissions": [""], 11 | "content_security_policy": { 12 | "extension_pages": "script-src 'self'; object-src 'self'; connect-src *;" 13 | }, 14 | "icons": { 15 | "16": "img/16x16.png", 16 | "24": "img/24x24.png", 17 | "32": "img/32x32.png", 18 | "128": "img/128x128.png" 19 | }, 20 | "action": { 21 | "default_icon": { 22 | "16": "img/16x16.png", 23 | "24": "img/24x24.png", 24 | "32": "img/32x32.png" 25 | }, 26 | "default_title": "MDN Observatory", 27 | "default_popup": "popup.html" 28 | }, 29 | "background": { 30 | "scripts": ["lib/browser-polyfill.min.js", "background.js"], 31 | "type": "module" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /mdn-observatory-webext/src/popup.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Obsi 8 | 9 | 10 | 11 | 44 |

MDN Observatory

45 | 46 |
47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /mdn-observatory-webext/src/popup.js: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | document.addEventListener("DOMContentLoaded", function () { 3 | const querying = browser.tabs.query({ 4 | active: true, 5 | currentWindow: true, 6 | }); 7 | querying.then(async (tabs) => { 8 | const url = new URL(tabs[0].url); 9 | const apiUrl = `https://http-observatory.security.mozilla.org/api/v1/analyze?host=${encodeURIComponent( 10 | url.host 11 | )}`; 12 | const res = await fetch(apiUrl, { method: "POST" }); 13 | const result = await res.json(); 14 | document.getElementById("grade").textContent = result.grade; 15 | }); 16 | document.getElementById("grade").textContent = "…"; 17 | }); 18 | -------------------------------------------------------------------------------- /mdn-observatory-webext/test/sample.js: -------------------------------------------------------------------------------- 1 | describe('Test extension', () => { 2 | 3 | it('This is a dummy test', () => { 4 | expect(true).to.be.true; 5 | }); 6 | }); 7 | -------------------------------------------------------------------------------- /migrations/001.do.sites.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS sites ( 2 | id SERIAL PRIMARY KEY, 3 | domain VARCHAR(255) NOT NULL, 4 | creation_time TIMESTAMP NOT NULL, 5 | public_headers JSONB NULL, 6 | private_headers JSONB NULL, 7 | cookies JSONB NULL 8 | ); 9 | 10 | CREATE INDEX sites_domain_idx ON sites (domain); 11 | -------------------------------------------------------------------------------- /migrations/001.undo.sites.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS sites; -------------------------------------------------------------------------------- /migrations/002.do.expectations.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS expectations ( 2 | id SERIAL PRIMARY KEY, 3 | site_id INTEGER REFERENCES sites (id), 4 | test_name VARCHAR NOT NULL, 5 | expectation VARCHAR NOT NULL 6 | ); -------------------------------------------------------------------------------- /migrations/002.undo.expectations.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS expectations; -------------------------------------------------------------------------------- /migrations/003.do.scans.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS scans ( 2 | id SERIAL PRIMARY KEY, 3 | site_id INTEGER REFERENCES sites (id) NOT NULL, 4 | state VARCHAR NOT NULL, 5 | start_time TIMESTAMP NOT NULL, 6 | end_time TIMESTAMP NULL, 7 | algorithm_version SMALLINT NOT NULL DEFAULT 1, 8 | tests_failed SMALLINT NOT NULL DEFAULT 0, 9 | tests_passed SMALLINT NOT NULL DEFAULT 0, 10 | tests_quantity SMALLINT NOT NULL, 11 | grade VARCHAR(2) NULL, 12 | score SMALLINT NULL, 13 | likelihood_indicator VARCHAR NULL, 14 | error VARCHAR NULL, 15 | response_headers JSONB NULL, 16 | hidden BOOL NOT NULL DEFAULT FALSE, 17 | status_code SMALLINT NULL 18 | ); 19 | 20 | CREATE INDEX scans_state_idx ON scans (state); 21 | CREATE INDEX scans_start_time_idx ON scans (start_time); 22 | CREATE INDEX scans_end_time_idx ON scans (end_time); 23 | CREATE INDEX scans_algorithm_version_idx ON scans (algorithm_version); 24 | CREATE INDEX scans_grade_idx ON scans (grade); 25 | CREATE INDEX scans_score_idx ON scans (score); 26 | CREATE INDEX scans_hidden_idx ON scans (hidden); 27 | CREATE INDEX scans_site_id_finished_state_end_time_idx ON scans (site_id, state, end_time DESC) WHERE state = 'FINISHED'; 28 | -------------------------------------------------------------------------------- /migrations/003.undo.scans.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS scans; -------------------------------------------------------------------------------- /migrations/004.do.tests.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS tests ( 2 | id BIGSERIAL PRIMARY KEY, 3 | site_id INTEGER REFERENCES sites (id) NOT NULL, 4 | scan_id INTEGER REFERENCES scans (id) NOT NULL, 5 | name VARCHAR NOT NULL, 6 | expectation VARCHAR NOT NULL, 7 | result VARCHAR NOT NULL, 8 | score_modifier SMALLINT NOT NULL, 9 | pass BOOL NOT NULL, 10 | output JSONB NOT NULL 11 | ); 12 | 13 | CREATE INDEX tests_scan_id_idx ON tests (scan_id); 14 | CREATE INDEX tests_name_idx ON tests (name); 15 | CREATE INDEX tests_result_idx ON tests (result); 16 | CREATE INDEX tests_pass_idx ON tests (pass); 17 | -------------------------------------------------------------------------------- /migrations/004.undo.tests.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS tests; -------------------------------------------------------------------------------- /migrations/005.do.httpobs-user.sql: -------------------------------------------------------------------------------- 1 | CREATE USER httpobsapi; 2 | GRANT USAGE ON SCHEMA public TO httpobsapi; 3 | GRANT SELECT ON expectations, scans, tests to httpobsapi; 4 | GRANT SELECT (id, domain, creation_time, public_headers) ON sites TO httpobsapi; 5 | GRANT INSERT ON sites, scans TO httpobsapi; 6 | GRANT UPDATE (public_headers, private_headers, cookies) ON sites TO httpobsapi; 7 | GRANT UPDATE ON scans TO httpobsapi; 8 | GRANT USAGE ON SEQUENCE sites_id_seq TO httpobsapi; 9 | GRANT USAGE ON SEQUENCE scans_id_seq TO httpobsapi; 10 | GRANT USAGE ON SEQUENCE expectations_id_seq TO httpobsapi; 11 | GRANT SELECT on sites, scans, expectations, tests TO httpobsapi; 12 | GRANT UPDATE (domain) ON sites to httpobsapi; /* TODO: there's got to be a better way with SELECT ... FOR UPDATE */ 13 | GRANT UPDATE on scans TO httpobsapi; 14 | GRANT INSERT on tests TO httpobsapi; 15 | GRANT USAGE ON SEQUENCE tests_id_seq TO httpobsapi; 16 | -------------------------------------------------------------------------------- /migrations/005.undo.httpobs-user.sql: -------------------------------------------------------------------------------- 1 | REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM httpobsapi; 2 | REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM httpobsapi; 3 | REVOKE ALL PRIVILEGES ON SCHEMA public FROM httpobsapi; 4 | DROP USER httpobsapi; -------------------------------------------------------------------------------- /migrations/006.do.mat-views.sql: -------------------------------------------------------------------------------- 1 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_scans 2 | AS SELECT latest_scans.site_id, latest_scans.scan_id, s.domain, latest_scans.state, 3 | latest_scans.start_time, latest_scans.end_time, latest_scans.tests_failed, latest_scans.tests_passed, 4 | latest_scans.grade, latest_scans.score, latest_scans.error 5 | FROM sites s, 6 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 7 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time DESC LIMIT 1 ) latest_scans; 8 | CREATE UNIQUE INDEX IF NOT EXISTS latest_scans_scan_id_idx ON latest_scans (scan_id); 9 | COMMENT ON MATERIALIZED VIEW latest_scans IS 'Most recently completed scan for a given website'; 10 | GRANT SELECT ON latest_scans TO httpobsapi; 11 | 12 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_tests 13 | AS SELECT latest_scans.domain, tests.site_id, tests.scan_id, name, result, pass, output 14 | FROM tests 15 | INNER JOIN latest_scans 16 | ON (latest_scans.scan_id = tests.scan_id); 17 | COMMENT ON MATERIALIZED VIEW latest_tests IS 'Test results from all the most recent scans'; 18 | 19 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution 20 | AS SELECT grade, count(*) 21 | FROM latest_scans 22 | GROUP BY grade; 23 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_grade_idx ON grade_distribution (grade); 24 | COMMENT ON MATERIALIZED VIEW grade_distribution IS 'The grades and how many latest scans have that score'; 25 | GRANT SELECT ON grade_distribution TO httpobsapi; 26 | 27 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution_all_scans 28 | AS SELECT grade, count(*) 29 | FROM scans 30 | WHERE state = 'FINISHED' 31 | GROUP BY grade; 32 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_all_scans_grade_idx ON grade_distribution_all_scans (grade); 33 | COMMENT ON MATERIALIZED VIEW grade_distribution_all_scans IS 'The grades and how many scans have that score'; 34 | GRANT SELECT ON grade_distribution_all_scans TO httpobsapi; 35 | 36 | 37 | CREATE MATERIALIZED VIEW IF NOT EXISTS earliest_scans 38 | AS SELECT earliest_scans.site_id, earliest_scans.scan_id, s.domain, earliest_scans.state, 39 | earliest_scans.start_time, earliest_scans.end_time, earliest_scans.tests_failed, earliest_scans.tests_passed, 40 | earliest_scans.grade, earliest_scans.score, earliest_scans.error 41 | FROM sites s, 42 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 43 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time ASC LIMIT 1 ) earliest_scans; 44 | CREATE UNIQUE INDEX IF NOT EXISTS earliest_scans_scan_id_idx ON earliest_scans (scan_id); 45 | COMMENT ON MATERIALIZED VIEW earliest_scans IS 'Oldest completed scan for a given website'; 46 | GRANT SELECT ON earliest_scans TO httpobsapi; 47 | 48 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution 49 | AS SELECT earliest_scans.site_id, earliest_scans.domain, earliest_scans.score AS before, latest_scans.score AS after, 50 | (latest_scans.score - earliest_scans.score) AS difference 51 | FROM earliest_scans, latest_scans 52 | WHERE earliest_scans.site_id = latest_scans.site_id; 53 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution IS 'How much score has changed since first scan'; 54 | GRANT SELECT ON scan_score_difference_distribution TO httpobsapi; 55 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_site_id_idx ON scan_score_difference_distribution (site_id); 56 | CREATE INDEX scan_score_difference_difference_distribution_idx ON scan_score_difference_distribution (difference); 57 | 58 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution_summation 59 | AS SELECT DISTINCT difference, COUNT(difference) AS num_sites 60 | FROM scan_score_difference_distribution 61 | GROUP BY difference 62 | ORDER BY difference DESC; 63 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_summation_difference_idx ON scan_score_difference_distribution_summation (difference); 64 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution_summation IS 'How many sites have improved by how many points'; 65 | GRANT SELECT ON scan_score_difference_distribution_summation TO httpobsapi; 66 | 67 | ALTER MATERIALIZED VIEW grade_distribution OWNER TO httpobsapi; /* so it can refresh */ 68 | ALTER MATERIALIZED VIEW grade_distribution_all_scans OWNER TO httpobsapi; /* so it can refresh */ 69 | ALTER MATERIALIZED VIEW latest_scans OWNER TO httpobsapi; 70 | ALTER MATERIALIZED VIEW earliest_scans OWNER TO httpobsapi; 71 | ALTER MATERIALIZED VIEW scan_score_difference_distribution OWNER TO httpobsapi; 72 | ALTER MATERIALIZED VIEW scan_score_difference_distribution_summation OWNER TO httpobsapi; 73 | ALTER MATERIALIZED VIEW latest_tests OWNER TO httpobsapi; -------------------------------------------------------------------------------- /migrations/006.undo.mat-views.sql: -------------------------------------------------------------------------------- 1 | DROP MATERIALIZED VIEW IF EXISTS latest_tests; 2 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution; 3 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution_all_scans; 4 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution_summation; 5 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution; 6 | DROP MATERIALIZED VIEW IF EXISTS earliest_scans; 7 | DROP MATERIALIZED VIEW IF EXISTS latest_scans; 8 | -------------------------------------------------------------------------------- /migrations/007.do.history-mat-view.sql: -------------------------------------------------------------------------------- 1 | DROP MATERIALIZED VIEW IF EXISTS latest_tests; 2 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution; 3 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution_all_scans; 4 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution_summation; 5 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution; 6 | DROP MATERIALIZED VIEW IF EXISTS earliest_scans; 7 | DROP MATERIALIZED VIEW IF EXISTS latest_scans; 8 | 9 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution AS 10 | SELECT grade, count(*) AS count FROM ( 11 | SELECT DISTINCT ON (scans.site_id) 12 | scans.grade 13 | FROM scans 14 | WHERE scans.end_time > (now() - INTERVAL '1 year') 15 | AND scans.state = 'FINISHED' 16 | ORDER BY scans.site_id, scans.end_time DESC 17 | ) s 18 | GROUP BY grade; 19 | CREATE UNIQUE INDEX IF NOT EXISTS grade_idx ON grade_distribution (grade); 20 | COMMENT ON MATERIALIZED VIEW grade_distribution IS 'Grade distribution over the last year'; 21 | -------------------------------------------------------------------------------- /migrations/007.undo.history-mat-view.sql: -------------------------------------------------------------------------------- 1 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution; 2 | 3 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_scans 4 | AS SELECT latest_scans.site_id, latest_scans.scan_id, s.domain, latest_scans.state, 5 | latest_scans.start_time, latest_scans.end_time, latest_scans.tests_failed, latest_scans.tests_passed, 6 | latest_scans.grade, latest_scans.score, latest_scans.error 7 | FROM sites s, 8 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 9 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time DESC LIMIT 1 ) latest_scans; 10 | CREATE UNIQUE INDEX IF NOT EXISTS latest_scans_scan_id_idx ON latest_scans (scan_id); 11 | COMMENT ON MATERIALIZED VIEW latest_scans IS 'Most recently completed scan for a given website'; 12 | 13 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_tests 14 | AS SELECT latest_scans.domain, tests.site_id, tests.scan_id, name, result, pass, output 15 | FROM tests 16 | INNER JOIN latest_scans 17 | ON (latest_scans.scan_id = tests.scan_id); 18 | COMMENT ON MATERIALIZED VIEW latest_tests IS 'Test results from all the most recent scans'; 19 | 20 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution 21 | AS SELECT grade, count(*) 22 | FROM latest_scans 23 | GROUP BY grade; 24 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_grade_idx ON grade_distribution (grade); 25 | COMMENT ON MATERIALIZED VIEW grade_distribution IS 'The grades and how many latest scans have that score'; 26 | 27 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution_all_scans 28 | AS SELECT grade, count(*) 29 | FROM scans 30 | WHERE state = 'FINISHED' 31 | GROUP BY grade; 32 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_all_scans_grade_idx ON grade_distribution_all_scans (grade); 33 | COMMENT ON MATERIALIZED VIEW grade_distribution_all_scans IS 'The grades and how many scans have that score'; 34 | 35 | 36 | CREATE MATERIALIZED VIEW IF NOT EXISTS earliest_scans 37 | AS SELECT earliest_scans.site_id, earliest_scans.scan_id, s.domain, earliest_scans.state, 38 | earliest_scans.start_time, earliest_scans.end_time, earliest_scans.tests_failed, earliest_scans.tests_passed, 39 | earliest_scans.grade, earliest_scans.score, earliest_scans.error 40 | FROM sites s, 41 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 42 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time ASC LIMIT 1 ) earliest_scans; 43 | CREATE UNIQUE INDEX IF NOT EXISTS earliest_scans_scan_id_idx ON earliest_scans (scan_id); 44 | COMMENT ON MATERIALIZED VIEW earliest_scans IS 'Oldest completed scan for a given website'; 45 | 46 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution 47 | AS SELECT earliest_scans.site_id, earliest_scans.domain, earliest_scans.score AS before, latest_scans.score AS after, 48 | (latest_scans.score - earliest_scans.score) AS difference 49 | FROM earliest_scans, latest_scans 50 | WHERE earliest_scans.site_id = latest_scans.site_id; 51 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution IS 'How much score has changed since first scan'; 52 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_site_id_idx ON scan_score_difference_distribution (site_id); 53 | CREATE INDEX scan_score_difference_difference_distribution_idx ON scan_score_difference_distribution (difference); 54 | 55 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution_summation 56 | AS SELECT DISTINCT difference, COUNT(difference) AS num_sites 57 | FROM scan_score_difference_distribution 58 | GROUP BY difference 59 | ORDER BY difference DESC; 60 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_summation_difference_idx ON scan_score_difference_distribution_summation (difference); 61 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution_summation IS 'How many sites have improved by how many points'; 62 | 63 | -------------------------------------------------------------------------------- /migrations/008.do.unique_index_on_sites_domain.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX IF NOT EXISTS sites_domain_unique_idx ON sites (domain); 2 | DROP INDEX IF EXISTS sites_domain_idx; -------------------------------------------------------------------------------- /migrations/008.undo.unique_index_on_sites_domain.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX IF EXISTS sites_domain_unique_idx; 2 | CREATE INDEX IF NOT EXISTS sites_domain_idx ON sites (domain); 3 | -------------------------------------------------------------------------------- /migrations/009.do.remove_sites_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE sites 2 | DROP COLUMN IF EXISTS public_headers, 3 | DROP COLUMN IF EXISTS private_headers, 4 | DROP COLUMN IF EXISTS cookies; 5 | -------------------------------------------------------------------------------- /migrations/009.undo.remove_sites_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE sites ADD COLUMN IF NOT EXISTS public_headers jsonb; 2 | ALTER TABLE sites ADD COLUMN IF NOT EXISTS private_headers jsonb; 3 | ALTER TABLE sites ADD COLUMN IF NOT EXISTS cookies jsonb; 4 | -------------------------------------------------------------------------------- /migrations/010.do.remove_scans_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE scans 2 | DROP COLUMN IF EXISTS "hidden", 3 | DROP COLUMN IF EXISTS likelihood_indicator; 4 | -------------------------------------------------------------------------------- /migrations/010.undo.remove_scans_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE scans ADD COLUMN IF NOT EXISTS hidden boolean NOT NULL DEFAULT false; 2 | ALTER TABLE scans ADD COLUMN IF NOT EXISTS likelihood_indicator VARCHAR NULL; 3 | 4 | CREATE INDEX IF NOT EXISTS scans_hidden_idx ON scans(hidden); 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@mdn/mdn-http-observatory", 3 | "version": "1.4.0", 4 | "author": "Mozilla Developer Network", 5 | "description": "The MDN HTTP Observatory is a set of tools to analyze your website and inform you if you are utilizing the many available methods to secure it.", 6 | "main": "src/index.js", 7 | "engines": { 8 | "node": ">=20.0.0", 9 | "npm": ">=9.0.0" 10 | }, 11 | "scripts": { 12 | "start": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem node src/api/index.js", 13 | "dev": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem nodemon src/api/index.js", 14 | "test": "CONFIG_FILE=conf/config-test.json mocha", 15 | "test:nodb": "CONFIG_FILE=conf/config-test.json SKIP_DB_TESTS=1 mocha", 16 | "tsc": "tsc -p jsconfig.json", 17 | "updateHsts": "node src/retrieve-hsts.js", 18 | "refreshMaterializedViews": "node src/maintenance/index.js", 19 | "migrate": "node -e 'import(\"./src/database/migrate.js\").then( m => m.migrateDatabase() )'" 20 | }, 21 | "bin": { 22 | "mdn-http-observatory-scan": "bin/wrapper.js" 23 | }, 24 | "type": "module", 25 | "license": "MPL-2.0", 26 | "devDependencies": { 27 | "@faker-js/faker": "^9.2.0", 28 | "@supercharge/promise-pool": "^3.2.0", 29 | "@types/chai": "^5.0.1", 30 | "@types/convict": "^6.1.6", 31 | "@types/ip": "^1.1.3", 32 | "@types/jsdom": "^21.1.7", 33 | "@types/mocha": "^10.0.10", 34 | "@types/pg-format": "^1.0.5", 35 | "@types/tough-cookie": "^4.0.5", 36 | "chai": "^5.1.2", 37 | "json-schema-to-jsdoc": "^1.1.1", 38 | "mocha": "^11.0.1", 39 | "nodemon": "^3.1.7", 40 | "prettier-eslint": "^16.3.0", 41 | "typescript": "^5.7.2" 42 | }, 43 | "dependencies": { 44 | "@fastify/cors": "^10.0.1", 45 | "@fastify/helmet": "^13.0.0", 46 | "@fastify/postgres": "^6.0.1", 47 | "@fastify/static": "^8.0.3", 48 | "@sentry/node": "^8.41.0", 49 | "axios": "^1.7.8", 50 | "axios-cookiejar-support": "^5.0.3", 51 | "change-case": "^5.4.4", 52 | "commander": "^13.1.0", 53 | "convict": "^6.2.4", 54 | "dayjs": "^1.11.13", 55 | "fastify": "^5.1.0", 56 | "fastify-simple-form": "^3.0.0", 57 | "http-cookie-agent": "^6.0.6", 58 | "ip": "^2.0.1", 59 | "jsdom": "^26.0.0", 60 | "node_extra_ca_certs_mozilla_bundle": "^1.0.6", 61 | "pg": "^8.13.1", 62 | "pg-format": "^1.0.4", 63 | "pg-pool": "^3.7.0", 64 | "postgrator": "^8.0.0", 65 | "postgrator-cli": "^9.0.0", 66 | "tldts": "^6.1.65", 67 | "tough-cookie": "^5.0.0" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /release-please-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", 3 | "last-release-sha": "0904b8b0cae851a13bb393affacf1982e3ea6222", 4 | "release-type": "node", 5 | "changelog-sections": [ 6 | { "type": "feat", "section": "Features", "hidden": false }, 7 | { "type": "fix", "section": "Bug Fixes", "hidden": false }, 8 | { "type": "enhance", "section": "Enhancements", "hidden": false }, 9 | { "type": "chore", "section": "Miscellaneous", "hidden": false } 10 | ], 11 | "include-component-in-tag": false, 12 | "packages": { 13 | ".": {} 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/analyzer/hsts.js: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "node:path"; 3 | import { fileURLToPath } from "node:url"; 4 | 5 | const dirname = path.dirname(fileURLToPath(import.meta.url)); 6 | 7 | /** 8 | * @type {import("../types.js").Hsts | null} 9 | */ 10 | let hstsMap = null; 11 | 12 | /** 13 | * @returns {import("../types.js").Hsts} 14 | */ 15 | export function hsts() { 16 | if (!hstsMap) { 17 | const filePath = path.join( 18 | dirname, 19 | "..", 20 | "..", 21 | "conf", 22 | "hsts-preload.json" 23 | ); 24 | hstsMap = new Map( 25 | Object.entries(JSON.parse(fs.readFileSync(filePath, "utf8"))) 26 | ); 27 | } 28 | return hstsMap; 29 | } 30 | 31 | /** 32 | * 33 | * @param {string} hostname 34 | * @returns {import("../types.js").Hst | null} 35 | */ 36 | export function isHstsPreloaded(hostname) { 37 | const h = hsts(); 38 | 39 | // Check if the hostname is in the HSTS list with the right mode 40 | const existing = h.get(hostname); 41 | if (existing && existing.mode === "force-https") { 42 | return existing; 43 | } 44 | 45 | // Either the hostname is in the list *or* the TLD is and includeSubDomains is true 46 | const hostParts = hostname.split("."); 47 | const levels = hostParts.length; 48 | 49 | // If hostname is foo.bar.baz.mozilla.org, check bar.baz.mozilla.org, 50 | // baz.mozilla.org, mozilla.org, and.org 51 | for (hostParts.shift(); hostParts.length > 0; hostParts.shift()) { 52 | const domain = hostParts.join("."); 53 | const exist = h.get(domain); 54 | if (exist && exist.mode === "force-https" && exist.includeSubDomains) { 55 | return exist; 56 | } 57 | } 58 | return null; 59 | } 60 | -------------------------------------------------------------------------------- /src/analyzer/tests/cors.js: -------------------------------------------------------------------------------- 1 | import { 2 | ACCESS_CONTROL_ALLOW_CREDENTIALS, 3 | ACCESS_CONTROL_ALLOW_ORIGIN, 4 | ORIGIN, 5 | } from "../../headers.js"; 6 | import { BaseOutput, Requests } from "../../types.js"; 7 | import { Expectation } from "../../types.js"; 8 | import { getFirstHttpHeader, getHttpHeaders } from "../utils.js"; 9 | 10 | export class CorsOutput extends BaseOutput { 11 | /** @type {string | null} */ 12 | data = null; 13 | static name = "cross-origin-resource-sharing"; 14 | static title = "Cross Origin Resource Sharing (CORS)"; 15 | static possibleResults = [ 16 | Expectation.CrossOriginResourceSharingNotImplemented, 17 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess, 18 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess, 19 | Expectation.CrossOriginResourceSharingImplementedWithUniversalAccess, 20 | ]; 21 | 22 | /** 23 | * 24 | * @param {Expectation} expectation 25 | */ 26 | constructor(expectation) { 27 | super(expectation); 28 | } 29 | } 30 | 31 | /** 32 | * 33 | * @param {Requests} requests 34 | * @param {Expectation} expectation 35 | * @returns {CorsOutput} 36 | */ 37 | export function crossOriginResourceSharingTest( 38 | requests, 39 | expectation = Expectation.CrossOriginResourceSharingNotImplemented 40 | ) { 41 | const output = new CorsOutput(expectation); 42 | output.result = Expectation.CrossOriginResourceSharingNotImplemented; 43 | const accessControlAllowOrigin = requests.responses.cors; 44 | 45 | const acaoHeader = getFirstHttpHeader( 46 | accessControlAllowOrigin, 47 | ACCESS_CONTROL_ALLOW_ORIGIN 48 | ); 49 | const originHeader = getFirstHttpHeader( 50 | accessControlAllowOrigin?.request, 51 | ORIGIN 52 | ); 53 | const credentialsHeader = getFirstHttpHeader( 54 | accessControlAllowOrigin, 55 | ACCESS_CONTROL_ALLOW_CREDENTIALS 56 | ); 57 | 58 | if (accessControlAllowOrigin && acaoHeader) { 59 | output.data = acaoHeader.slice(0, 256).trim().toLowerCase(); 60 | if (output.data === "*") { 61 | output.result = 62 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess; 63 | } else if ( 64 | originHeader && 65 | acaoHeader && 66 | originHeader === acaoHeader && 67 | credentialsHeader && 68 | credentialsHeader.toLowerCase().trim() === "true" 69 | ) { 70 | output.result = 71 | Expectation.CrossOriginResourceSharingImplementedWithUniversalAccess; 72 | } else { 73 | output.result = 74 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess; 75 | } 76 | } 77 | 78 | // Check to see if the test passed or failed 79 | if ( 80 | [ 81 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess, 82 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess, 83 | expectation, 84 | ].includes(output.result) 85 | ) { 86 | output.pass = true; 87 | } 88 | return output; 89 | } 90 | -------------------------------------------------------------------------------- /src/analyzer/tests/cross-origin-resource-policy.js: -------------------------------------------------------------------------------- 1 | import { CROSS_ORIGIN_RESOURCE_POLICY } from "../../headers.js"; 2 | import { BaseOutput, Requests } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getFirstHttpHeader } from "../utils.js"; 5 | 6 | export class CrossOriginResourcePolicyOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | http = false; 10 | meta = false; 11 | static name = "cross-origin-resource-policy"; 12 | static title = "Cross Origin Resource Policy"; 13 | static possibleResults = [ 14 | Expectation.CrossOriginResourcePolicyNotImplemented, 15 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin, 16 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite, 17 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin, 18 | Expectation.CrossOriginResourcePolicyHeaderInvalid, 19 | ]; 20 | 21 | /** 22 | * 23 | * @param {Expectation} expectation 24 | */ 25 | constructor(expectation) { 26 | super(expectation); 27 | } 28 | } 29 | 30 | /** 31 | * 32 | * @param {Requests} requests 33 | * @param {Expectation} expectation 34 | * @returns {CrossOriginResourcePolicyOutput} 35 | */ 36 | export function crossOriginResourcePolicyTest( 37 | requests, 38 | expectation = Expectation.CrossOriginResourcePolicyImplementedWithSameSite 39 | ) { 40 | const output = new CrossOriginResourcePolicyOutput(expectation); 41 | output.result = Expectation.CrossOriginResourcePolicyNotImplemented; 42 | 43 | const resp = requests.responses.auto; 44 | if (!resp) { 45 | return output; 46 | } 47 | 48 | const httpHeader = getFirstHttpHeader(resp, CROSS_ORIGIN_RESOURCE_POLICY); 49 | const equivHeaders = 50 | resp.httpEquiv?.get(CROSS_ORIGIN_RESOURCE_POLICY) ?? null; 51 | 52 | // Store whether the header or the meta tag were present 53 | output.http = !!httpHeader; 54 | output.meta = equivHeaders ? equivHeaders.length > 0 : false; 55 | 56 | // If it is both a header and a http-equiv, http-equiv has precedence (last value) 57 | let corpHeader; 58 | if (output.http && httpHeader) { 59 | corpHeader = httpHeader.slice(0, 256).trim().toLowerCase(); 60 | } else if (output.meta) { 61 | // const headers = resp.httpEquiv?.get("cross-origin-resource-policy"); 62 | if (equivHeaders && equivHeaders.length) { 63 | corpHeader = equivHeaders[equivHeaders.length - 1] 64 | .slice(0, 256) 65 | .trim() 66 | .toLowerCase(); 67 | } 68 | } 69 | 70 | if (corpHeader) { 71 | output.data = corpHeader; 72 | if (corpHeader === "same-site") { 73 | output.result = 74 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite; 75 | } else if (corpHeader === "same-origin") { 76 | output.result = 77 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin; 78 | } else if (corpHeader === "cross-origin") { 79 | output.result = 80 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin; 81 | } else { 82 | output.result = Expectation.CrossOriginResourcePolicyHeaderInvalid; 83 | } 84 | } 85 | 86 | // Check to see if the test passed or failed 87 | output.pass = [ 88 | expectation, 89 | Expectation.CrossOriginResourcePolicyNotImplemented, 90 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite, 91 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin, 92 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin, 93 | ].includes(output.result ?? ""); 94 | 95 | return output; 96 | } 97 | -------------------------------------------------------------------------------- /src/analyzer/tests/redirection.js: -------------------------------------------------------------------------------- 1 | import { BaseOutput, Requests } from "../../types.js"; 2 | import { Expectation } from "../../types.js"; 3 | import { isHstsPreloaded } from "../hsts.js"; 4 | 5 | export class RedirectionOutput extends BaseOutput { 6 | /** @type {string | null} */ 7 | destination = null; 8 | redirects = true; 9 | /** @type {string[]} */ 10 | route = []; 11 | /** @type {number | null} */ 12 | statusCode = null; 13 | static name = "redirection"; 14 | static title = "Redirection"; 15 | static possibleResults = [ 16 | Expectation.RedirectionAllRedirectsPreloaded, 17 | Expectation.RedirectionToHttps, 18 | Expectation.RedirectionNotNeededNoHttp, 19 | Expectation.RedirectionOffHostFromHttp, 20 | Expectation.RedirectionNotToHttpsOnInitialRedirection, 21 | Expectation.RedirectionNotToHttps, 22 | Expectation.RedirectionMissing, 23 | Expectation.RedirectionInvalidCert, 24 | ]; 25 | 26 | /** 27 | * 28 | * @param {Expectation} expectation 29 | */ 30 | constructor(expectation) { 31 | super(expectation); 32 | } 33 | } 34 | 35 | /** 36 | * 37 | * @param {Requests} requests 38 | * @param {Expectation} expectation 39 | * @returns {RedirectionOutput} 40 | */ 41 | export function redirectionTest( 42 | requests, 43 | expectation = Expectation.RedirectionToHttps 44 | ) { 45 | const output = new RedirectionOutput(expectation); 46 | const response = requests.responses.http; 47 | 48 | if (requests.responses.httpRedirects.length > 0) { 49 | output.destination = 50 | requests.responses.httpRedirects[ 51 | requests.responses.httpRedirects.length - 1 52 | ].url.href; 53 | } else if (requests.responses.httpsRedirects.length > 0) { 54 | output.destination = 55 | requests.responses.httpsRedirects[ 56 | requests.responses.httpsRedirects.length - 1 57 | ].url.href; 58 | } 59 | output.statusCode = response ? response.status : null; 60 | 61 | if (!response) { 62 | output.result = Expectation.RedirectionNotNeededNoHttp; 63 | } else if (!response.verified) { 64 | output.result = Expectation.RedirectionInvalidCert; 65 | } else { 66 | const route = requests.responses.httpRedirects; 67 | output.route = route.map((r) => r.url.href); 68 | 69 | // Check to see if every redirection was covered by the preload list 70 | const allRedirectsPreloaded = route.every((re) => 71 | isHstsPreloaded(re.url.hostname) 72 | ); 73 | if (allRedirectsPreloaded) { 74 | output.result = Expectation.RedirectionAllRedirectsPreloaded; 75 | } else if (route.length === 1) { 76 | // No redirection, so you just stayed on the http website 77 | output.result = Expectation.RedirectionMissing; 78 | output.redirects = false; 79 | } else if (route[route.length - 1].url.protocol !== "https:") { 80 | // Final destination wasn't an https website 81 | output.result = Expectation.RedirectionNotToHttps; 82 | } else if (route[1].url.protocol === "http:") { 83 | // http should never redirect to another http location -- should always go to https first 84 | output.result = Expectation.RedirectionNotToHttpsOnInitialRedirection; 85 | output.statusCode = route[route.length - 1].status; 86 | } else if ( 87 | route[0].url.protocol === "http:" && 88 | route[1].url.protocol === "https:" && 89 | route[0].url.hostname !== route[1].url.hostname 90 | ) { 91 | output.result = Expectation.RedirectionOffHostFromHttp; 92 | } else { 93 | // Yeah, you're good 94 | output.result = Expectation.RedirectionToHttps; 95 | } 96 | } 97 | // Code defensively against infinite routing loops and other shenanigans 98 | output.route = JSON.stringify(output.route).length > 8192 ? [] : output.route; 99 | output.statusCode = 100 | `${output.statusCode}`.length < 5 ? output.statusCode : null; 101 | 102 | // Check to see if the test passed or failed 103 | if ( 104 | [ 105 | Expectation.RedirectionNotNeededNoHttp, 106 | Expectation.RedirectionAllRedirectsPreloaded, 107 | expectation, 108 | ].includes(output.result) 109 | ) { 110 | output.pass = true; 111 | } 112 | 113 | return output; 114 | } 115 | -------------------------------------------------------------------------------- /src/analyzer/tests/referrer-policy.js: -------------------------------------------------------------------------------- 1 | import { REFERRER_POLICY } from "../../headers.js"; 2 | import { Requests, BaseOutput } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getFirstHttpHeader, getHttpHeaders } from "../utils.js"; 5 | 6 | export class ReferrerOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | http = false; 10 | meta = false; 11 | static name = "referrer-policy"; 12 | static title = "Referrer Policy"; 13 | static possibleResults = [ 14 | Expectation.ReferrerPolicyPrivate, 15 | Expectation.ReferrerPolicyNotImplemented, 16 | Expectation.ReferrerPolicyUnsafe, 17 | Expectation.ReferrerPolicyHeaderInvalid, 18 | ]; 19 | 20 | /** 21 | * 22 | * @param {Expectation} expectation 23 | */ 24 | constructor(expectation) { 25 | super(expectation); 26 | } 27 | } 28 | 29 | /** 30 | * 31 | * @param {Requests} requests 32 | * @param {Expectation} expectation 33 | * @returns {ReferrerOutput} 34 | */ 35 | export function referrerPolicyTest( 36 | requests, 37 | expectation = Expectation.ReferrerPolicyPrivate 38 | ) { 39 | const output = new ReferrerOutput(expectation); 40 | const goodness = [ 41 | "no-referrer", 42 | "same-origin", 43 | "strict-origin", 44 | "strict-origin-when-cross-origin", 45 | ]; 46 | const badness = [ 47 | "origin", 48 | "origin-when-cross-origin", 49 | "unsafe-url", 50 | "no-referrer-when-downgrade", 51 | ]; 52 | const valid = goodness.concat(badness); 53 | 54 | const response = requests.responses.auto; 55 | if (!response) { 56 | output.result = Expectation.ReferrerPolicyNotImplemented; 57 | return output; 58 | } 59 | 60 | const httpHeaders = getHttpHeaders(response, REFERRER_POLICY); 61 | const equivHeaders = response.httpEquiv?.get(REFERRER_POLICY) ?? []; 62 | 63 | // Store whether the header or the meta tag were present 64 | output.http = httpHeaders.length > 0; 65 | output.meta = equivHeaders ? equivHeaders?.length > 0 : false; 66 | 67 | // If it is both a header and a http-equiv, http-equiv has precedence (last value) 68 | if (output.http || output.meta) { 69 | output.data = [...httpHeaders, ...equivHeaders].join(", "); 70 | } else { 71 | output.result = Expectation.ReferrerPolicyNotImplemented; 72 | output.pass = true; 73 | return output; 74 | } 75 | 76 | // Find the last known valid policy value in the referrer policy 77 | let policy = 78 | output.data 79 | ?.split(",") 80 | .filter((e) => valid.includes(e.toLowerCase().trim())) 81 | .reverse()[0] 82 | ?.toLowerCase() 83 | .trim() ?? ""; 84 | 85 | if (goodness.includes(policy)) { 86 | output.result = Expectation.ReferrerPolicyPrivate; 87 | } else if (badness.includes(policy)) { 88 | output.result = Expectation.ReferrerPolicyUnsafe; 89 | } else { 90 | output.result = Expectation.ReferrerPolicyHeaderInvalid; 91 | } 92 | 93 | // Test if passed or fail 94 | output.pass = [ 95 | Expectation.ReferrerPolicyPrivate, 96 | Expectation.ReferrerPolicyNotImplemented, 97 | ].includes(output.result); 98 | 99 | return output; 100 | } 101 | -------------------------------------------------------------------------------- /src/analyzer/tests/strict-transport-security.js: -------------------------------------------------------------------------------- 1 | import { STRICT_TRANSPORT_SECURITY } from "../../headers.js"; 2 | import { Requests, BaseOutput } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { isHstsPreloaded } from "../hsts.js"; 5 | import { getHttpHeaders } from "../utils.js"; 6 | export class StrictTransportSecurityOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | includeSubDomains = false; 10 | /** @type {number | null} */ 11 | maxAge = null; 12 | preload = false; 13 | preloaded = false; 14 | static name = "strict-transport-security"; 15 | static title = "Strict Transport Security (HSTS)"; 16 | static possibleResults = [ 17 | Expectation.HstsPreloaded, 18 | Expectation.HstsImplementedMaxAgeAtLeastSixMonths, 19 | Expectation.HstsImplementedMaxAgeLessThanSixMonths, 20 | Expectation.HstsNotImplemented, 21 | Expectation.HstsHeaderInvalid, 22 | Expectation.HstsNotImplementedNoHttps, 23 | Expectation.HstsInvalidCert, 24 | ]; 25 | /** 26 | * 27 | * @param {Expectation} expectation 28 | */ 29 | constructor(expectation) { 30 | super(expectation); 31 | } 32 | } 33 | 34 | // 15768000 is six months, but a lot of sites use 15552000, so a white lie is in order 35 | const SIX_MONTHS = 15552000; 36 | 37 | /** 38 | * 39 | * @param {Requests} requests 40 | * @param {Expectation} expectation 41 | * @returns {StrictTransportSecurityOutput} 42 | */ 43 | export function strictTransportSecurityTest( 44 | requests, 45 | expectation = Expectation.HstsImplementedMaxAgeAtLeastSixMonths 46 | ) { 47 | const output = new StrictTransportSecurityOutput(expectation); 48 | output.result = Expectation.HstsNotImplemented; 49 | const response = requests.responses.https; 50 | if (!response) { 51 | // If there's no HTTPS, we can't have HSTS 52 | output.result = Expectation.HstsNotImplementedNoHttps; 53 | } else if (!response.verified) { 54 | // Also need a valid certificate chain for HSTS 55 | output.result = Expectation.HstsInvalidCert; 56 | } else if (getHttpHeaders(response, STRICT_TRANSPORT_SECURITY).length > 0) { 57 | const header = getHttpHeaders(response, STRICT_TRANSPORT_SECURITY)[0]; 58 | output.data = header.slice(0, 1024); // code against malicious headers 59 | 60 | try { 61 | let sts = output.data.split(";").map((i) => i.trim().toLowerCase()); 62 | // Throw an error if the header is set twice 63 | if (output.data.includes(",")) { 64 | throw new Error("Header set multiple times"); 65 | } 66 | sts.forEach((parameter) => { 67 | if (parameter.startsWith("max-age=")) { 68 | // Use slice to get the part of the string after 'max-age=' 69 | // Parse it to an integer. We're slicing up to 128 characters as a defense mechanism. 70 | output.maxAge = parseInt(parameter.slice(8, 128), 10); 71 | } else if (parameter === "includesubdomains") { 72 | output.includeSubDomains = true; 73 | } else if (parameter === "preload") { 74 | output.preload = true; 75 | } 76 | }); 77 | if (output.maxAge !== null) { 78 | if (output.maxAge < SIX_MONTHS) { 79 | output.result = Expectation.HstsImplementedMaxAgeLessThanSixMonths; 80 | } else { 81 | output.result = Expectation.HstsImplementedMaxAgeAtLeastSixMonths; 82 | } 83 | } else { 84 | throw new Error("MaxAge missing"); 85 | } 86 | } catch (e) { 87 | output.result = Expectation.HstsHeaderInvalid; 88 | } 89 | } 90 | 91 | // If they're in the preloaded list, this overrides most anything else 92 | if (response) { 93 | const preloaded = isHstsPreloaded(requests.hostname); 94 | if (preloaded) { 95 | output.result = Expectation.HstsPreloaded; 96 | output.includeSubDomains = preloaded.includeSubDomains; 97 | output.preloaded = true; 98 | } 99 | } 100 | // Check to see if the test passed or failed 101 | if ( 102 | [ 103 | Expectation.HstsImplementedMaxAgeAtLeastSixMonths, 104 | Expectation.HstsPreloaded, 105 | expectation, 106 | ].includes(output.result) 107 | ) { 108 | output.pass = true; 109 | } 110 | return output; 111 | } 112 | -------------------------------------------------------------------------------- /src/analyzer/tests/x-content-type-options.js: -------------------------------------------------------------------------------- 1 | import { X_CONTENT_TYPE_OPTIONS } from "../../headers.js"; 2 | import { BaseOutput, Requests } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getFirstHttpHeader } from "../utils.js"; 5 | 6 | export class XContentTypeOptionsOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | static name = "x-content-type-options"; 10 | static title = "X-Content-Type-Options"; 11 | static possibleResults = [ 12 | Expectation.XContentTypeOptionsNosniff, 13 | Expectation.XContentTypeOptionsHeaderInvalid, 14 | Expectation.XContentTypeOptionsNotImplemented, 15 | ]; 16 | 17 | /** 18 | * 19 | * @param {Expectation} expectation 20 | */ 21 | constructor(expectation) { 22 | super(expectation); 23 | } 24 | } 25 | 26 | /** 27 | * 28 | * @param {Requests} requests 29 | * @param {Expectation} expectation 30 | * @returns {XContentTypeOptionsOutput} 31 | */ 32 | export function xContentTypeOptionsTest( 33 | requests, 34 | expectation = Expectation.XContentTypeOptionsNosniff 35 | ) { 36 | const output = new XContentTypeOptionsOutput(expectation); 37 | const resp = requests.responses.auto; 38 | 39 | if (!resp) { 40 | output.result = Expectation.XContentTypeOptionsNotImplemented; 41 | return output; 42 | } 43 | 44 | const header = getFirstHttpHeader(resp, X_CONTENT_TYPE_OPTIONS); 45 | 46 | if (header) { 47 | output.data = header.slice(0, 256); 48 | if (output.data.trim().toLowerCase() === "nosniff") { 49 | output.result = Expectation.XContentTypeOptionsNosniff; 50 | } else { 51 | output.result = Expectation.XContentTypeOptionsHeaderInvalid; 52 | } 53 | } else { 54 | output.result = Expectation.XContentTypeOptionsNotImplemented; 55 | } 56 | 57 | // Check to see if the test passed or failed 58 | output.pass = output.result === expectation; 59 | return output; 60 | } 61 | -------------------------------------------------------------------------------- /src/analyzer/tests/x-frame-options.js: -------------------------------------------------------------------------------- 1 | import { X_FRAME_OPTIONS } from "../../headers.js"; 2 | import { BaseOutput, Requests } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getFirstHttpHeader } from "../utils.js"; 5 | import { contentSecurityPolicyTest } from "./csp.js"; 6 | 7 | export class XFrameOptionsOutput extends BaseOutput { 8 | /** @type {string | null} */ 9 | data = null; 10 | static name = "x-frame-options"; 11 | static title = "X-Frame-Options"; 12 | static possibleResults = [ 13 | Expectation.XFrameOptionsImplementedViaCsp, 14 | Expectation.XFrameOptionsSameoriginOrDeny, 15 | Expectation.XFrameOptionsAllowFromOrigin, 16 | Expectation.XFrameOptionsNotImplemented, 17 | Expectation.XFrameOptionsHeaderInvalid, 18 | ]; 19 | 20 | /** 21 | * 22 | * @param {Expectation} expectation 23 | */ 24 | constructor(expectation) { 25 | super(expectation); 26 | } 27 | } 28 | 29 | /** 30 | * 31 | * @param {Requests} requests 32 | * @param {Expectation} expectation 33 | * @returns {XFrameOptionsOutput} 34 | */ 35 | export function xFrameOptionsTest( 36 | requests, 37 | expectation = Expectation.XFrameOptionsSameoriginOrDeny 38 | ) { 39 | const output = new XFrameOptionsOutput(expectation); 40 | const resp = requests.responses.auto; 41 | 42 | if (!resp) { 43 | output.result = Expectation.XFrameOptionsNotImplemented; 44 | return output; 45 | } 46 | 47 | const header = getFirstHttpHeader(resp, X_FRAME_OPTIONS); 48 | 49 | if (header) { 50 | output.data = header.slice(0, 1024); 51 | const xfo = output.data.trim().toLowerCase(); 52 | if (["deny", "sameorigin"].includes(xfo)) { 53 | output.result = Expectation.XFrameOptionsSameoriginOrDeny; 54 | } else if (xfo.startsWith("allow-from")) { 55 | output.result = Expectation.XFrameOptionsAllowFromOrigin; 56 | } else { 57 | output.result = Expectation.XFrameOptionsHeaderInvalid; 58 | } 59 | } else { 60 | output.result = Expectation.XFrameOptionsNotImplemented; 61 | } 62 | 63 | // Check to see if frame-ancestors is implemented in CSP; if it is, then it isn't needed 64 | const csp = contentSecurityPolicyTest(requests); 65 | if (csp.data && csp.data["frame-ancestors"]) { 66 | output.result = Expectation.XFrameOptionsImplementedViaCsp; 67 | } 68 | 69 | // Check to see if the test passed or failed 70 | if ( 71 | [ 72 | Expectation.XFrameOptionsAllowFromOrigin, 73 | Expectation.XFrameOptionsSameoriginOrDeny, 74 | Expectation.XFrameOptionsImplementedViaCsp, 75 | expectation, 76 | ].includes(output.result) 77 | ) { 78 | output.pass = true; 79 | } 80 | return output; 81 | } 82 | -------------------------------------------------------------------------------- /src/analyzer/utils.js: -------------------------------------------------------------------------------- 1 | import { Expectation } from "../types.js"; 2 | 3 | /** 4 | * Return the new result if it's worse than the existing result, otherwise just the current result. 5 | * @param {Expectation} newResult - The new result to compare. 6 | * @param {Expectation | null} oldResult - The existing result to compare against. 7 | * @param {Expectation[]} order - An array defining the order of results from best to worst. 8 | * @returns {Expectation} - The worse of the two results. 9 | */ 10 | export function onlyIfWorse(newResult, oldResult, order) { 11 | if (!oldResult) { 12 | return newResult; 13 | } else if (order.indexOf(newResult) > order.indexOf(oldResult)) { 14 | return newResult; 15 | } else { 16 | return oldResult; 17 | } 18 | } 19 | 20 | /** 21 | * @param {import("../types.js").Response | null} response 22 | * @param {string} name 23 | * @returns {string[]} 24 | */ 25 | export function getHttpHeaders(response, name) { 26 | if (!response) { 27 | return []; 28 | } 29 | const axiosHeaders = response.headers; 30 | if (!axiosHeaders) { 31 | return []; 32 | } 33 | const lcName = name.toLowerCase(); 34 | const headers = Object.entries(axiosHeaders) 35 | .filter(([headerName, _value]) => { 36 | return headerName.toLowerCase() === lcName; 37 | }) 38 | .map(([_headerName, value]) => value) 39 | .flat(); 40 | return headers; 41 | } 42 | 43 | /** 44 | * @param {import("../types.js").Response | null} response 45 | * @param {string} name 46 | * @returns {string | null} 47 | */ 48 | export function getFirstHttpHeader(response, name) { 49 | if (!response) { 50 | return null; 51 | } 52 | return getHttpHeaders(response, name)[0] ?? null; 53 | } 54 | -------------------------------------------------------------------------------- /src/api/errors.js: -------------------------------------------------------------------------------- 1 | import { STATUS_CODES } from "./utils.js"; 2 | 3 | export class AppError extends Error { 4 | // @ts-ignore 5 | constructor(...args) { 6 | super(...args); 7 | this.name = "error-unknown"; 8 | this.statusCode = STATUS_CODES.internalServerError; 9 | } 10 | } 11 | 12 | export class SiteIsDownError extends AppError { 13 | constructor() { 14 | super("Site is down"); 15 | this.name = "site-down"; 16 | this.statusCode = STATUS_CODES.badRequest; 17 | } 18 | } 19 | 20 | export class NotFoundError extends AppError { 21 | constructor() { 22 | super("Resource Not Found"); 23 | this.name = "not-found"; 24 | this.statusCode = STATUS_CODES.notFound; 25 | } 26 | } 27 | export class ScanFailedError extends AppError { 28 | /** 29 | * @param {Error} e 30 | */ 31 | constructor(e) { 32 | super("Scan Failed"); 33 | this.name = "scan-failed"; 34 | this.statusCode = STATUS_CODES.internalServerError; 35 | this.message = e.message; 36 | } 37 | } 38 | export class InvalidHostNameIpError extends AppError { 39 | constructor() { 40 | super("Cannot scan IP addresses"); 41 | this.name = "invalid-hostname-ip"; 42 | this.statusCode = STATUS_CODES.unprocessableEntity; 43 | } 44 | } 45 | 46 | export class InvalidHostNameError extends AppError { 47 | constructor() { 48 | super(`Invalid hostname`); 49 | this.name = "invalid-hostname"; 50 | this.statusCode = STATUS_CODES.unprocessableEntity; 51 | } 52 | } 53 | 54 | export class InvalidHostNameLookupError extends AppError { 55 | /** 56 | * 57 | * @param {string} hostname 58 | */ 59 | constructor(hostname) { 60 | super(`${hostname} cannot be resolved`); 61 | this.name = "invalid-hostname-lookup"; 62 | this.statusCode = STATUS_CODES.unprocessableEntity; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/api/global-error-handler.js: -------------------------------------------------------------------------------- 1 | import { AppError, InvalidHostNameError } from "./errors.js"; 2 | import { STATUS_CODES } from "./utils.js"; 3 | 4 | /** 5 | * @type {import("../types.js").StringMap} 6 | */ 7 | const errorInfo = { 8 | FST_ERR_VALIDATION: "Validation error", 9 | }; 10 | 11 | /** 12 | * Global error handler 13 | * @param {import("fastify").FastifyError} error 14 | * @param {import("fastify").FastifyRequest} request 15 | * @param {import("fastify").FastifyReply} reply 16 | * @returns {Promise} 17 | */ 18 | export default async function globalErrorHandler(error, request, reply) { 19 | if (error instanceof AppError) { 20 | return reply.status(error.statusCode).send({ 21 | error: error.name, 22 | message: error.message, 23 | }); 24 | } 25 | return reply 26 | .status(error.statusCode ?? STATUS_CODES.internalServerError) 27 | .send({ 28 | error: "error-unknown", 29 | message: error.message, 30 | }); 31 | } 32 | -------------------------------------------------------------------------------- /src/api/index.js: -------------------------------------------------------------------------------- 1 | import { CONFIG } from "../config.js"; 2 | import { migrateDatabase } from "../database/migrate.js"; 3 | import { createServer } from "./server.js"; 4 | 5 | async function main() { 6 | const server = await createServer(); 7 | try { 8 | await server.listen({ 9 | host: "0.0.0.0", 10 | port: CONFIG.api.port, 11 | }); 12 | } catch (error) { 13 | console.error(error); 14 | process.exit(1); 15 | } 16 | } 17 | 18 | main(); 19 | -------------------------------------------------------------------------------- /src/api/server.js: -------------------------------------------------------------------------------- 1 | import Fastify from "fastify"; 2 | import simpleFormPlugin from "fastify-simple-form"; 3 | import cors from "@fastify/cors"; 4 | import helmet from "@fastify/helmet"; 5 | import { init as initSentry, setupFastifyErrorHandler } from "@sentry/node"; 6 | 7 | // import analyzeApiV1 from "./v1/analyze/index.js"; 8 | import analyzeApiV2 from "./v2/analyze/index.js"; 9 | import scanApiV2 from "./v2/scan/index.js"; 10 | import statsApiV2 from "./v2/stats/index.js"; 11 | import recommendationMatrixApiV2 from "./v2/recommendations/index.js"; 12 | import version from "./version/index.js"; 13 | import globalErrorHandler from "./global-error-handler.js"; 14 | import pool from "@fastify/postgres"; 15 | import { poolOptions } from "../database/repository.js"; 16 | import { CONFIG } from "../config.js"; 17 | 18 | if (CONFIG.sentry.dsn) { 19 | initSentry({ 20 | dsn: CONFIG.sentry.dsn, 21 | }); 22 | } 23 | 24 | /** 25 | * Creates a Fastify server instance 26 | * @returns {Promise} 27 | */ 28 | export async function createServer() { 29 | const server = Fastify({ 30 | logger: CONFIG.api.enableLogging, 31 | }); 32 | 33 | if (CONFIG.sentry.dsn) { 34 | setupFastifyErrorHandler(server); 35 | } 36 | 37 | // @ts-ignore 38 | server.register(simpleFormPlugin); 39 | await server.register(cors, { 40 | origin: "*", 41 | methods: ["GET", "OPTIONS", "HEAD", "POST"], 42 | maxAge: 86400, 43 | }); 44 | await server.register(helmet, { 45 | contentSecurityPolicy: { 46 | useDefaults: false, 47 | directives: { 48 | defaultSrc: ["'none'"], 49 | baseUri: ["'none'"], 50 | formAction: ["'none'"], 51 | frameAnchestors: ["'none'"], 52 | }, 53 | }, 54 | 55 | hsts: { 56 | maxAge: 63072000, 57 | includeSubDomains: false, 58 | }, 59 | frameguard: { 60 | action: "deny", 61 | }, 62 | xXssProtection: true, 63 | referrerPolicy: { 64 | policy: "no-referrer", 65 | }, 66 | }); 67 | server.register(pool, poolOptions); 68 | server.setErrorHandler(globalErrorHandler); 69 | 70 | server.get("/", {}, async (_request, _reply) => { 71 | return "Welcome to the MDN Observatory!"; 72 | }); 73 | 74 | // await Promise.all([server.register(analyzeApiV1, { prefix: "/api/v1" })]); 75 | await Promise.all([ 76 | server.register(analyzeApiV2, { prefix: "/api/v2" }), 77 | server.register(scanApiV2, { prefix: "/api/v2" }), 78 | server.register(statsApiV2, { prefix: "/api/v2" }), 79 | server.register(recommendationMatrixApiV2, { prefix: "/api/v2" }), 80 | server.register(version, { prefix: "/api/v2" }), 81 | ]); 82 | 83 | ["SIGINT", "SIGTERM"].forEach((signal) => { 84 | process.on(signal, async () => { 85 | await server.close(); 86 | process.exit(0); 87 | }); 88 | }); 89 | 90 | return server; 91 | } 92 | -------------------------------------------------------------------------------- /src/api/utils.js: -------------------------------------------------------------------------------- 1 | export const STATUS_CODES = { 2 | badRequest: 400, 3 | internalServerError: 500, 4 | notFound: 404, 5 | ok: 200, 6 | unauthorized: 401, 7 | unprocessableEntity: 422, 8 | tooManyRequests: 429, 9 | }; 10 | -------------------------------------------------------------------------------- /src/api/v2/analyze/index.js: -------------------------------------------------------------------------------- 1 | import { CONFIG } from "../../../config.js"; 2 | import { selectScanLatestScanByHost } from "../../../database/repository.js"; 3 | import { SCHEMAS } from "../schemas.js"; 4 | import { 5 | checkHostname, 6 | executeScan, 7 | historyForSite, 8 | hydrateTests, 9 | testsForScan, 10 | } from "../utils.js"; 11 | 12 | /** 13 | * @typedef {import("pg").Pool} Pool 14 | */ 15 | 16 | /** 17 | * Register the API - default export 18 | * @param {import('fastify').FastifyInstance} fastify 19 | * @returns {Promise} 20 | */ 21 | export default async function (fastify) { 22 | const pool = fastify.pg.pool; 23 | fastify.get( 24 | "/analyze", 25 | { schema: SCHEMAS.analyzeGet }, 26 | async (request, reply) => { 27 | const query = 28 | /** @type {import("../../v2/schemas.js").AnalyzeReqQuery} */ ( 29 | request.query 30 | ); 31 | let hostname = query.host.trim().toLowerCase(); 32 | hostname = await checkHostname(hostname); 33 | return await scanOrReturnRecent( 34 | fastify, 35 | pool, 36 | hostname, 37 | CONFIG.api.cacheTimeForGet 38 | ); 39 | } 40 | ); 41 | 42 | fastify.post( 43 | "/analyze", 44 | { schema: SCHEMAS.analyzePost }, 45 | async (request, reply) => { 46 | const query = 47 | /** @type {import("../../v2/schemas.js").AnalyzeReqQuery} */ ( 48 | request.query 49 | ); 50 | let hostname = query.host.trim().toLowerCase(); 51 | hostname = await checkHostname(hostname); 52 | return await scanOrReturnRecent( 53 | fastify, 54 | pool, 55 | hostname, 56 | CONFIG.api.cooldown 57 | ); 58 | } 59 | ); 60 | } 61 | 62 | /** 63 | * 64 | * @param {import("fastify").FastifyInstance} fastify 65 | * @param {Pool} pool 66 | * @param {string} hostname 67 | * @param {number} age 68 | * @returns {Promise} 69 | */ 70 | async function scanOrReturnRecent(fastify, pool, hostname, age) { 71 | let scanRow = await selectScanLatestScanByHost(pool, hostname, age); 72 | if (!scanRow) { 73 | // do a rescan 74 | fastify.log.info("Rescanning because no recent scan could be found"); 75 | scanRow = await executeScan(pool, hostname); 76 | } else { 77 | fastify.log.info("Returning a recent scan result"); 78 | } 79 | const scanId = scanRow.id; 80 | const siteId = scanRow.site_id; 81 | 82 | const [rawTests, history] = await Promise.all([ 83 | testsForScan(pool, scanId), 84 | historyForSite(pool, siteId), 85 | ]); 86 | const tests = hydrateTests(rawTests); 87 | scanRow.scanned_at = scanRow.start_time; 88 | 89 | return { 90 | scan: scanRow, 91 | tests, 92 | history, 93 | }; 94 | } 95 | -------------------------------------------------------------------------------- /src/api/v2/recommendations/index.js: -------------------------------------------------------------------------------- 1 | import { ALL_RESULTS, ALL_TESTS } from "../../../constants.js"; 2 | import { SCORE_TABLE, TEST_TOPIC_LINKS } from "../../../grader/charts.js"; 3 | import { SCHEMAS } from "../schemas.js"; 4 | 5 | /** 6 | * Register the API - default export 7 | * @param {import('fastify').FastifyInstance} fastify 8 | * @returns {Promise} 9 | */ 10 | export default async function (fastify) { 11 | const pool = fastify.pg.pool; 12 | 13 | fastify.get( 14 | "/recommendation_matrix", 15 | { schema: SCHEMAS.recommendationMatrix }, 16 | async (request, reply) => { 17 | const res = ALL_RESULTS.map((output) => { 18 | return { 19 | name: output.name, 20 | title: output.title, 21 | mdnLink: TEST_TOPIC_LINKS.get(output.name) || "", 22 | results: output.possibleResults.map((pr) => { 23 | const data = SCORE_TABLE.get(pr); 24 | return data 25 | ? { 26 | name: pr, 27 | scoreModifier: data.modifier, 28 | description: data.description, 29 | recommendation: data.recommendation, 30 | } 31 | : { 32 | name: pr, 33 | scoreModifier: 0, 34 | description: "", 35 | recommendation: "", 36 | }; 37 | }), 38 | }; 39 | }); 40 | return res; 41 | } 42 | ); 43 | } 44 | -------------------------------------------------------------------------------- /src/api/v2/scan/index.js: -------------------------------------------------------------------------------- 1 | import { CONFIG } from "../../../config.js"; 2 | import { selectScanLatestScanByHost } from "../../../database/repository.js"; 3 | import { SCHEMAS } from "../schemas.js"; 4 | import { checkHostname, executeScan } from "../utils.js"; 5 | 6 | /** 7 | * @typedef {import("pg").Pool} Pool 8 | */ 9 | 10 | /** 11 | * Register the API - default export 12 | * @param {import('fastify').FastifyInstance} fastify 13 | * @returns {Promise} 14 | */ 15 | export default async function (fastify) { 16 | const pool = fastify.pg.pool; 17 | fastify.post("/scan", { schema: SCHEMAS.scan }, async (request, reply) => { 18 | const query = /** @type {import("../../v2/schemas.js").ScanQuery} */ ( 19 | request.query 20 | ); 21 | let hostname = query.host.trim().toLowerCase(); 22 | hostname = await checkHostname(hostname); 23 | return await scanOrReturnRecent( 24 | fastify, 25 | pool, 26 | hostname, 27 | CONFIG.api.cooldown 28 | ); 29 | }); 30 | } 31 | 32 | /** 33 | * 34 | * @param {import("fastify").FastifyInstance} fastify 35 | * @param {Pool} pool 36 | * @param {string} hostname 37 | * @param {number} age 38 | * @returns {Promise} 39 | */ 40 | async function scanOrReturnRecent(fastify, pool, hostname, age) { 41 | let scanRow = await selectScanLatestScanByHost(pool, hostname, age); 42 | if (!scanRow) { 43 | // do a rescan 44 | fastify.log.info("Rescanning because no recent scan could be found"); 45 | scanRow = await executeScan(pool, hostname); 46 | } else { 47 | fastify.log.info("Returning a recent scan result"); 48 | } 49 | scanRow.scanned_at = scanRow.start_time; 50 | const siteLink = `https://developer.mozilla.org/en-US/observatory/analyze?host=${encodeURIComponent(hostname)}`; 51 | return { details_url: siteLink, ...scanRow }; 52 | } 53 | -------------------------------------------------------------------------------- /src/api/v2/stats/index.js: -------------------------------------------------------------------------------- 1 | import { selectGradeDistribution } from "../../../database/repository.js"; 2 | import { SCHEMAS } from "../schemas.js"; 3 | 4 | /** 5 | * Register the API - default export 6 | * @param {import('fastify').FastifyInstance} fastify 7 | * @returns {Promise} 8 | */ 9 | export default async function (fastify) { 10 | const pool = fastify.pg.pool; 11 | 12 | fastify.get( 13 | "/grade_distribution", 14 | { schema: SCHEMAS.gradeDistribution }, 15 | async (request, reply) => { 16 | const res = await selectGradeDistribution(pool); 17 | return res; 18 | } 19 | ); 20 | } 21 | -------------------------------------------------------------------------------- /src/api/version/index.js: -------------------------------------------------------------------------------- 1 | import { version } from "tough-cookie"; 2 | import { SCHEMAS } from "../v2/schemas.js"; 3 | import fs from "node:fs"; 4 | import path from "path"; 5 | import { fileURLToPath } from "url"; 6 | 7 | // Get the directory name of the current module 8 | const __filename = fileURLToPath(import.meta.url); 9 | const __dirname = path.dirname(__filename); 10 | 11 | const packageJson = JSON.parse( 12 | fs.readFileSync( 13 | path.join(__dirname, "..", "..", "..", "package.json"), 14 | "utf8" 15 | ) 16 | ); 17 | 18 | /** 19 | * Register the API - default export 20 | * @param {import('fastify').FastifyInstance} fastify 21 | * @returns {Promise} 22 | */ 23 | export default async function (fastify) { 24 | const pool = fastify.pg.pool; 25 | 26 | fastify.get( 27 | "/version", 28 | { schema: SCHEMAS.version }, 29 | async (request, reply) => { 30 | /** @type {import("../../types.js").VersionResponse} */ 31 | const ret = { 32 | version: packageJson.version, 33 | commit: process.env.GIT_SHA || "unknown", 34 | source: "https://github.com/mdn/mdn-http-observatory", 35 | build: process.env.RUN_ID || "unknown", 36 | }; 37 | return ret; 38 | } 39 | ); 40 | } 41 | -------------------------------------------------------------------------------- /src/config.js: -------------------------------------------------------------------------------- 1 | import convict from "convict"; 2 | 3 | const SCHEMA = { 4 | retriever: { 5 | retrieverUserAgent: { 6 | doc: "The user agent to use for retriever requests.", 7 | format: "String", 8 | default: 9 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:129.0) Gecko/20100101 Firefox/129.0 Observatory/129.0", 10 | env: "RETRIEVER_USER_AGENT", 11 | }, 12 | corsOrigin: { 13 | doc: "The CORS origin to use for CORS origin retriever requests.", 14 | format: "String", 15 | default: "https://http-observatory.security.mozilla.org", 16 | env: "CORS_ORIGIN", 17 | }, 18 | abortTimeout: { 19 | doc: "The overall timeout for a request, in ms", 20 | format: "Number", 21 | default: 10000, 22 | env: "ABORT_TIMEOUT", 23 | }, 24 | clientTimeout: { 25 | doc: "The timeout once the request has been sent, in ms", 26 | format: "Number", 27 | default: 9000, 28 | env: "CLIENT_TIMEOUT", 29 | }, 30 | }, 31 | database: { 32 | database: { 33 | doc: "The name of the database to use", 34 | format: "String", 35 | default: "httpobservatory", 36 | env: "PGDATABASE", 37 | }, 38 | host: { 39 | doc: "The database server hostname", 40 | format: "String", 41 | default: "localhost", 42 | env: "PGHOST", 43 | }, 44 | user: { 45 | doc: "Database username", 46 | format: "String", 47 | default: "postgres", 48 | env: "PGUSER", 49 | }, 50 | pass: { 51 | doc: "Database password", 52 | format: "String", 53 | default: "", 54 | sensitive: true, 55 | env: "PGPASSWORD", 56 | }, 57 | port: { 58 | doc: "The port of the database service", 59 | format: "port", 60 | default: 5432, 61 | env: "PGPORT", 62 | }, 63 | sslmode: { 64 | doc: "Database SSL mode", 65 | format: "Boolean", 66 | default: false, 67 | env: "PGSSLMODE", 68 | }, 69 | }, 70 | api: { 71 | cooldown: { 72 | doc: "Cached result time for API V2, in Seconds. Defaults to 1 minute", 73 | format: "nat", 74 | default: 60, 75 | env: "HTTPOBS_API_COOLDOWN", 76 | }, 77 | cacheTimeForGet: { 78 | doc: "Maximum scan age a GET request returns before initiating a new scan, in seconds. Defaults to 24 hours.", 79 | format: "nat", 80 | default: 86400, 81 | env: "HTTPOBS_API_GET_CACHE", 82 | }, 83 | port: { 84 | doc: "The port to bind to", 85 | format: "Number", 86 | default: 8080, 87 | env: "HTTPOBS_API_PORT", 88 | }, 89 | enableLogging: { 90 | doc: "Enable server logging", 91 | format: "Boolean", 92 | default: true, 93 | env: "HTTPOBS_ENABLE_LOGGING", 94 | }, 95 | }, 96 | sentry: { 97 | dsn: { 98 | doc: "The Sentry data source name (DSN) to use for error reporting.", 99 | format: "String", 100 | default: "", 101 | env: "SENTRY_DSN", 102 | }, 103 | }, 104 | }; 105 | 106 | /** 107 | * 108 | * @param {string | undefined} configFile 109 | * @returns 110 | */ 111 | export function load(configFile) { 112 | const configuration = convict(SCHEMA); 113 | try { 114 | if (configFile) { 115 | configuration.loadFile(configFile); 116 | } 117 | configuration.validate({ allowed: "strict" }); 118 | return configuration.getProperties(); 119 | } catch (e) { 120 | throw new Error(`error reading config: ${e}`); 121 | } 122 | } 123 | 124 | export const CONFIG = load(process.env["CONFIG_FILE"]); 125 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | import { CookiesOutput, cookiesTest } from "./analyzer/tests/cookies.js"; 2 | import { 3 | CorsOutput, 4 | crossOriginResourceSharingTest, 5 | } from "./analyzer/tests/cors.js"; 6 | import { 7 | CrossOriginResourcePolicyOutput, 8 | crossOriginResourcePolicyTest, 9 | } from "./analyzer/tests/cross-origin-resource-policy.js"; 10 | import { contentSecurityPolicyTest, CspOutput } from "./analyzer/tests/csp.js"; 11 | import { 12 | RedirectionOutput, 13 | redirectionTest, 14 | } from "./analyzer/tests/redirection.js"; 15 | import { 16 | ReferrerOutput, 17 | referrerPolicyTest, 18 | } from "./analyzer/tests/referrer-policy.js"; 19 | import { 20 | StrictTransportSecurityOutput, 21 | strictTransportSecurityTest, 22 | } from "./analyzer/tests/strict-transport-security.js"; 23 | import { 24 | SubresourceIntegrityOutput, 25 | subresourceIntegrityTest, 26 | } from "./analyzer/tests/subresource-integrity.js"; 27 | import { 28 | XContentTypeOptionsOutput, 29 | xContentTypeOptionsTest, 30 | } from "./analyzer/tests/x-content-type-options.js"; 31 | import { 32 | XFrameOptionsOutput, 33 | xFrameOptionsTest, 34 | } from "./analyzer/tests/x-frame-options.js"; 35 | 36 | export const ALL_TESTS = [ 37 | contentSecurityPolicyTest, 38 | cookiesTest, 39 | crossOriginResourceSharingTest, 40 | redirectionTest, 41 | referrerPolicyTest, 42 | strictTransportSecurityTest, 43 | subresourceIntegrityTest, 44 | xContentTypeOptionsTest, 45 | xFrameOptionsTest, 46 | crossOriginResourcePolicyTest, 47 | ]; 48 | 49 | export const ALL_RESULTS = [ 50 | CspOutput, 51 | CookiesOutput, 52 | CorsOutput, 53 | RedirectionOutput, 54 | ReferrerOutput, 55 | StrictTransportSecurityOutput, 56 | SubresourceIntegrityOutput, 57 | XContentTypeOptionsOutput, 58 | XFrameOptionsOutput, 59 | CrossOriginResourcePolicyOutput, 60 | ]; 61 | 62 | export const NUM_TESTS = ALL_TESTS.length; 63 | 64 | export const ALGORITHM_VERSION = 4; 65 | -------------------------------------------------------------------------------- /src/database/migrate.js: -------------------------------------------------------------------------------- 1 | import Postgrator from "postgrator"; 2 | import path, { dirname } from "node:path"; 3 | import { fileURLToPath } from "node:url"; 4 | import { createPool } from "./repository.js"; 5 | 6 | const MIGRATION_PATTERN = path.join( 7 | dirname(fileURLToPath(import.meta.url)), 8 | "..", 9 | "..", 10 | "migrations", 11 | "*" 12 | ); 13 | 14 | /** 15 | * @typedef {import("pg").Pool} Pool 16 | */ 17 | 18 | /** 19 | * 20 | * @param {string} version 21 | * @param {Pool} [pool] 22 | */ 23 | export async function migrateDatabase(version, pool) { 24 | const owned_pool = !pool; 25 | if (owned_pool) { 26 | pool = createPool(); 27 | } 28 | if (!pool) { 29 | throw new Error("Pool is invalid"); 30 | } 31 | 32 | try { 33 | const postgrator = new Postgrator({ 34 | migrationPattern: MIGRATION_PATTERN, 35 | driver: "pg", 36 | execQuery: (query) => pool.query(query), 37 | }); 38 | const _appliedMigrations = await postgrator.migrate(version); 39 | } catch (e) { 40 | console.error(e); 41 | } finally { 42 | if (owned_pool) { 43 | await pool.end(); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/grader/grader.js: -------------------------------------------------------------------------------- 1 | import { Expectation } from "../types.js"; 2 | import { GRADE_CHART, SCORE_TABLE, TEST_TOPIC_LINKS } from "./charts.js"; 3 | 4 | /** 5 | * @typedef {Object} GradeAndScore 6 | * @property {number} score 7 | * @property {string} grade 8 | */ 9 | 10 | /** 11 | * 12 | * @param {number} score - raw score based on all of the tests 13 | * @returns {GradeAndScore} - normalized score and grade 14 | */ 15 | export function getGradeForScore(score) { 16 | score = Math.max(score, 0); 17 | 18 | // If score>100, just use the grade for 100, otherwise round down to the nearest multiple of 5 19 | const scoreMapKey = Math.min(score - (score % 5), 100); 20 | const grade = GRADE_CHART.get(scoreMapKey); 21 | 22 | if (!grade) { 23 | throw new Error(`Score of ${scoreMapKey} did not map to a grade`); 24 | } 25 | 26 | return { 27 | score, 28 | grade, 29 | }; 30 | } 31 | 32 | /** 33 | * @param {Expectation} expectation 34 | * @returns {string} 35 | */ 36 | export function getScoreDescription(expectation) { 37 | return SCORE_TABLE.get(expectation)?.description ?? ""; 38 | } 39 | 40 | /** 41 | * @param {Expectation} expectation 42 | * @returns {string} 43 | */ 44 | export function getRecommendation(expectation) { 45 | return SCORE_TABLE.get(expectation)?.recommendation ?? ""; 46 | } 47 | 48 | /** 49 | * @param {string} testName 50 | * @returns {string} 51 | */ 52 | export function getTopicLink(testName) { 53 | return TEST_TOPIC_LINKS.get(testName) ?? ""; 54 | } 55 | 56 | /** 57 | * @param {Expectation} expectation 58 | * @returns {number} 59 | */ 60 | export function getScoreModifier(expectation) { 61 | return SCORE_TABLE.get(expectation)?.modifier ?? 0; 62 | } 63 | 64 | // 65 | // Helper function to line up score table and expectations 66 | export function matchScoreTableAndExpectations() { 67 | for (const exp of Object.values(Expectation)) { 68 | if (!SCORE_TABLE.get(exp)) { 69 | console.log(`No entry in SCORE_TABLE for Expectation ${exp}`); 70 | } 71 | } 72 | const eset = new Set(Object.values(Expectation)); 73 | SCORE_TABLE.forEach((_, key) => { 74 | if (!eset.has(key)) { 75 | console.log(`Expectation '${key}' has no matching entry in SCORE_TABLE`); 76 | } 77 | }); 78 | } 79 | -------------------------------------------------------------------------------- /src/headers.js: -------------------------------------------------------------------------------- 1 | export const CONTENT_SECURITY_POLICY = "content-security-policy"; 2 | export const CONTENT_SECURITY_POLICY_REPORT_ONLY = 3 | "content-security-policy-report-only"; 4 | export const REFERRER_POLICY = "referrer-policy"; 5 | export const SET_COOKIE = "set-cookie"; 6 | export const ACCESS_CONTROL_ALLOW_ORIGIN = "access-control-allow-origin"; 7 | export const ORIGIN = "origin"; 8 | export const ACCESS_CONTROL_ALLOW_CREDENTIALS = 9 | "access-control-allow-credentials"; 10 | export const CROSS_ORIGIN_RESOURCE_POLICY = "cross-origin-resource-policy"; 11 | export const STRICT_TRANSPORT_SECURITY = "strict-transport-security"; 12 | export const CONTENT_TYPE = "content-type"; 13 | export const X_CONTENT_TYPE_OPTIONS = "x-content-type-options"; 14 | export const X_FRAME_OPTIONS = "x-frame-options"; 15 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | export { scan } from "./scanner/index.js"; 2 | -------------------------------------------------------------------------------- /src/maintenance/index.js: -------------------------------------------------------------------------------- 1 | import { 2 | createPool, 3 | refreshMaterializedViews, 4 | } from "../database/repository.js"; 5 | 6 | console.log("Starting MV refresh."); 7 | const pool = createPool(); 8 | const res = await refreshMaterializedViews(pool); 9 | console.log("Successfully refreshed materialized views."); 10 | -------------------------------------------------------------------------------- /src/retrieve-hsts.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { writeFile } from "fs/promises"; 3 | import path from "node:path"; 4 | import { fileURLToPath } from "node:url"; 5 | 6 | const HSTS_URL = new URL( 7 | "https://raw.githubusercontent.com/chromium/chromium/main/net/http/transport_security_state_static.json" 8 | ); 9 | 10 | const SCANNER_PINNED_DOMAINS = [ 11 | "accounts.firefox.com", 12 | "addons.mozilla.org", 13 | "aus4.mozilla.org", 14 | "aus5.mozilla.org", 15 | "cdn.mozilla.org", 16 | "services.mozilla.com", 17 | ]; 18 | 19 | const dirname = path.dirname(fileURLToPath(import.meta.url)); 20 | 21 | /** 22 | * 23 | * @typedef {Object} RawData 24 | * @property {RawEntry[]} entries 25 | * @typedef {Object} RawEntry 26 | * @property {string} name 27 | * @property {string} policy 28 | * @property {string} mode 29 | * @property {string} [include_subdomains] 30 | * @property {string} [include_subdomains_for_pinning] 31 | * @typedef {Object} HstsEntry 32 | * @property {boolean} includeSubDomains 33 | * @property {boolean} includeSubDomainsForPinning 34 | * @property {string} mode 35 | * @property {boolean} pinned 36 | * @typedef {{ [key: string]: HstsEntry }} HstsMap 37 | */ 38 | 39 | /** 40 | * Download the Google HSTS preload list 41 | * @returns 42 | */ 43 | async function retrieveAndStoreHsts() { 44 | let r; 45 | try { 46 | r = await axios.get(HSTS_URL.href); 47 | } catch (error) { 48 | console.error("Error getting data:", error); 49 | return; 50 | } 51 | const data = removeJsonComments(r.data); 52 | /** @type RawData */ 53 | const rawData = JSON.parse(data); 54 | 55 | const hstsMap = rawData.entries.reduce((acc, entry) => { 56 | const domain = entry.name.trim().toLowerCase(); 57 | acc[domain] = { 58 | includeSubDomains: !!entry.include_subdomains, 59 | includeSubDomainsForPinning: 60 | !!entry.include_subdomains || !!entry.include_subdomains_for_pinning, 61 | mode: entry.mode, 62 | // Add in the manually pinned domains 63 | pinned: SCANNER_PINNED_DOMAINS.includes(domain), 64 | }; 65 | return acc; 66 | }, /** @type {HstsMap} */ ({})); 67 | 68 | const filePath = path.join(dirname, "..", "conf", "hsts-preload.json"); 69 | try { 70 | await writeFile(filePath, JSON.stringify(hstsMap, null, 2)); 71 | console.log(`File written to ${filePath}`); 72 | } catch (error) { 73 | console.error("Error writing file:", error); 74 | return; 75 | } 76 | } 77 | 78 | /** 79 | * 80 | * @param {string} jsonString 81 | * @returns {string} 82 | */ 83 | function removeJsonComments(jsonString) { 84 | return jsonString.replace(/\/\/.*$/gm, ""); 85 | } 86 | 87 | await retrieveAndStoreHsts(); 88 | -------------------------------------------------------------------------------- /src/retriever/retriever.js: -------------------------------------------------------------------------------- 1 | import { AxiosHeaders } from "axios"; 2 | import { CONFIG } from "../config.js"; 3 | import { HTML_TYPES, Requests } from "../types.js"; 4 | import { Session, getPageText } from "./session.js"; 5 | import { urls } from "./url.js"; 6 | import { parseHttpEquivHeaders } from "./utils.js"; 7 | 8 | const STANDARD_HEADERS = [ 9 | "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 10 | ]; 11 | const ROBOTS_HEADERS = ["Accept: text/plain,*/*;q=0.8"]; 12 | 13 | /** 14 | * 15 | * @param {*} hostname 16 | * @param {import("../types.js").Options} options 17 | * @returns {Promise} 18 | */ 19 | export async function retrieve(hostname, options = {}) { 20 | const retrievals = new Requests(hostname); 21 | 22 | const { http, https } = urls(hostname, options); 23 | const [httpSession, httpsSession] = await Promise.all([ 24 | Session.fromUrl(http, { headers: STANDARD_HEADERS, ...options }), 25 | Session.fromUrl(https, { headers: STANDARD_HEADERS, ...options }), 26 | ]); 27 | 28 | if (!httpSession && !httpsSession) { 29 | return retrievals; 30 | } 31 | 32 | retrievals.responses.http = httpSession.response; 33 | retrievals.responses.https = httpsSession.response; 34 | 35 | // use the http redirect chain 36 | retrievals.responses.httpRedirects = httpSession.redirectHistory; 37 | retrievals.responses.httpsRedirects = httpSession.redirectHistory; 38 | 39 | if (httpsSession.clientInstanceRecordingRedirects) { 40 | retrievals.responses.auto = httpsSession.response; 41 | retrievals.session = httpsSession; 42 | } else { 43 | retrievals.responses.auto = httpSession.response; 44 | retrievals.session = httpSession; 45 | } 46 | 47 | // Store the contents of the "base" page 48 | retrievals.resources.path = getPageText(retrievals.responses.auto, true); 49 | 50 | // Get robots.txt to gather additional cookies, if any. 51 | await retrievals.session?.get({ 52 | path: "/robots.txt", 53 | headers: new AxiosHeaders(ROBOTS_HEADERS.join("\n")), 54 | }); 55 | 56 | // Do a CORS preflight request 57 | const corsUrl = retrievals.session.redirectHistory[ 58 | retrievals.session.redirectHistory.length - 1 59 | ] 60 | ? retrievals.session.redirectHistory[ 61 | retrievals.session.redirectHistory.length - 1 62 | ].url.href 63 | : retrievals.session.url.href; 64 | const cors_resp = 65 | (await retrievals.session?.options({ 66 | url: corsUrl, 67 | headers: { 68 | "Access-Control-Request-Method": "GET", 69 | Origin: CONFIG.retriever.corsOrigin, 70 | }, 71 | })) || null; 72 | 73 | if (cors_resp) { 74 | retrievals.responses.cors = { 75 | ...cors_resp, 76 | verified: retrievals.session.response?.verified ?? false, 77 | }; 78 | } else { 79 | retrievals.responses.cors = null; 80 | } 81 | 82 | if (retrievals.responses.auto) { 83 | if ( 84 | HTML_TYPES.has( 85 | retrievals.responses.auto.headers["content-type"]?.split(";")[0] 86 | ) && 87 | retrievals.resources.path 88 | ) { 89 | retrievals.responses.auto.httpEquiv = parseHttpEquivHeaders( 90 | retrievals.resources.path, 91 | retrievals.session.url.href 92 | ); 93 | } else { 94 | retrievals.responses.auto.httpEquiv = new Map(); 95 | } 96 | } 97 | 98 | return retrievals; 99 | } 100 | -------------------------------------------------------------------------------- /src/retriever/url.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @param {string} hostname 4 | * @param {import("../types.js").Options} [options] 5 | */ 6 | export function urls(hostname, options = {}) { 7 | return { 8 | http: url(hostname, false, options), 9 | https: url(hostname, true, options), 10 | }; 11 | } 12 | 13 | /** 14 | * 15 | * @param {string} hostname 16 | * @param {boolean} [https] 17 | * @param {import("../types.js").Options} options 18 | * @returns 19 | */ 20 | function url(hostname, https = true, options = {}) { 21 | let port = (https ? options.httpsPort : options.httpPort) ?? ""; 22 | port = port === "" ? "" : `:${port}`; 23 | const url = new URL( 24 | `${https ? "https" : "http"}://${hostname}${port}${options.path ?? ""}` 25 | ); 26 | return url; 27 | } 28 | -------------------------------------------------------------------------------- /src/retriever/utils.js: -------------------------------------------------------------------------------- 1 | import { JSDOM } from "jsdom"; 2 | import { CONTENT_SECURITY_POLICY, REFERRER_POLICY } from "../headers.js"; 3 | 4 | /** 5 | * 6 | * @param {string} html 7 | * @param {string} baseUrl 8 | * @returns {Map} 9 | */ 10 | export function parseHttpEquivHeaders(html, baseUrl) { 11 | /** @type {Map} */ 12 | const httpEquivHeaders = new Map([[CONTENT_SECURITY_POLICY, []]]); 13 | 14 | try { 15 | const dom = JSDOM.fragment(html); 16 | const metas = [...dom.querySelectorAll("meta")]; 17 | 18 | for (const meta of metas) { 19 | if (meta.hasAttribute("http-equiv") && meta.hasAttribute("content")) { 20 | const httpEquiv = meta.getAttribute("http-equiv")?.toLowerCase().trim(); 21 | const content = meta.getAttribute("content"); 22 | if (content && httpEquiv === CONTENT_SECURITY_POLICY) { 23 | httpEquivHeaders.get(CONTENT_SECURITY_POLICY)?.push(content); 24 | } 25 | } else if ( 26 | // Technically not HTTP Equiv, but we're treating it that way 27 | meta.getAttribute("name")?.toLowerCase().trim() === "referrer" 28 | ) { 29 | const attr = meta.getAttribute("content"); 30 | if (attr) { 31 | httpEquivHeaders.set(REFERRER_POLICY, [attr]); 32 | } 33 | } 34 | } 35 | } catch (e) { 36 | console.error("Error parsing HTTP Equiv headers", e); 37 | } 38 | return httpEquivHeaders; 39 | } 40 | -------------------------------------------------------------------------------- /src/scan.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { Command } from "commander"; 4 | import { scan } from "./scanner/index.js"; 5 | 6 | const NAME = "mdn-http-observatory-scan"; 7 | const program = new Command(); 8 | 9 | program 10 | .name(NAME) 11 | .description("CLI for the MDN HTTP Observatory scan functionality") 12 | .version("1.0.0") 13 | .argument("", "hostname to scan") 14 | .action(async (hostname, _options) => { 15 | try { 16 | const result = await scan(hostname); 17 | const tests = Object.fromEntries( 18 | Object.entries(result.tests).map(([key, test]) => { 19 | const { scoreDescription, ...rest } = test; 20 | return [key, rest]; 21 | }) 22 | ); 23 | const ret = { 24 | scan: result.scan, 25 | tests: tests, 26 | }; 27 | console.log(JSON.stringify(ret, null, 2)); 28 | } catch (e) { 29 | if (e instanceof Error) { 30 | console.log(JSON.stringify({ error: e.message })); 31 | process.exit(1); 32 | } 33 | } 34 | }); 35 | 36 | program.parse(); 37 | -------------------------------------------------------------------------------- /src/scanner/index.js: -------------------------------------------------------------------------------- 1 | import { MINIMUM_SCORE_FOR_EXTRA_CREDIT } from "../grader/charts.js"; 2 | import { 3 | getGradeForScore, 4 | getScoreDescription, 5 | getScoreModifier, 6 | } from "../grader/grader.js"; 7 | import { retrieve } from "../retriever/retriever.js"; 8 | import { ALGORITHM_VERSION } from "../constants.js"; 9 | import { NUM_TESTS } from "../constants.js"; 10 | import { ALL_TESTS } from "../constants.js"; 11 | 12 | /** 13 | * @typedef {Object} Options 14 | */ 15 | 16 | /** 17 | * @typedef {import("../types.js").ScanResult} ScanResult 18 | * @typedef {import("../types.js").Output} Output 19 | * @typedef {import("../types.js").StringMap} StringMap 20 | * @typedef {import("../types.js").TestMap} TestMap 21 | */ 22 | 23 | /** 24 | * @param {string} hostname 25 | * @param {Options} [options] 26 | * @returns {Promise} 27 | */ 28 | export async function scan(hostname, options) { 29 | let r = await retrieve(hostname); 30 | if (!r.responses.auto) { 31 | // We cannot connect at all, abort the test. 32 | throw new Error("The site seems to be down."); 33 | } 34 | 35 | // We allow 2xx, 3xx, 401 and 403 status codes 36 | const { status } = r.responses.auto; 37 | if (status < 200 || (status >= 400 && ![401, 403].includes(status))) { 38 | throw new Error( 39 | `Site did respond with an unexpected HTTP status code ${status}.` 40 | ); 41 | } 42 | 43 | // Run all the tests on the result 44 | /** @type {Output[]} */ 45 | const results = ALL_TESTS.map((test) => { 46 | return test(r); 47 | }); 48 | 49 | /** @type {StringMap} */ 50 | const responseHeaders = Object.entries(r.responses.auto.headers).reduce( 51 | (acc, [key, value]) => { 52 | acc[key] = value; 53 | return acc; 54 | }, 55 | /** @type {StringMap} */ ({}) 56 | ); 57 | const statusCode = r.responses.auto.status; 58 | 59 | let testsPassed = 0; 60 | let scoreWithExtraCredit = 100; 61 | let uncurvedScore = scoreWithExtraCredit; 62 | 63 | results.forEach((result) => { 64 | if (result.result) { 65 | result.scoreDescription = getScoreDescription(result.result); 66 | result.scoreModifier = getScoreModifier(result.result); 67 | testsPassed += result.pass ? 1 : 0; 68 | scoreWithExtraCredit += result.scoreModifier; 69 | if (result.scoreModifier < 0) { 70 | uncurvedScore += result.scoreModifier; 71 | } 72 | } 73 | }); 74 | 75 | // Only record the full score if the uncurved score already receives an A 76 | const score = 77 | uncurvedScore >= MINIMUM_SCORE_FOR_EXTRA_CREDIT 78 | ? scoreWithExtraCredit 79 | : uncurvedScore; 80 | 81 | const final = getGradeForScore(score); 82 | 83 | const tests = results.reduce((obj, result) => { 84 | const name = result.constructor.name; 85 | obj[name] = result; 86 | return obj; 87 | }, /** @type {TestMap} */ ({})); 88 | 89 | return { 90 | scan: { 91 | algorithmVersion: ALGORITHM_VERSION, 92 | grade: final.grade, 93 | error: null, 94 | score: final.score, 95 | statusCode: statusCode, 96 | testsFailed: NUM_TESTS - testsPassed, 97 | testsPassed: testsPassed, 98 | testsQuantity: NUM_TESTS, 99 | responseHeaders: responseHeaders, 100 | }, 101 | tests, 102 | }; 103 | } 104 | -------------------------------------------------------------------------------- /test/analyzer-utils.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { AxiosHeaders } from "axios"; 3 | import { getFirstHttpHeader, getHttpHeaders } from "../src/analyzer/utils.js"; 4 | 5 | function emptyResponse() { 6 | return { 7 | headers: new AxiosHeaders("Content-Type: text/html"), 8 | request: { 9 | headers: new AxiosHeaders(), 10 | }, 11 | status: 200, 12 | statusText: "OK", 13 | verified: true, 14 | data: "", 15 | config: { 16 | headers: new AxiosHeaders(), 17 | }, 18 | }; 19 | } 20 | 21 | describe("getHttpHeaders", () => { 22 | it("gets all http headers for a header name", function () { 23 | const response = emptyResponse(); 24 | let headers = getHttpHeaders(response, "content-type"); 25 | assert.isArray(headers); 26 | assert.lengthOf(headers, 1); 27 | assert.equal(headers[0], "text/html"); 28 | headers = getHttpHeaders(response, "Content-Type"); 29 | assert.isArray(headers); 30 | assert.lengthOf(headers, 1); 31 | assert.equal(headers[0], "text/html"); 32 | headers = getHttpHeaders(response, "Non-Existing"); 33 | assert.isArray(headers); 34 | assert.lengthOf(headers, 0); 35 | }); 36 | 37 | it("gets headers correctly when set multiple times", function () { 38 | const response = emptyResponse(); 39 | response.headers.set("X-Test", "hello"); 40 | let headers = getHttpHeaders(response, "x-test"); 41 | assert.isArray(headers); 42 | assert.lengthOf(headers, 1); 43 | response.headers.set("X-Test", ["hello", "world", "1234"]); 44 | headers = getHttpHeaders(response, "x-test"); 45 | assert.isArray(headers); 46 | assert.lengthOf(headers, 3); 47 | assert.equal(headers[0], "hello"); 48 | assert.equal(headers[1], "world"); 49 | assert.equal(headers[2], "1234"); 50 | }); 51 | 52 | it("returns an empty array if the passed in value is `null`", function () { 53 | const headers = getHttpHeaders(null, "content-type"); 54 | assert.isArray(headers); 55 | assert.lengthOf(headers, 0); 56 | }); 57 | }); 58 | 59 | describe("getFirstHttpHeader", () => { 60 | it("gets the first header", function () { 61 | const response = emptyResponse(); 62 | const header = getFirstHttpHeader(response, "content-Type"); 63 | assert.isNotNull(header); 64 | assert.isString(header); 65 | assert.equal(header, "text/html"); 66 | }); 67 | 68 | it("gets the first header on multiple values", function () { 69 | const response = emptyResponse(); 70 | response.headers.set("X-test", ["hello", "world", "1234"]); 71 | const header = getFirstHttpHeader(response, "x-test"); 72 | assert.isNotNull(header); 73 | assert.isString(header); 74 | assert.equal(header, "hello"); 75 | }); 76 | }); 77 | -------------------------------------------------------------------------------- /test/cors.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { crossOriginResourceSharingTest } from "../src/analyzer/tests/cors.js"; 5 | 6 | describe("Cross Origin Resource Sharing", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests(); 11 | }); 12 | 13 | it("checks for not implemented", function () { 14 | const result = crossOriginResourceSharingTest(reqs); 15 | assert.equal( 16 | result.result, 17 | Expectation.CrossOriginResourceSharingNotImplemented 18 | ); 19 | assert.isTrue(result.pass); 20 | }); 21 | 22 | it("checks for public access", function () { 23 | assert.isNotNull(reqs.responses.cors); 24 | reqs.responses.cors.headers["access-control-allow-origin"] = "*"; 25 | const result = crossOriginResourceSharingTest(reqs); 26 | assert.equal( 27 | result.result, 28 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess 29 | ); 30 | assert.equal(result.data, "*"); 31 | assert.isTrue(result.pass); 32 | }); 33 | 34 | it("checks for restricted access", function () { 35 | assert.isNotNull(reqs.responses.cors); 36 | reqs.responses.cors.request.headers["origin"] = 37 | "https://http-observatory.security.mozilla.org"; 38 | reqs.responses.cors.headers["access-control-allow-origin"] = 39 | "https://mozilla.org"; 40 | 41 | const result = crossOriginResourceSharingTest(reqs); 42 | assert.equal( 43 | result.result, 44 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess 45 | ); 46 | assert.isTrue(result.pass); 47 | }); 48 | 49 | it("checks for universal access", function () { 50 | assert.isNotNull(reqs.responses.cors); 51 | reqs.responses.cors.request.headers["origin"] = 52 | "https://http-observatory.security.mozilla.org"; 53 | reqs.responses.cors.headers["access-control-allow-origin"] = 54 | "https://http-observatory.security.mozilla.org"; 55 | reqs.responses.cors.headers["access-control-allow-credentials"] = "true"; 56 | 57 | const result = crossOriginResourceSharingTest(reqs); 58 | assert.equal( 59 | result.result, 60 | Expectation.CrossOriginResourceSharingImplementedWithUniversalAccess 61 | ); 62 | assert.isFalse(result.pass); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /test/cross-origin-resource-policy.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { crossOriginResourcePolicyTest } from "../src/analyzer/tests/cross-origin-resource-policy.js"; 5 | 6 | describe("Cross Origin Resource Policy", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests(); 11 | }); 12 | 13 | it("checks for missing", function () { 14 | const result = crossOriginResourcePolicyTest(reqs); 15 | assert.equal( 16 | result.result, 17 | Expectation.CrossOriginResourcePolicyNotImplemented 18 | ); 19 | assert.isTrue(result.pass); 20 | }); 21 | 22 | it("checks header validity", function () { 23 | const values = ["whimsy"]; 24 | assert.isNotNull(reqs.responses.auto); 25 | for (const value of values) { 26 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 27 | const result = crossOriginResourcePolicyTest(reqs); 28 | assert.equal( 29 | result.result, 30 | Expectation.CrossOriginResourcePolicyHeaderInvalid 31 | ); 32 | assert.isFalse(result.pass); 33 | } 34 | }); 35 | 36 | it("checks for same-site", function () { 37 | const values = ["same-site"]; 38 | assert.isNotNull(reqs.responses.auto); 39 | for (const value of values) { 40 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 41 | const result = crossOriginResourcePolicyTest(reqs); 42 | assert.equal( 43 | result.result, 44 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite 45 | ); 46 | assert.isTrue(result.pass); 47 | } 48 | }); 49 | it("checks for same-origin", function () { 50 | const values = ["same-origin"]; 51 | assert.isNotNull(reqs.responses.auto); 52 | for (const value of values) { 53 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 54 | const result = crossOriginResourcePolicyTest(reqs); 55 | assert.equal( 56 | result.result, 57 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin 58 | ); 59 | assert.isTrue(result.pass); 60 | } 61 | }); 62 | it("checks for cross-origin", function () { 63 | const values = ["cross-origin"]; 64 | assert.isNotNull(reqs.responses.auto); 65 | for (const value of values) { 66 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 67 | const result = crossOriginResourcePolicyTest(reqs); 68 | assert.equal( 69 | result.result, 70 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin 71 | ); 72 | assert.isTrue(result.pass); 73 | } 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_http.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_https1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_https2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_noproto.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_sameorigin.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_content_sri_no_scripts.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /test/files/test_content_sri_notimpl_external_http.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /test/files/test_content_sri_notimpl_external_https.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /test/files/test_content_sri_notimpl_external_noproto.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /test/files/test_content_sri_sameorigin1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /test/files/test_content_sri_sameorigin2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /test/files/test_content_sri_sameorigin3.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_case_insensitivity.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_csp1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_csp2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_csp_multiple_http_equiv1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 10 | 14 | 18 | 22 | Title 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_not_allowed.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | Title 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_referrer1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_x_frame_options.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/grader.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { 3 | getGradeForScore, 4 | getRecommendation, 5 | getScoreDescription, 6 | getScoreModifier, 7 | getTopicLink, 8 | } from "../src/grader/grader.js"; 9 | import { Expectation } from "../src/types.js"; 10 | 11 | describe("Grader", () => { 12 | it("gets the score description", function () { 13 | const res = getScoreDescription(Expectation.CspNotImplemented); 14 | assert.include(res, "Content Security Policy (CSP) header not implemented"); 15 | }); 16 | 17 | it("gets the score modifier", function () { 18 | const res = getScoreModifier(Expectation.CspNotImplemented); 19 | assert.equal(res, -25); 20 | }); 21 | 22 | it("gets the recommendation", function () { 23 | const res = getRecommendation(Expectation.CspNotImplemented); 24 | assert.include(res, "Implement one, see "); 25 | }); 26 | 27 | it("gets the topic link for a test name", function () { 28 | const res = getTopicLink("cookies"); 29 | assert.include( 30 | res, 31 | "/en-US/docs/Web/Security/Practical_implementation_guides/Cookies" 32 | ); 33 | }); 34 | 35 | it("gets the grade", function () { 36 | let res = getGradeForScore(100); 37 | assert.deepEqual(res, { 38 | score: 100, 39 | grade: "A+", 40 | }); 41 | 42 | res = getGradeForScore(0); 43 | assert.deepEqual(res, { 44 | score: 0, 45 | grade: "F", 46 | }); 47 | 48 | res = getGradeForScore(120); 49 | assert.deepEqual(res, { 50 | score: 120, 51 | grade: "A+", 52 | }); 53 | 54 | res = getGradeForScore(-10); 55 | assert.deepEqual(res, { 56 | score: 0, 57 | grade: "F", 58 | }); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /test/helpers.js: -------------------------------------------------------------------------------- 1 | import fs from "node:fs"; 2 | 3 | import { AxiosHeaders } from "axios"; 4 | 5 | import { Requests } from "../src/types.js"; 6 | import { Session } from "../src/retriever/session.js"; 7 | import path from "node:path"; 8 | import { parseHttpEquivHeaders } from "../src/retriever/utils.js"; 9 | 10 | /** 11 | * 12 | * @param {import("../src/types.js").Response | null} response 13 | * @param {string} header 14 | * @param {string} value 15 | */ 16 | export function setHeader(response, header, value) { 17 | if (typeof response?.headers.set === "function") { 18 | response.headers.set(header, value); 19 | } 20 | } 21 | 22 | /** 23 | * 24 | * @param {string | null} [httpEquivFile] 25 | * @returns {Requests} 26 | */ 27 | export function emptyRequests(httpEquivFile = null) { 28 | const req = new Requests("mozilla.org"); 29 | 30 | // Parse the HTML file for its own headers, if requested 31 | if (httpEquivFile) { 32 | const html = fs.readFileSync( 33 | path.join("test", "files", httpEquivFile), 34 | "utf8" 35 | ); 36 | 37 | // Load the HTML file into the object for content tests. 38 | req.resources.path = html; 39 | } 40 | 41 | req.responses.auto = { 42 | headers: new AxiosHeaders("Content-Type: text/html"), 43 | request: { 44 | headers: new AxiosHeaders(), 45 | }, 46 | status: 200, 47 | statusText: "OK", 48 | verified: true, 49 | data: "", 50 | config: { 51 | headers: new AxiosHeaders(), 52 | }, 53 | }; 54 | 55 | req.responses.cors = structuredClone(req.responses.auto); 56 | req.responses.http = structuredClone(req.responses.auto); 57 | req.responses.https = structuredClone(req.responses.auto); 58 | 59 | req.responses.httpRedirects = [ 60 | { 61 | url: new URL("http://mozilla.org/"), 62 | status: 301, 63 | }, 64 | { 65 | url: new URL("https://mozilla.org/"), 66 | status: 301, 67 | }, 68 | { 69 | url: new URL("https://www.mozilla.org/"), 70 | status: 200, 71 | }, 72 | ]; 73 | req.responses.httpsRedirects = [ 74 | { 75 | url: new URL("https://mozilla.org/"), 76 | status: 301, 77 | }, 78 | { 79 | url: new URL("https://www.mozilla.org/"), 80 | status: 200, 81 | }, 82 | { 83 | url: new URL("https://mozilla.org/"), 84 | status: 301, 85 | }, 86 | { 87 | url: new URL("https://www.mozilla.org/"), 88 | status: 200, 89 | }, 90 | { 91 | url: new URL("https://mozilla.org/robots.txt"), 92 | status: 301, 93 | }, 94 | { 95 | url: new URL("https://www.mozilla.org/robots.txt"), 96 | status: 200, 97 | }, 98 | ]; 99 | 100 | req.responses.auto.httpEquiv = new Map(); 101 | 102 | req.session = new Session(new URL("https://mozilla.org/")); 103 | 104 | // Parse the HTML file for its own headers, if requested 105 | if (req.resources.path) { 106 | req.responses.auto.httpEquiv = parseHttpEquivHeaders( 107 | req.resources.path, 108 | req.session.url.href 109 | ); 110 | } 111 | return req; 112 | } 113 | -------------------------------------------------------------------------------- /test/helpers/db.js: -------------------------------------------------------------------------------- 1 | import { faker } from "@faker-js/faker"; 2 | import { ensureSite, ScanState } from "../../src/database/repository.js"; 3 | import { GRADE_CHART } from "../../src/grader/charts.js"; 4 | import { Expectation } from "../../src/types.js"; 5 | import { ALGORITHM_VERSION } from "../../src/constants.js"; 6 | 7 | /** 8 | * @typedef {import("pg").Pool} Pool 9 | */ 10 | 11 | /** 12 | * 13 | * @param {Pool} pool 14 | * @param {any} site 15 | */ 16 | export async function insertSite(pool, site) { 17 | await pool.query( 18 | `INSERT INTO sites (domain, creation_time, public_headers, private_headers, cookies) VALUES ($1, NOW(), $2, $3, $4)`, 19 | [site.domain, site.public_headers, site.private_headers, site.cookies] 20 | ); 21 | } 22 | 23 | /** 24 | * 25 | * @param {Pool} pool 26 | */ 27 | export async function insertSeeds(pool) { 28 | // create a bunch of sites 29 | const siteIds = await Promise.all( 30 | [...Array(10).keys()].map((i) => { 31 | if (i === 0) { 32 | return ensureSite(pool, "www.mozilla.org"); 33 | } else { 34 | return ensureSite(pool, faker.internet.domainName()); 35 | } 36 | }) 37 | ); 38 | // make some random scans for those 39 | const scanIds = ( 40 | await Promise.all( 41 | [...Array(20).keys()].map(async (i) => { 42 | let score = Math.floor(Math.random() * 120); 43 | score -= score % 5; 44 | const grade = GRADE_CHART.get(Math.min(score, 100)); 45 | const siteId = siteIds[i % siteIds.length]; 46 | return pool.query( 47 | `INSERT INTO scans (site_id, state, start_time, end_time, grade, score, tests_quantity, algorithm_version, status_code) 48 | VALUES ($1, 49 | $2, 50 | NOW() - INTERVAL '${(i + 1) * 2000} seconds', 51 | NOW() - INTERVAL '${(i + 1) * 2000} seconds', 52 | $3, 53 | $4, 54 | 9, 55 | $5, 56 | 200) RETURNING id`, 57 | [siteId, ScanState.FINISHED, grade, score, ALGORITHM_VERSION] 58 | ); 59 | }) 60 | ) 61 | ).map((r) => r.rows[0].id); 62 | 63 | await Promise.all( 64 | [...Array(100).keys()].map((i) => { 65 | const siteId = siteIds[i % siteIds.length]; 66 | const scanId = scanIds[i % scanIds.length]; 67 | const expectation = 68 | Object.values(Expectation)[ 69 | Math.floor(Math.random() * Object.values(Expectation).length) 70 | ]; 71 | return pool.query( 72 | `INSERT INTO tests (site_id, scan_id, name, expectation, result, score_modifier, pass, output) 73 | VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, 74 | [ 75 | siteId, 76 | scanId, 77 | ["redirection", "cookies", "referrer-policy"][i % 3], 78 | expectation, 79 | expectation, 80 | [-20, -10, 0, 0, 0, 5, 10][i % 7], 81 | Math.random() > 0.5, 82 | { data: "some data" }, 83 | ] 84 | ); 85 | }) 86 | ); 87 | } 88 | -------------------------------------------------------------------------------- /test/referrer-policy.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { referrerPolicyTest } from "../src/analyzer/tests/referrer-policy.js"; 5 | 6 | describe("ReferrerPolicy", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | 10 | beforeEach(() => { 11 | reqs = emptyRequests(); 12 | }); 13 | 14 | it("checks for private referrer header", function () { 15 | const privValues = [ 16 | "no-referrer", 17 | "same-origin", 18 | "strict-origin", 19 | "STRICT-ORIGIN", 20 | "strict-origin-when-cross-origin", 21 | ]; 22 | assert.isNotNull(reqs.responses.auto); 23 | for (const v of privValues) { 24 | reqs.responses.auto.headers["Referrer-Policy"] = v; 25 | const result = referrerPolicyTest(reqs); 26 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 27 | assert.isTrue(result.http); 28 | assert.isFalse(result.meta); 29 | assert.isTrue(result.pass); 30 | } 31 | 32 | { 33 | // Test with a meta/http-equiv header 34 | reqs = emptyRequests("test_parse_http_equiv_headers_referrer1.html"); 35 | const result = referrerPolicyTest(reqs); 36 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 37 | assert.equal(result.data, "no-referrer, same-origin"); 38 | assert.isFalse(result.http); 39 | assert.isTrue(result.meta); 40 | assert.isTrue(result.pass); 41 | } 42 | 43 | { 44 | // The meta/http-equiv header has precendence over the http header 45 | assert.isNotNull(reqs.responses.auto); 46 | reqs.responses.auto.headers["referrer-policy"] = "unsafe-url"; 47 | const result = referrerPolicyTest(reqs); 48 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 49 | assert.equal(result.data, "unsafe-url, no-referrer, same-origin"); 50 | assert.isTrue(result.http); 51 | assert.isTrue(result.meta); 52 | assert.isTrue(result.pass); 53 | } 54 | }); 55 | 56 | it("checks for missing referrer header", function () { 57 | const result = referrerPolicyTest(reqs); 58 | assert.equal(result.result, Expectation.ReferrerPolicyNotImplemented); 59 | assert.isTrue(result.pass); 60 | }); 61 | 62 | it("checks for invalid referrer header", function () { 63 | assert.isNotNull(reqs.responses.auto); 64 | reqs.responses.auto.headers["referrer-policy"] = "whimsy"; 65 | const result = referrerPolicyTest(reqs); 66 | assert.equal(result.result, Expectation.ReferrerPolicyHeaderInvalid); 67 | assert.isFalse(result.pass); 68 | }); 69 | 70 | it("checks for unsafe referrer header", function () { 71 | const policies = ["origin", "origin-when-cross-origin", "unsafe-url"]; 72 | for (const policy of policies) { 73 | assert.isNotNull(reqs.responses.auto); 74 | reqs.responses.auto.headers["referrer-policy"] = policy; 75 | const result = referrerPolicyTest(reqs); 76 | assert.equal(result.result, Expectation.ReferrerPolicyUnsafe); 77 | assert.isFalse(result.pass); 78 | } 79 | }); 80 | 81 | it("checks for multiple valid referrer headers", function () { 82 | const valid_but_unsafe_policies = [ 83 | "origin-when-cross-origin, no-referrer, unsafe-url", // safe middle value 84 | "no-referrer, unsafe-url", // safe first value 85 | ]; 86 | for (const policy of valid_but_unsafe_policies) { 87 | assert.isNotNull(reqs.responses.auto); 88 | reqs.responses.auto.headers["referrer-policy"] = policy; 89 | const result = referrerPolicyTest(reqs); 90 | assert.equal(result.result, Expectation.ReferrerPolicyUnsafe); 91 | assert.isFalse(result.pass); 92 | } 93 | }); 94 | 95 | it("checks for multiple referrer headers with valid and invalid mixed", function () { 96 | const mixed_valid_invalid_policies = ["no-referrer, whimsy"]; 97 | for (const policy of mixed_valid_invalid_policies) { 98 | assert.isNotNull(reqs.responses.auto); 99 | reqs.responses.auto.headers["referrer-policy"] = policy; 100 | const result = referrerPolicyTest(reqs); 101 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 102 | assert.isTrue(result.pass); 103 | } 104 | }); 105 | 106 | it("checks for multiple invalid referrer headers", function () { 107 | const invalid_policies = ["whimsy, whimsy1, whimsy2"]; 108 | for (const policy of invalid_policies) { 109 | assert.isNotNull(reqs.responses.auto); 110 | reqs.responses.auto.headers["referrer-policy"] = policy; 111 | const result = referrerPolicyTest(reqs); 112 | assert.equal(result.result, Expectation.ReferrerPolicyHeaderInvalid); 113 | assert.isFalse(result.pass); 114 | } 115 | }); 116 | }); 117 | -------------------------------------------------------------------------------- /test/retriever.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | 3 | import { retrieve } from "../src/retriever/retriever.js"; 4 | import { Session } from "../src/retriever/session.js"; 5 | import { Resources } from "../src/types.js"; 6 | 7 | describe("TestRetriever", () => { 8 | it("test retrieve non-existent domain", async function () { 9 | const domain = 10 | Array(223) 11 | .fill(0) 12 | .map(() => String.fromCharCode(Math.random() * 26 + 97)) 13 | .join("") + ".net"; 14 | const requests = await retrieve(domain); 15 | assert.isNull(requests.responses.auto); 16 | assert.isNull(requests.responses.cors); 17 | assert.isNull(requests.responses.http); 18 | assert.isNull(requests.responses.https); 19 | assert.isNotNull(requests.session); 20 | assert.isNull(requests.session.response); 21 | assert.equal(domain, requests.hostname); 22 | assert.deepEqual(new Resources(), requests.resources); 23 | }); 24 | 25 | it("test retrieve mdn", async () => { 26 | const requests = await retrieve("developer.mozilla.org"); 27 | assert.isNotNull(requests.resources.path); 28 | assert.isNotNull(requests.responses.auto); 29 | assert.isNotNull(requests.responses.http); 30 | assert.isNotNull(requests.responses.https); 31 | assert.isNumber(requests.responses.http.status); 32 | assert.isNumber(requests.responses.https.status); 33 | assert.instanceOf(requests.session, Session); 34 | assert.equal(requests.hostname, "developer.mozilla.org"); 35 | assert.equal(requests.responses.httpRedirects.length, 3); 36 | assert.equal( 37 | "text/html", 38 | requests.responses.auto.headers["content-type"].substring(0, 9) 39 | ); 40 | assert.equal(200, requests.responses.auto.status); 41 | assert.equal( 42 | "https://developer.mozilla.org/en-US/", 43 | requests.responses.httpRedirects[ 44 | requests.responses.httpRedirects.length - 1 45 | ].url.href 46 | ); 47 | }).timeout(10000); 48 | 49 | // test site seems to have outage from time to time, disable for now 50 | it.skip("test_retrieve_invalid_cert", async function () { 51 | const reqs = await retrieve("expired.badssl.com"); 52 | assert.isNotNull(reqs.responses.auto); 53 | assert.isFalse(reqs.responses.auto.verified); 54 | }).timeout(10000); 55 | }); 56 | -------------------------------------------------------------------------------- /test/scanner.test.js: -------------------------------------------------------------------------------- 1 | import { assert, expect } from "chai"; 2 | import { scan } from "../src/scanner/index.js"; 3 | 4 | /** @typedef {import("../src/scanner/index.js").ScanResult} ScanResult */ 5 | 6 | describe("Scanner", () => { 7 | it("returns an error on an unknown host", async function () { 8 | const domain = 9 | Array(223) 10 | .fill(0) 11 | .map(() => String.fromCharCode(Math.random() * 26 + 97)) 12 | .join("") + ".net"; 13 | 14 | try { 15 | const scanResult = await scan(domain); 16 | throw new Error("scan should throw"); 17 | } catch (e) { 18 | if (e instanceof Error) { 19 | assert.equal(e.message, "The site seems to be down."); 20 | } else { 21 | throw new Error("Unexpected error type"); 22 | } 23 | } 24 | }); 25 | 26 | it("returns expected results on observatory.mozilla.org", async function () { 27 | const domain = "observatory.mozilla.org"; 28 | const scanResult = await scan(domain); 29 | 30 | assert.equal(scanResult.scan.algorithmVersion, 4); 31 | assert.equal(scanResult.scan.grade, "A+"); 32 | assert.equal(scanResult.scan.score, 100); 33 | assert.equal(scanResult.scan.testsFailed, 0); 34 | assert.equal(scanResult.scan.testsPassed, 10); 35 | assert.equal(scanResult.scan.testsQuantity, 10); 36 | assert.equal(scanResult.scan.statusCode, 200); 37 | assert.equal(scanResult.scan.responseHeaders["content-type"], "text/html"); 38 | }).timeout(5000); 39 | 40 | it("returns expected results on mozilla.org", async function () { 41 | const domain = "mozilla.org"; 42 | const scanResult = await scan(domain); 43 | assert.equal(scanResult.scan.algorithmVersion, 4); 44 | assert.equal(scanResult.scan.grade, "B+"); 45 | assert.equal(scanResult.scan.score, 80); 46 | assert.equal(scanResult.scan.testsFailed, 1); 47 | assert.equal(scanResult.scan.testsPassed, 9); 48 | assert.equal(scanResult.scan.testsQuantity, 10); 49 | assert.equal(scanResult.scan.statusCode, 200); 50 | assert.equal( 51 | // @ts-ignore 52 | scanResult.scan.responseHeaders["content-type"], 53 | "text/html; charset=utf-8" 54 | ); 55 | }).timeout(5000); 56 | }); 57 | -------------------------------------------------------------------------------- /test/strict-transport-security.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { strictTransportSecurityTest } from "../src/analyzer/tests/strict-transport-security.js"; 4 | import { Expectation } from "../src/types.js"; 5 | 6 | describe("Strict Transport Security", () => { 7 | /** 8 | * @type {import("../src/types.js").Requests} 9 | */ 10 | let reqs; 11 | beforeEach(() => { 12 | reqs = emptyRequests(); 13 | }); 14 | 15 | it("missing", function () { 16 | const result = strictTransportSecurityTest(reqs); 17 | assert.equal(result.result, Expectation.HstsNotImplemented); 18 | assert.isFalse(result.pass); 19 | }); 20 | 21 | it("header invalid", function () { 22 | assert.isNotNull(reqs.responses.https); 23 | reqs.responses.https.headers["strict-transport-security"] = 24 | "includeSubDomains; preload"; 25 | let result = strictTransportSecurityTest(reqs); 26 | assert.equal(result.result, Expectation.HstsHeaderInvalid); 27 | assert.isFalse(result.pass); 28 | 29 | reqs.responses.https.headers["strict-transport-security"] = 30 | "max-age=15768000; includeSubDomains, max-age=15768000; includeSubDomains"; 31 | result = strictTransportSecurityTest(reqs); 32 | assert.equal(result.result, Expectation.HstsHeaderInvalid); 33 | assert.isFalse(result.pass); 34 | }); 35 | 36 | it("no https", function () { 37 | assert.isNotNull(reqs.responses.auto); 38 | reqs.responses.auto.headers["strict-transport-security"] = 39 | "max-age=15768000"; 40 | assert.isNotNull(reqs.responses.http); 41 | reqs.responses.http.headers["strict-transport-security"] = 42 | "max-age=15768000"; 43 | reqs.responses.https = null; 44 | 45 | const result = strictTransportSecurityTest(reqs); 46 | assert.equal(result.result, Expectation.HstsNotImplementedNoHttps); 47 | assert.isFalse(result.pass); 48 | }); 49 | 50 | it("invalid cert", function () { 51 | assert.isNotNull(reqs.responses.https); 52 | reqs.responses.https.headers["strict-transport-security"] = 53 | "max-age=15768000; includeSubDomains; preload"; 54 | reqs.responses.https.verified = false; 55 | 56 | const result = strictTransportSecurityTest(reqs); 57 | assert.equal(result.result, Expectation.HstsInvalidCert); 58 | assert.isFalse(result.pass); 59 | }); 60 | 61 | it("max age too low", function () { 62 | assert.isNotNull(reqs.responses.https); 63 | reqs.responses.https.headers["Strict-Transport-Security"] = "max-age=86400"; 64 | 65 | const result = strictTransportSecurityTest(reqs); 66 | assert.equal( 67 | result.result, 68 | Expectation.HstsImplementedMaxAgeLessThanSixMonths 69 | ); 70 | assert.isFalse(result.pass); 71 | }); 72 | 73 | it("implemented", function () { 74 | assert.isNotNull(reqs.responses.https); 75 | reqs.responses.https.headers["strict-transport-security"] = 76 | "max-age=15768000; includeSubDomains; preload"; 77 | 78 | const result = strictTransportSecurityTest(reqs); 79 | assert.equal( 80 | result.result, 81 | Expectation.HstsImplementedMaxAgeAtLeastSixMonths 82 | ); 83 | assert.equal(result.maxAge, 15768000); 84 | assert.isTrue(result.includeSubDomains); 85 | assert.isTrue(result.preload); 86 | assert.isTrue(result.pass); 87 | }); 88 | 89 | it("preloaded", function () { 90 | reqs.hostname = "bugzilla.mozilla.org"; 91 | let result = strictTransportSecurityTest(reqs); 92 | assert.equal(result.result, Expectation.HstsPreloaded); 93 | assert.isTrue(result.includeSubDomains); 94 | assert.isTrue(result.pass); 95 | assert.isTrue(result.preloaded); 96 | 97 | reqs.hostname = "facebook.com"; 98 | result = strictTransportSecurityTest(reqs); 99 | assert.equal(result.result, Expectation.HstsPreloaded); 100 | assert.isFalse(result.includeSubDomains); 101 | assert.isTrue(result.pass); 102 | assert.isTrue(result.preloaded); 103 | 104 | reqs.hostname = "dropboxusercontent.com"; 105 | result = strictTransportSecurityTest(reqs); 106 | assert.equal(result.result, Expectation.HstsNotImplemented); 107 | assert.isFalse(result.includeSubDomains); 108 | assert.isFalse(result.pass); 109 | assert.isFalse(result.preloaded); 110 | }); 111 | }); 112 | -------------------------------------------------------------------------------- /test/x-content-type-options.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { xContentTypeOptionsTest } from "../src/analyzer/tests/x-content-type-options.js"; 5 | 6 | describe("X-Content-Type-Options", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests(); 11 | }); 12 | 13 | it("checks for missing", function () { 14 | const result = xContentTypeOptionsTest(reqs); 15 | assert.equal(result.result, Expectation.XContentTypeOptionsNotImplemented); 16 | assert.isFalse(result.pass); 17 | }); 18 | 19 | it("checks header validity", function () { 20 | const values = ["whimsy", "nosniff, nosniff"]; 21 | assert.isNotNull(reqs.responses.auto); 22 | for (const value of values) { 23 | reqs.responses.auto.headers["x-content-type-options"] = value; 24 | const result = xContentTypeOptionsTest(reqs); 25 | assert.equal(result.result, Expectation.XContentTypeOptionsHeaderInvalid); 26 | assert.isFalse(result.pass); 27 | } 28 | }); 29 | 30 | it("checks for nosniff", function () { 31 | const values = ["nosniff", "nosniff "]; 32 | assert.isNotNull(reqs.responses.auto); 33 | for (const value of values) { 34 | reqs.responses.auto.headers["x-content-type-options"] = value; 35 | const result = xContentTypeOptionsTest(reqs); 36 | assert.equal(result.result, Expectation.XContentTypeOptionsNosniff); 37 | assert.isTrue(result.pass); 38 | } 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /test/x-frame-options.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { xFrameOptionsTest } from "../src/analyzer/tests/x-frame-options.js"; 4 | import { Expectation } from "../src/types.js"; 5 | 6 | describe("X-Frame-Options", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests("test_content_sri_no_scripts.html"); 11 | }); 12 | 13 | it("checks for missing", function () { 14 | const result = xFrameOptionsTest(reqs); 15 | assert.equal(result.result, Expectation.XFrameOptionsNotImplemented); 16 | assert.isFalse(result.pass); 17 | }); 18 | 19 | it("checks validity", function () { 20 | assert.isNotNull(reqs.responses.auto); 21 | reqs.responses.auto.headers["x-frame-options"] = "whimsy"; 22 | let result = xFrameOptionsTest(reqs); 23 | assert.equal(result.result, Expectation.XFrameOptionsHeaderInvalid); 24 | assert.isFalse(result.pass); 25 | 26 | // common to see this header sent multiple times 27 | reqs.responses.auto.headers["x-frame-options"] = "SAMEORIGIN, SAMEORIGIN"; 28 | result = xFrameOptionsTest(reqs); 29 | assert.equal(result.result, Expectation.XFrameOptionsHeaderInvalid); 30 | assert.isFalse(result.pass); 31 | }); 32 | 33 | it("checks allow from origin", function () { 34 | assert.isNotNull(reqs.responses.auto); 35 | reqs.responses.auto.headers["x-frame-options"] = 36 | "ALLOW-FROM https://mozilla.org"; 37 | const result = xFrameOptionsTest(reqs); 38 | assert.equal(result.result, Expectation.XFrameOptionsAllowFromOrigin); 39 | assert.isTrue(result.pass); 40 | }); 41 | 42 | it("checks deny", function () { 43 | assert.isNotNull(reqs.responses.auto); 44 | reqs.responses.auto.headers["x-frame-options"] = "DENY"; 45 | let result = xFrameOptionsTest(reqs); 46 | assert.equal(result.result, Expectation.XFrameOptionsSameoriginOrDeny); 47 | assert.isTrue(result.pass); 48 | 49 | reqs.responses.auto.headers["x-frame-options"] = "DENY "; 50 | result = xFrameOptionsTest(reqs); 51 | assert.equal(result.result, Expectation.XFrameOptionsSameoriginOrDeny); 52 | assert.isTrue(result.pass); 53 | }); 54 | 55 | it("checks implemented via CSP", function () { 56 | assert.isNotNull(reqs.responses.auto); 57 | reqs.responses.auto.headers["x-frame-options"] = "DENY"; 58 | reqs.responses.auto.headers["content-security-policy"] = 59 | "frame-ancestors https://mozilla.org"; 60 | const result = xFrameOptionsTest(reqs); 61 | assert.equal(result.result, Expectation.XFrameOptionsImplementedViaCsp); 62 | assert.isTrue(result.pass); 63 | }); 64 | 65 | it("does not obey x-frame-options in meta equiv tags", function () { 66 | reqs = emptyRequests("test_parse_http_equiv_headers_x_frame_options.html"); 67 | const result = xFrameOptionsTest(reqs); 68 | assert.equal(result.result, Expectation.XFrameOptionsNotImplemented); 69 | assert.isFalse(result.pass); 70 | }); 71 | }); 72 | --------------------------------------------------------------------------------