├── .nvmrc ├── .npmrc ├── .prettierignore ├── .prettierrc.json ├── migrations ├── 001.undo.sites.sql ├── 003.undo.scans.sql ├── 004.undo.tests.sql ├── 002.undo.expectations.sql ├── 010.do.remove_scans_fields.sql ├── 008.do.unique_index_on_sites_domain.sql ├── 008.undo.unique_index_on_sites_domain.sql ├── 009.do.remove_sites_fields.sql ├── 009.undo.remove_sites_fields.sql ├── 005.undo.httpobs-user.sql ├── 010.undo.remove_scans_fields.sql ├── 002.do.expectations.sql ├── 006.undo.mat-views.sql ├── 001.do.sites.sql ├── 004.do.tests.sql ├── 005.do.httpobs-user.sql ├── 007.do.history-mat-view.sql ├── 003.do.scans.sql ├── 007.undo.history-mat-view.sql └── 006.do.mat-views.sql ├── src ├── index.js ├── api │ ├── utils.js │ ├── index.js │ ├── v2 │ │ ├── stats │ │ │ └── index.js │ │ ├── recommendations │ │ │ └── index.js │ │ ├── scan │ │ │ └── index.js │ │ ├── analyze │ │ │ └── index.js │ │ └── utils.js │ ├── global-error-handler.js │ ├── version │ │ └── index.js │ ├── errors.js │ └── server.js ├── maintenance │ └── index.js ├── headers.js ├── database │ └── migrate.js ├── scan.js ├── retriever │ ├── utils.js │ ├── url.js │ └── retriever.js ├── retrieve-tld-list.js ├── analyzer │ ├── utils.js │ ├── hsts.js │ ├── tests │ │ ├── x-content-type-options.js │ │ ├── x-frame-options.js │ │ ├── cors.js │ │ ├── referrer-policy.js │ │ ├── cross-origin-resource-policy.js │ │ ├── strict-transport-security.js │ │ ├── redirection.js │ │ ├── subresource-integrity.js │ │ └── cookies.js │ └── cspParser.js ├── constants.js ├── grader │ └── grader.js ├── retrieve-hsts.js ├── scanner │ └── index.js ├── config.js └── site.js ├── .github ├── release-please-manifest.json ├── dependabot.yml ├── workflows │ ├── release-please.yml │ ├── prod-build.yml │ ├── stage-build.yml │ ├── auto-merge.yml │ ├── test.yml │ └── _build.yml ├── CODEOWNERS ├── release-please-config.json ├── PULL_REQUEST_TEMPLATE ├── ISSUE_TEMPLATE │ ├── config.yml │ └── bug.yml └── settings.yml ├── test ├── files │ ├── test_content_sri_no_scripts.html │ ├── test_content_sri_sameorigin1.html │ ├── test_content_sri_sameorigin3.html │ ├── test_content_sri_sameorigin2.html │ ├── test_content_sri_notimpl_external_noproto.html │ ├── test_content_sri_notimpl_external_http.html │ ├── test_content_sri_notimpl_external_https.html │ ├── test_parse_http_equiv_headers_referrer1.html │ ├── test_parse_http_equiv_headers_x_frame_options.html │ ├── test_parse_http_equiv_headers_case_insensitivity.html │ ├── test_parse_http_equiv_headers_csp1.html │ ├── test_parse_http_equiv_headers_csp2.html │ ├── test_content_sri_impl_external_noproto.html │ ├── test_content_sri_impl_sameorigin.html │ ├── test_parse_http_equiv_headers_not_allowed.html │ ├── test_content_sri_impl_external_https1.html │ ├── test_content_sri_impl_external_https2.html │ ├── test_content_sri_impl_external_http.html │ └── test_parse_http_equiv_headers_csp_multiple_http_equiv1.html ├── x-content-type-options.test.js ├── grader.test.js ├── scanner.test.js ├── cors.test.js ├── site-utils.test.js ├── analyzer-utils.test.js ├── cross-origin-resource-policy.test.js ├── helpers │ └── db.js ├── x-frame-options.test.js ├── helpers.js ├── retriever.test.js ├── strict-transport-security.test.js ├── referrer-policy.test.js ├── redirection.test.js └── subresource-integrity.test.js ├── conf ├── config-example.json └── config-test.json ├── .gitignore ├── .editorconfig ├── CODE_OF_CONDUCT.md ├── .dockerignore ├── jsconfig.json ├── Dockerfile ├── REVIEWING.md ├── bin └── wrapper.js ├── SECURITY.md ├── package.json ├── CONTRIBUTING.md └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | v24 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | engine-strict=true 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | CHANGELOG.md 2 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5" 3 | } 4 | -------------------------------------------------------------------------------- /migrations/001.undo.sites.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS sites; -------------------------------------------------------------------------------- /migrations/003.undo.scans.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS scans; -------------------------------------------------------------------------------- /migrations/004.undo.tests.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS tests; -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | export { scan } from "./scanner/index.js"; 2 | -------------------------------------------------------------------------------- /.github/release-please-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "1.5.1" 3 | } 4 | -------------------------------------------------------------------------------- /migrations/002.undo.expectations.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS expectations; -------------------------------------------------------------------------------- /test/files/test_content_sri_no_scripts.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /conf/config-example.json: -------------------------------------------------------------------------------- 1 | { 2 | "database": { 3 | "database": "observatory", 4 | "user": "postgres" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /migrations/010.do.remove_scans_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE scans 2 | DROP COLUMN IF EXISTS "hidden", 3 | DROP COLUMN IF EXISTS likelihood_indicator; 4 | -------------------------------------------------------------------------------- /test/files/test_content_sri_sameorigin1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /migrations/008.do.unique_index_on_sites_domain.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX IF NOT EXISTS sites_domain_unique_idx ON sites (domain); 2 | DROP INDEX IF EXISTS sites_domain_idx; -------------------------------------------------------------------------------- /migrations/008.undo.unique_index_on_sites_domain.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX IF EXISTS sites_domain_unique_idx; 2 | CREATE INDEX IF NOT EXISTS sites_domain_idx ON sites (domain); 3 | -------------------------------------------------------------------------------- /test/files/test_content_sri_sameorigin3.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /test/files/test_content_sri_sameorigin2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /migrations/009.do.remove_sites_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE sites 2 | DROP COLUMN IF EXISTS public_headers, 3 | DROP COLUMN IF EXISTS private_headers, 4 | DROP COLUMN IF EXISTS cookies; 5 | -------------------------------------------------------------------------------- /test/files/test_content_sri_notimpl_external_noproto.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/api/utils.js: -------------------------------------------------------------------------------- 1 | export const STATUS_CODES = { 2 | badRequest: 400, 3 | internalServerError: 500, 4 | notFound: 404, 5 | ok: 200, 6 | unauthorized: 401, 7 | unprocessableEntity: 422, 8 | tooManyRequests: 429, 9 | }; 10 | -------------------------------------------------------------------------------- /test/files/test_content_sri_notimpl_external_http.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /test/files/test_content_sri_notimpl_external_https.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /migrations/009.undo.remove_sites_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE sites ADD COLUMN IF NOT EXISTS public_headers jsonb; 2 | ALTER TABLE sites ADD COLUMN IF NOT EXISTS private_headers jsonb; 3 | ALTER TABLE sites ADD COLUMN IF NOT EXISTS cookies jsonb; 4 | -------------------------------------------------------------------------------- /migrations/005.undo.httpobs-user.sql: -------------------------------------------------------------------------------- 1 | REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM httpobsapi; 2 | REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM httpobsapi; 3 | REVOKE ALL PRIVILEGES ON SCHEMA public FROM httpobsapi; 4 | DROP USER httpobsapi; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .env 3 | conf/config.json 4 | conf/hsts-preload.json 5 | conf/public-suffix-list.js 6 | conf/tld-list.json 7 | api-examples.http 8 | mdn-observatory-webext/dist 9 | .DS_Store 10 | compare_output.txt 11 | load-script.js 12 | .vscode/ 13 | -------------------------------------------------------------------------------- /migrations/010.undo.remove_scans_fields.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE scans ADD COLUMN IF NOT EXISTS hidden boolean NOT NULL DEFAULT false; 2 | ALTER TABLE scans ADD COLUMN IF NOT EXISTS likelihood_indicator VARCHAR NULL; 3 | 4 | CREATE INDEX IF NOT EXISTS scans_hidden_idx ON scans(hidden); 5 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_referrer1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_x_frame_options.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_case_insensitivity.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_csp1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_csp2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Title 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /conf/config-test.json: -------------------------------------------------------------------------------- 1 | { 2 | "database": { 3 | "database": "observatory_test", 4 | "user": "postgres" 5 | }, 6 | "api": { 7 | "enableLogging": false, 8 | "cooldown": 3 9 | }, 10 | "tests": { 11 | "enableDBTests": true, 12 | "hostForPortAndPathChecks": "" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /migrations/002.do.expectations.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS expectations ( 2 | id SERIAL PRIMARY KEY, 3 | site_id INTEGER REFERENCES sites (id), 4 | test_name VARCHAR NOT NULL, 5 | expectation VARCHAR NOT NULL 6 | ); -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_noproto.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_sameorigin.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_not_allowed.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | Title 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | [*] 7 | indent_style = space 8 | indent_size = 2 9 | end_of_line = lf 10 | charset = utf-8 11 | trim_trailing_whitespace = true 12 | insert_final_newline = true 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false 16 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_https1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_https2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/api/index.js: -------------------------------------------------------------------------------- 1 | import { CONFIG } from "../config.js"; 2 | import { createServer } from "./server.js"; 3 | 4 | async function main() { 5 | const server = await createServer(); 6 | try { 7 | await server.listen({ 8 | host: "0.0.0.0", 9 | port: CONFIG.api.port, 10 | }); 11 | } catch (error) { 12 | console.error(error); 13 | process.exit(1); 14 | } 15 | } 16 | 17 | main(); 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | commit-message: 9 | prefix: "ci(deps): " 10 | 11 | - package-ecosystem: npm 12 | directory: "/" 13 | schedule: 14 | interval: weekly 15 | commit-message: 16 | prefix: "chore(deps): " 17 | prefix-development: "chore(deps-dev): " 18 | -------------------------------------------------------------------------------- /migrations/006.undo.mat-views.sql: -------------------------------------------------------------------------------- 1 | DROP MATERIALIZED VIEW IF EXISTS latest_tests; 2 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution; 3 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution_all_scans; 4 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution_summation; 5 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution; 6 | DROP MATERIALIZED VIEW IF EXISTS earliest_scans; 7 | DROP MATERIALIZED VIEW IF EXISTS latest_scans; 8 | -------------------------------------------------------------------------------- /migrations/001.do.sites.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS sites ( 2 | id SERIAL PRIMARY KEY, 3 | domain VARCHAR(255) NOT NULL, 4 | creation_time TIMESTAMP NOT NULL, 5 | public_headers JSONB NULL, 6 | private_headers JSONB NULL, 7 | cookies JSONB NULL 8 | ); 9 | 10 | CREATE INDEX sites_domain_idx ON sites (domain); 11 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of conduct 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, read [Mozilla's Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 5 | 6 | ## Reporting violations 7 | 8 | For more information on how to report violations of the Community Participation Guidelines, read the [How to report](https://www.mozilla.org/about/governance/policies/participation/reporting/) page. 9 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .github 2 | .git 3 | .vscode 4 | .idea 5 | test 6 | node_modules 7 | npm-debug.log 8 | Dockerfile 9 | .dockerignore 10 | .env 11 | conf/config.json 12 | api-examples.http 13 | .DS_Store 14 | .editorconfig 15 | .gitignore 16 | .prettierrc.json 17 | 18 | # Exclude Temporary and Log Files. 19 | # See: https://wiki.mozilla.org/GitHub/Repository_Security/GitHub_Workflows_%26_Actions#Docker_Security_Best_Practices 20 | *.log 21 | *.tmp 22 | 23 | # Ignore generated credentials from google-github-actions/auth 24 | gha-creds-*.json 25 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": true, 4 | "checkJs": true, 5 | "noEmit": true, 6 | "target": "esnext", 7 | "module": "nodenext", 8 | "moduleResolution": "nodenext", 9 | "maxNodeModuleJsDepth": 0, 10 | "strict": true, 11 | "skipLibCheck": true, 12 | "noUncheckedIndexedAccess": true, 13 | "noUnusedLocals": true, 14 | "noUnusedParameters": true, 15 | "noImplicitReturns": true, 16 | "noFallthroughCasesInSwitch": true 17 | }, 18 | "exclude": ["node_modules"] 19 | } 20 | -------------------------------------------------------------------------------- /test/files/test_content_sri_impl_external_http.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/api/v2/stats/index.js: -------------------------------------------------------------------------------- 1 | import { selectGradeDistribution } from "../../../database/repository.js"; 2 | import { SCHEMAS } from "../schemas.js"; 3 | 4 | /** 5 | * Register the API - default export 6 | * @param {import('fastify').FastifyInstance} fastify 7 | * @returns {Promise} 8 | */ 9 | export default async function (fastify) { 10 | fastify.get( 11 | "/grade_distribution", 12 | { schema: SCHEMAS.gradeDistribution }, 13 | async (_request, _reply) => { 14 | const res = await selectGradeDistribution(fastify.pg.pool); 15 | return res; 16 | } 17 | ); 18 | } 19 | -------------------------------------------------------------------------------- /src/maintenance/index.js: -------------------------------------------------------------------------------- 1 | import { 2 | createPool, 3 | refreshMaterializedViews, 4 | } from "../database/repository.js"; 5 | 6 | console.log("Starting MV refresh."); 7 | const pool = createPool(); 8 | await refreshMaterializedViews(pool); 9 | console.log("Successfully refreshed materialized views."); 10 | 11 | import { retrieveAndStoreTldList } from "../retrieve-tld-list.js"; 12 | await retrieveAndStoreTldList(); 13 | console.log("Successfully updated TLD list."); 14 | 15 | import { retrieveAndStoreHsts } from "../retrieve-hsts.js"; 16 | await retrieveAndStoreHsts(); 17 | console.log("Successfully updated HSTS data."); 18 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:24 2 | 3 | RUN apt-get -y update && \ 4 | apt-get install -y git libpq-dev && \ 5 | mkdir -p /home/node/app/node_modules && \ 6 | chown -R node:node /home/node/app 7 | 8 | WORKDIR /home/node/app 9 | USER node 10 | COPY --chown=node:node . . 11 | # This also installs hsts and tld data files in a postinstall script: 12 | RUN npm ci 13 | 14 | ARG GIT_SHA=dev 15 | ARG RUN_ID=unknown 16 | 17 | RUN env 18 | 19 | ENV RUN_ID=${RUN_ID} 20 | ENV GIT_SHA=${GIT_SHA} 21 | ENV NODE_EXTRA_CA_CERTS=node_modules/extra_certs/ca_bundle/ca_intermediate_bundle.pem 22 | EXPOSE 8080 23 | CMD [ "node", "src/api/index.js" ] 24 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yml: -------------------------------------------------------------------------------- 1 | name: release-please 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | pull-requests: write 11 | 12 | jobs: 13 | release-please: 14 | if: github.repository == 'mdn/mdn-http-observatory' 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Release 18 | uses: GoogleCloudPlatform/release-please-action@16a9c90856f42705d54a6fda1823352bdc62cf38 # v4.4.0 19 | id: release 20 | with: 21 | config-file: .github/release-please-config.json 22 | manifest-file: .github/release-please-manifest.json 23 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # ---------------------------------------------------------------------------- 2 | # CODEOWNERS 3 | # ---------------------------------------------------------------------------- 4 | # Order is important. The last matching pattern takes precedence. 5 | # See: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners 6 | # ---------------------------------------------------------------------------- 7 | 8 | * @mdn/engineering 9 | 10 | /package.json @mdn/engineering @mdn-bot 11 | /package-lock.json @mdn/engineering @mdn-bot 12 | /SECURITY.md @mdn/engineering 13 | -------------------------------------------------------------------------------- /.github/release-please-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", 3 | "last-release-sha": "0904b8b0cae851a13bb393affacf1982e3ea6222", 4 | "release-type": "node", 5 | "changelog-sections": [ 6 | { "type": "feat", "section": "Features", "hidden": false }, 7 | { "type": "fix", "section": "Bug Fixes", "hidden": false }, 8 | { "type": "enhance", "section": "Enhancements", "hidden": false }, 9 | { "type": "chore", "section": "Miscellaneous", "hidden": false } 10 | ], 11 | "include-component-in-tag": false, 12 | "packages": { 13 | ".": {} 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /.github/workflows/prod-build.yml: -------------------------------------------------------------------------------- 1 | name: Prod build 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | workflow_dispatch: 9 | inputs: 10 | tag: 11 | description: "Tag to build (e.g. v1.13.0)" 12 | required: true 13 | 14 | permissions: 15 | # Read/write GHA cache. 16 | actions: write 17 | # Checkout. 18 | contents: read 19 | # Authenticate with GCP. 20 | id-token: write 21 | 22 | jobs: 23 | build: 24 | if: github.repository_owner == 'mdn' 25 | uses: ./.github/workflows/_build.yml 26 | secrets: inherit 27 | with: 28 | environment: prod 29 | ref: ${{ inputs.tag || github.ref }} 30 | tag: ${{ inputs.tag || github.ref_name }} 31 | -------------------------------------------------------------------------------- /.github/workflows/stage-build.yml: -------------------------------------------------------------------------------- 1 | name: Stage build 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | workflow_dispatch: 9 | inputs: 10 | ref: 11 | description: "Branch to build (default: main)" 12 | required: false 13 | 14 | permissions: 15 | # Read/write GHA cache. 16 | actions: write 17 | # Checkout. 18 | contents: read 19 | # Authenticate with GCP. 20 | id-token: write 21 | 22 | jobs: 23 | build: 24 | if: github.repository_owner == 'mdn' && github.actor != 'dependabot[bot]' 25 | uses: ./.github/workflows/_build.yml 26 | secrets: inherit 27 | with: 28 | environment: stage 29 | ref: ${{ inputs.ref || github.event.repository.default_branch }} 30 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE: -------------------------------------------------------------------------------- 1 | 2 | 3 | ### Description 4 | 5 | 6 | 7 | ### Motivation 8 | 9 | 10 | 11 | ### Additional details 12 | 13 | 14 | 15 | ### Related issues and pull requests 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/headers.js: -------------------------------------------------------------------------------- 1 | export const CONTENT_SECURITY_POLICY = "content-security-policy"; 2 | export const CONTENT_SECURITY_POLICY_REPORT_ONLY = 3 | "content-security-policy-report-only"; 4 | export const REFERRER_POLICY = "referrer-policy"; 5 | export const SET_COOKIE = "set-cookie"; 6 | export const ACCESS_CONTROL_ALLOW_ORIGIN = "access-control-allow-origin"; 7 | export const ORIGIN = "origin"; 8 | export const ACCESS_CONTROL_ALLOW_CREDENTIALS = 9 | "access-control-allow-credentials"; 10 | export const CROSS_ORIGIN_RESOURCE_POLICY = "cross-origin-resource-policy"; 11 | export const STRICT_TRANSPORT_SECURITY = "strict-transport-security"; 12 | export const CONTENT_TYPE = "content-type"; 13 | export const X_CONTENT_TYPE_OPTIONS = "x-content-type-options"; 14 | export const X_FRAME_OPTIONS = "x-frame-options"; 15 | -------------------------------------------------------------------------------- /test/files/test_parse_http_equiv_headers_csp_multiple_http_equiv1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 10 | 14 | 18 | 22 | Title 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Content or feature request 4 | url: https://github.com/mdn/mdn/issues/new/choose 5 | about: Propose new content for MDN Web Docs or submit a feature request using this link. 6 | - name: MDN GitHub Discussions 7 | url: https://github.com/orgs/mdn/discussions 8 | about: Does the issue involve a lot of changes, or is it hard to split it into actionable tasks? Start a discussion before opening an issue. 9 | - name: MDN Web Docs on Discourse 10 | url: https://discourse.mozilla.org/c/mdn/learn/250 11 | about: Need help with assessments on MDN Web Docs? We have a support community for this purpose on Discourse. 12 | - name: Help with code 13 | url: https://stackoverflow.com/ 14 | about: If you are stuck and need help with code, StackOverflow is a great resource. 15 | -------------------------------------------------------------------------------- /src/api/global-error-handler.js: -------------------------------------------------------------------------------- 1 | import { AppError } from "./errors.js"; 2 | import { STATUS_CODES } from "./utils.js"; 3 | 4 | /** 5 | * @type {import("../types.js").StringMap} 6 | */ 7 | 8 | /** 9 | * Global error handler 10 | * @param {import("fastify").FastifyError} error 11 | * @param {import("fastify").FastifyRequest} _request 12 | * @param {import("fastify").FastifyReply} reply 13 | * @returns {Promise} 14 | */ 15 | export default async function globalErrorHandler(error, _request, reply) { 16 | if (error instanceof AppError) { 17 | return reply.status(error.statusCode).send({ 18 | error: error.name, 19 | message: error.message, 20 | }); 21 | } 22 | return reply 23 | .status(error.statusCode ?? STATUS_CODES.internalServerError) 24 | .send({ 25 | error: "error-unknown", 26 | message: error.message, 27 | }); 28 | } 29 | -------------------------------------------------------------------------------- /migrations/004.do.tests.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS tests ( 2 | id BIGSERIAL PRIMARY KEY, 3 | site_id INTEGER REFERENCES sites (id) NOT NULL, 4 | scan_id INTEGER REFERENCES scans (id) NOT NULL, 5 | name VARCHAR NOT NULL, 6 | expectation VARCHAR NOT NULL, 7 | result VARCHAR NOT NULL, 8 | score_modifier SMALLINT NOT NULL, 9 | pass BOOL NOT NULL, 10 | output JSONB NOT NULL 11 | ); 12 | 13 | CREATE INDEX tests_scan_id_idx ON tests (scan_id); 14 | CREATE INDEX tests_name_idx ON tests (name); 15 | CREATE INDEX tests_result_idx ON tests (result); 16 | CREATE INDEX tests_pass_idx ON tests (pass); 17 | -------------------------------------------------------------------------------- /migrations/005.do.httpobs-user.sql: -------------------------------------------------------------------------------- 1 | CREATE USER httpobsapi; 2 | GRANT USAGE ON SCHEMA public TO httpobsapi; 3 | GRANT SELECT ON expectations, scans, tests to httpobsapi; 4 | GRANT SELECT (id, domain, creation_time, public_headers) ON sites TO httpobsapi; 5 | GRANT INSERT ON sites, scans TO httpobsapi; 6 | GRANT UPDATE (public_headers, private_headers, cookies) ON sites TO httpobsapi; 7 | GRANT UPDATE ON scans TO httpobsapi; 8 | GRANT USAGE ON SEQUENCE sites_id_seq TO httpobsapi; 9 | GRANT USAGE ON SEQUENCE scans_id_seq TO httpobsapi; 10 | GRANT USAGE ON SEQUENCE expectations_id_seq TO httpobsapi; 11 | GRANT SELECT on sites, scans, expectations, tests TO httpobsapi; 12 | GRANT UPDATE (domain) ON sites to httpobsapi; /* TODO: there's got to be a better way with SELECT ... FOR UPDATE */ 13 | GRANT UPDATE on scans TO httpobsapi; 14 | GRANT INSERT on tests TO httpobsapi; 15 | GRANT USAGE ON SEQUENCE tests_id_seq TO httpobsapi; 16 | -------------------------------------------------------------------------------- /migrations/007.do.history-mat-view.sql: -------------------------------------------------------------------------------- 1 | DROP MATERIALIZED VIEW IF EXISTS latest_tests; 2 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution; 3 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution_all_scans; 4 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution_summation; 5 | DROP MATERIALIZED VIEW IF EXISTS scan_score_difference_distribution; 6 | DROP MATERIALIZED VIEW IF EXISTS earliest_scans; 7 | DROP MATERIALIZED VIEW IF EXISTS latest_scans; 8 | 9 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution AS 10 | SELECT grade, count(*) AS count FROM ( 11 | SELECT DISTINCT ON (scans.site_id) 12 | scans.grade 13 | FROM scans 14 | WHERE scans.end_time > (now() - INTERVAL '1 year') 15 | AND scans.state = 'FINISHED' 16 | ORDER BY scans.site_id, scans.end_time DESC 17 | ) s 18 | GROUP BY grade; 19 | CREATE UNIQUE INDEX IF NOT EXISTS grade_idx ON grade_distribution (grade); 20 | COMMENT ON MATERIALIZED VIEW grade_distribution IS 'Grade distribution over the last year'; 21 | -------------------------------------------------------------------------------- /REVIEWING.md: -------------------------------------------------------------------------------- 1 | # Reviewing guide 2 | 3 | All reviewers must abide by the [code of conduct](CODE_OF_CONDUCT.md); they are also protected by the code of conduct. 4 | A reviewer should not tolerate poor behavior and is encouraged to [report any behavior](CODE_OF_CONDUCT.md#Reporting_violations) that violates the code of conduct. 5 | 6 | ## Review process 7 | 8 | The MDN Web Docs team has a well-defined review process that must be followed by reviewers in all repositories under the GitHub MDN organization. 9 | This process is described in detail on the [Pull request guidelines](https://developer.mozilla.org/en-US/docs/MDN/Community/Pull_requests) page. 10 | 11 | 18 | -------------------------------------------------------------------------------- /.github/workflows/auto-merge.yml: -------------------------------------------------------------------------------- 1 | name: auto-merge 2 | 3 | on: 4 | pull_request_target: 5 | branches: 6 | - main 7 | 8 | # No GITHUB_TOKEN permissions, as we use AUTOMERGE_TOKEN instead. 9 | permissions: {} 10 | 11 | jobs: 12 | auto-merge: 13 | runs-on: ubuntu-latest 14 | if: github.event.pull_request.user.login == 'dependabot[bot]' 15 | 16 | steps: 17 | - name: Dependabot metadata 18 | id: dependabot-metadata 19 | uses: dependabot/fetch-metadata@08eff52bf64351f401fb50d4972fa95b9f2c2d1b # v2.4.0 20 | with: 21 | github-token: ${{ secrets.AUTOMERGE_TOKEN }} 22 | 23 | - name: Squash and merge 24 | if: ${{ steps.dependabot-metadata.outputs.update-type == 'version-update:semver-minor' || steps.dependabot-metadata.outputs.update-type == 'version-update:semver-patch' }} 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} 27 | run: | 28 | gh pr review ${{ github.event.pull_request.html_url }} --approve 29 | gh pr comment ${{ github.event.pull_request.html_url }} --body "@dependabot squash and merge" 30 | -------------------------------------------------------------------------------- /src/database/migrate.js: -------------------------------------------------------------------------------- 1 | import Postgrator from "postgrator"; 2 | import path, { dirname } from "node:path"; 3 | import { fileURLToPath } from "node:url"; 4 | import { createPool } from "./repository.js"; 5 | 6 | const MIGRATION_PATTERN = path.join( 7 | dirname(fileURLToPath(import.meta.url)), 8 | "..", 9 | "..", 10 | "migrations", 11 | "*" 12 | ); 13 | 14 | /** 15 | * @typedef {import("pg").Pool} Pool 16 | */ 17 | 18 | /** 19 | * 20 | * @param {string} version 21 | * @param {Pool} [pool] 22 | */ 23 | export async function migrateDatabase(version, pool) { 24 | const owned_pool = !pool; 25 | if (owned_pool) { 26 | pool = createPool(); 27 | } 28 | if (!pool) { 29 | throw new Error("Pool is invalid"); 30 | } 31 | 32 | try { 33 | const postgrator = new Postgrator({ 34 | migrationPattern: MIGRATION_PATTERN, 35 | driver: "pg", 36 | execQuery: (query) => pool.query(query), 37 | }); 38 | await postgrator.migrate(version); 39 | } catch (e) { 40 | console.error(e); 41 | } finally { 42 | if (owned_pool) { 43 | await pool.end(); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/api/version/index.js: -------------------------------------------------------------------------------- 1 | import { SCHEMAS } from "../v2/schemas.js"; 2 | import fs from "node:fs"; 3 | import path from "path"; 4 | import { fileURLToPath } from "url"; 5 | 6 | // Get the directory name of the current module 7 | const __filename = fileURLToPath(import.meta.url); 8 | const __dirname = path.dirname(__filename); 9 | 10 | const packageJson = JSON.parse( 11 | fs.readFileSync( 12 | path.join(__dirname, "..", "..", "..", "package.json"), 13 | "utf8" 14 | ) 15 | ); 16 | 17 | /** 18 | * Register the API - default export 19 | * @param {import('fastify').FastifyInstance} fastify 20 | * @returns {Promise} 21 | */ 22 | export default async function (fastify) { 23 | fastify.get( 24 | "/version", 25 | { schema: SCHEMAS.version }, 26 | async (_request, _reply) => { 27 | /** @type {import("../../types.js").VersionResponse} */ 28 | const ret = { 29 | version: packageJson.version, 30 | commit: process.env.GIT_SHA || "unknown", 31 | source: "https://github.com/mdn/mdn-http-observatory", 32 | build: process.env.RUN_ID || "unknown", 33 | }; 34 | return ret; 35 | } 36 | ); 37 | } 38 | -------------------------------------------------------------------------------- /src/scan.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import { Command } from "commander"; 4 | import { scan } from "./scanner/index.js"; 5 | import { Site } from "./site.js"; 6 | 7 | const NAME = "mdn-http-observatory-scan"; 8 | const program = new Command(); 9 | 10 | program 11 | .name(NAME) 12 | .description("CLI for the MDN HTTP Observatory scan functionality") 13 | .version("1.0.0") 14 | .argument("", "hostname to scan") 15 | .action(async (siteString, _options) => { 16 | try { 17 | const site = Site.fromSiteString(siteString); 18 | const result = await scan(site); 19 | const tests = Object.fromEntries( 20 | Object.entries(result.tests).map(([key, test]) => { 21 | const { scoreDescription, ...rest } = test; 22 | return [key, rest]; 23 | }) 24 | ); 25 | const ret = { 26 | scan: result.scan, 27 | tests: tests, 28 | }; 29 | console.log(JSON.stringify(ret, null, 2)); 30 | } catch (e) { 31 | if (e instanceof Error) { 32 | console.log(JSON.stringify({ error: e.message })); 33 | process.exit(1); 34 | } 35 | } 36 | }); 37 | 38 | program.parse(); 39 | -------------------------------------------------------------------------------- /bin/wrapper.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | "use strict"; 3 | 4 | import { spawnSync } from "node:child_process"; 5 | import path from "node:path"; 6 | import { fileURLToPath } from "node:url"; 7 | 8 | // Resolve __dirname from ESM environment 9 | const __filename = fileURLToPath(import.meta.url); 10 | const __dirname = path.dirname(__filename); 11 | 12 | // Set the environment variable for extra CA certificates 13 | let caCertPath = import.meta.resolve("node_extra_ca_certs_mozilla_bundle"); 14 | caCertPath = new URL(caCertPath).pathname; 15 | caCertPath = path.dirname(caCertPath); 16 | caCertPath = path.join(caCertPath, "ca_bundle", "ca_intermediate_bundle.pem"); 17 | process.env.NODE_EXTRA_CA_CERTS = caCertPath; 18 | 19 | // The target script you want to run (relative to this script's directory) 20 | const targetScript = path.join(__dirname, "..", "src", "scan.js"); 21 | 22 | // Forward any arguments passed to this script 23 | const args = process.argv.slice(2); 24 | 25 | // Spawn a new Node process to run the target script with inherited stdio 26 | const result = spawnSync(process.execPath, [targetScript, ...args], { 27 | stdio: "inherit", 28 | env: process.env, 29 | }); 30 | 31 | // Exit with the same code the spawned script returned 32 | process.exit(result.status ?? 1); 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.yml: -------------------------------------------------------------------------------- 1 | name: "Issue report" 2 | description: Report an unexpected problem or unintended behavior. 3 | labels: ["needs triage"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | ### Before you start 9 | 10 | **Want to fix the problem yourself?** This project is open source and we welcome fixes and improvements from the community! 11 | 12 | ↩ Check the project [CONTRIBUTING.md](../blob/main/CONTRIBUTING.md) guide to see how to get started. 13 | 14 | --- 15 | - type: textarea 16 | id: problem 17 | attributes: 18 | label: What information was incorrect, unhelpful, or incomplete? 19 | validations: 20 | required: true 21 | - type: textarea 22 | id: expected 23 | attributes: 24 | label: What did you expect to see? 25 | validations: 26 | required: true 27 | - type: textarea 28 | id: references 29 | attributes: 30 | label: Do you have any supporting links, references, or citations? 31 | description: Link to information that helps us confirm your issue. 32 | - type: textarea 33 | id: more-info 34 | attributes: 35 | label: Do you have anything more you want to share? 36 | description: For example, steps to reproduce, screenshots, screen recordings, or sample code. 37 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Overview 4 | 5 | This policy applies to MDN's website (`developer.mozilla.org`), backend services, and GitHub repositories in the [`mdn`](https://github.com/mdn) organization. Issues affecting other Mozilla products or services should be reported through the [Mozilla Security Bug Bounty Program](https://www.mozilla.org/en-US/security/bug-bounty/). 6 | 7 | For non-security issues, please file a [content bug](https://github.com/mdn/content/issues/new/choose), a [website bug](https://github.com/mdn/fred/issues/new/choose) or a [content/feature suggestion](https://github.com/mdn/mdn/issues/new/choose). 8 | 9 | ## Reporting a Vulnerability 10 | 11 | If you discover a potential security issue, please report it privately via . 12 | 13 | If you prefer not to use HackerOne, you can report it via . 14 | 15 | ## Bounty Program 16 | 17 | Vulnerabilities in MDN may qualify for Mozilla's Bug Bounty Program. Eligibility and reward amounts are described on . 18 | 19 | Please use the above channels even if you are not interested in a bounty reward. 20 | 21 | ## Responsible Disclosure 22 | 23 | Please do not publicly disclose details until Mozilla's security team and the MDN engineering team have verified and fixed the issue. 24 | 25 | We appreciate your efforts to keep MDN and its users safe. 26 | -------------------------------------------------------------------------------- /src/api/v2/recommendations/index.js: -------------------------------------------------------------------------------- 1 | import { ALL_RESULTS } from "../../../constants.js"; 2 | import { SCORE_TABLE, TEST_TOPIC_LINKS } from "../../../grader/charts.js"; 3 | import { SCHEMAS } from "../schemas.js"; 4 | 5 | /** 6 | * Register the API - default export 7 | * @param {import('fastify').FastifyInstance} fastify 8 | * @returns {Promise} 9 | */ 10 | export default async function (fastify) { 11 | fastify.get( 12 | "/recommendation_matrix", 13 | { schema: SCHEMAS.recommendationMatrix }, 14 | async (_request, _reply) => { 15 | const res = ALL_RESULTS.map((output) => { 16 | return { 17 | name: output.name, 18 | title: output.title, 19 | mdnLink: TEST_TOPIC_LINKS.get(output.name) || "", 20 | results: output.possibleResults.map((pr) => { 21 | const data = SCORE_TABLE.get(pr); 22 | return data 23 | ? { 24 | name: pr, 25 | scoreModifier: data.modifier, 26 | description: data.description, 27 | recommendation: data.recommendation, 28 | } 29 | : { 30 | name: pr, 31 | scoreModifier: 0, 32 | description: "", 33 | recommendation: "", 34 | }; 35 | }), 36 | }; 37 | }); 38 | return res; 39 | } 40 | ); 41 | } 42 | -------------------------------------------------------------------------------- /src/retriever/utils.js: -------------------------------------------------------------------------------- 1 | import { JSDOM } from "jsdom"; 2 | import { CONTENT_SECURITY_POLICY, REFERRER_POLICY } from "../headers.js"; 3 | 4 | /** 5 | * 6 | * @param {string} html 7 | * @param {string} _baseUrl 8 | * @returns {Map} 9 | */ 10 | export function parseHttpEquivHeaders(html, _baseUrl) { 11 | /** @type {Map} */ 12 | const httpEquivHeaders = new Map([[CONTENT_SECURITY_POLICY, []]]); 13 | 14 | try { 15 | const dom = JSDOM.fragment(html); 16 | const metas = [...dom.querySelectorAll("meta")]; 17 | 18 | for (const meta of metas) { 19 | if (meta.hasAttribute("http-equiv") && meta.hasAttribute("content")) { 20 | const httpEquiv = meta.getAttribute("http-equiv")?.toLowerCase().trim(); 21 | const content = meta.getAttribute("content"); 22 | if (content && httpEquiv === CONTENT_SECURITY_POLICY) { 23 | httpEquivHeaders.get(CONTENT_SECURITY_POLICY)?.push(content); 24 | } 25 | } else if ( 26 | // Technically not HTTP Equiv, but we're treating it that way 27 | meta.getAttribute("name")?.toLowerCase().trim() === "referrer" 28 | ) { 29 | const attr = meta.getAttribute("content"); 30 | if (attr) { 31 | httpEquivHeaders.set(REFERRER_POLICY, [attr]); 32 | } 33 | } 34 | } 35 | } catch (e) { 36 | console.error("Error parsing HTTP Equiv headers", e); 37 | } 38 | return httpEquivHeaders; 39 | } 40 | -------------------------------------------------------------------------------- /src/retrieve-tld-list.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { writeFile } from "fs/promises"; 3 | import path from "node:path"; 4 | import { fileURLToPath } from "node:url"; 5 | 6 | const TLD_LIST_URL = new URL( 7 | "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" 8 | ); 9 | 10 | const dirname = path.dirname(fileURLToPath(import.meta.url)); 11 | 12 | /** 13 | * Download the IANA-maintained public suffix list 14 | */ 15 | export async function retrieveAndStoreTldList() { 16 | let r; 17 | try { 18 | r = await axios.get(TLD_LIST_URL.href); 19 | } catch (error) { 20 | console.error("Error getting data:", error); 21 | return; 22 | } 23 | const data = cleanData(r.data); 24 | const filePath = path.join(dirname, "..", "conf", "tld-list.json"); 25 | try { 26 | await writeFile(filePath, data); 27 | console.log(`File written to ${filePath}`); 28 | } catch (error) { 29 | console.error("Error writing file:", error); 30 | return; 31 | } 32 | } 33 | 34 | /** 35 | * 36 | * @param {string} data 37 | * @returns {string} 38 | */ 39 | function cleanData(data) { 40 | const ret = data 41 | .replace(/#.*$/gm, "") 42 | .split("\n") 43 | .filter((line) => !line.startsWith("#")) 44 | .filter((line) => line.trim() !== "") 45 | .map((line) => line.trim().toLowerCase()); 46 | return JSON.stringify(ret); 47 | } 48 | 49 | // Execute when run directly 50 | if (import.meta.url === `file://${process.argv[1]}`) { 51 | retrieveAndStoreTldList().catch(console.error); 52 | } 53 | -------------------------------------------------------------------------------- /test/x-content-type-options.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { xContentTypeOptionsTest } from "../src/analyzer/tests/x-content-type-options.js"; 5 | 6 | describe("X-Content-Type-Options", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests(); 11 | }); 12 | 13 | it("checks for missing", function () { 14 | const result = xContentTypeOptionsTest(reqs); 15 | assert.equal(result.result, Expectation.XContentTypeOptionsNotImplemented); 16 | assert.isFalse(result.pass); 17 | }); 18 | 19 | it("checks header validity", function () { 20 | const values = ["whimsy", "nosniff, nosniff"]; 21 | assert.isNotNull(reqs.responses.auto); 22 | for (const value of values) { 23 | reqs.responses.auto.headers["x-content-type-options"] = value; 24 | const result = xContentTypeOptionsTest(reqs); 25 | assert.equal(result.result, Expectation.XContentTypeOptionsHeaderInvalid); 26 | assert.isFalse(result.pass); 27 | } 28 | }); 29 | 30 | it("checks for nosniff", function () { 31 | const values = ["nosniff", "nosniff "]; 32 | assert.isNotNull(reqs.responses.auto); 33 | for (const value of values) { 34 | reqs.responses.auto.headers["x-content-type-options"] = value; 35 | const result = xContentTypeOptionsTest(reqs); 36 | assert.equal(result.result, Expectation.XContentTypeOptionsNosniff); 37 | assert.isTrue(result.pass); 38 | } 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /migrations/003.do.scans.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS scans ( 2 | id SERIAL PRIMARY KEY, 3 | site_id INTEGER REFERENCES sites (id) NOT NULL, 4 | state VARCHAR NOT NULL, 5 | start_time TIMESTAMP NOT NULL, 6 | end_time TIMESTAMP NULL, 7 | algorithm_version SMALLINT NOT NULL DEFAULT 1, 8 | tests_failed SMALLINT NOT NULL DEFAULT 0, 9 | tests_passed SMALLINT NOT NULL DEFAULT 0, 10 | tests_quantity SMALLINT NOT NULL, 11 | grade VARCHAR(2) NULL, 12 | score SMALLINT NULL, 13 | likelihood_indicator VARCHAR NULL, 14 | error VARCHAR NULL, 15 | response_headers JSONB NULL, 16 | hidden BOOL NOT NULL DEFAULT FALSE, 17 | status_code SMALLINT NULL 18 | ); 19 | 20 | CREATE INDEX scans_state_idx ON scans (state); 21 | CREATE INDEX scans_start_time_idx ON scans (start_time); 22 | CREATE INDEX scans_end_time_idx ON scans (end_time); 23 | CREATE INDEX scans_algorithm_version_idx ON scans (algorithm_version); 24 | CREATE INDEX scans_grade_idx ON scans (grade); 25 | CREATE INDEX scans_score_idx ON scans (score); 26 | CREATE INDEX scans_hidden_idx ON scans (hidden); 27 | CREATE INDEX scans_site_id_finished_state_end_time_idx ON scans (site_id, state, end_time DESC) WHERE state = 'FINISHED'; 28 | -------------------------------------------------------------------------------- /src/analyzer/utils.js: -------------------------------------------------------------------------------- 1 | import { Expectation } from "../types.js"; 2 | 3 | /** 4 | * Return the new result if it's worse than the existing result, otherwise just the current result. 5 | * @param {Expectation} newResult - The new result to compare. 6 | * @param {Expectation | null} oldResult - The existing result to compare against. 7 | * @param {Expectation[]} order - An array defining the order of results from best to worst. 8 | * @returns {Expectation} - The worse of the two results. 9 | */ 10 | export function onlyIfWorse(newResult, oldResult, order) { 11 | if (!oldResult) { 12 | return newResult; 13 | } else if (order.indexOf(newResult) > order.indexOf(oldResult)) { 14 | return newResult; 15 | } else { 16 | return oldResult; 17 | } 18 | } 19 | 20 | /** 21 | * @param {import("../types.js").Response | null} response 22 | * @param {string} name 23 | * @returns {string[]} 24 | */ 25 | export function getHttpHeaders(response, name) { 26 | if (!response) { 27 | return []; 28 | } 29 | const axiosHeaders = response.headers; 30 | if (!axiosHeaders) { 31 | return []; 32 | } 33 | const lcName = name.toLowerCase(); 34 | const headers = Object.entries(axiosHeaders) 35 | .filter(([headerName, _value]) => { 36 | return headerName.toLowerCase() === lcName; 37 | }) 38 | .map(([_headerName, value]) => value) 39 | .flat(); 40 | return headers; 41 | } 42 | 43 | /** 44 | * @param {import("../types.js").Response | null} response 45 | * @param {string} name 46 | * @returns {string | null} 47 | */ 48 | export function getFirstHttpHeader(response, name) { 49 | if (!response) { 50 | return null; 51 | } 52 | return getHttpHeaders(response, name)[0] ?? null; 53 | } 54 | -------------------------------------------------------------------------------- /test/grader.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { 3 | getGradeForScore, 4 | getRecommendation, 5 | getScoreDescription, 6 | getScoreModifier, 7 | getTopicLink, 8 | } from "../src/grader/grader.js"; 9 | import { Expectation } from "../src/types.js"; 10 | 11 | describe("Grader", () => { 12 | it("gets the score description", function () { 13 | const res = getScoreDescription(Expectation.CspNotImplemented); 14 | assert.include(res, "Content Security Policy (CSP) header not implemented"); 15 | }); 16 | 17 | it("gets the score modifier", function () { 18 | const res = getScoreModifier(Expectation.CspNotImplemented); 19 | assert.equal(res, -25); 20 | }); 21 | 22 | it("gets the recommendation", function () { 23 | const res = getRecommendation(Expectation.CspNotImplemented); 24 | assert.include(res, "Implement one, see "); 25 | }); 26 | 27 | it("gets the topic link for a test name", function () { 28 | const res = getTopicLink("cookies"); 29 | assert.include( 30 | res, 31 | "/en-US/docs/Web/Security/Practical_implementation_guides/Cookies" 32 | ); 33 | }); 34 | 35 | it("gets the grade", function () { 36 | let res = getGradeForScore(100); 37 | assert.deepEqual(res, { 38 | score: 100, 39 | grade: "A+", 40 | }); 41 | 42 | res = getGradeForScore(0); 43 | assert.deepEqual(res, { 44 | score: 0, 45 | grade: "F", 46 | }); 47 | 48 | res = getGradeForScore(120); 49 | assert.deepEqual(res, { 50 | score: 120, 51 | grade: "A+", 52 | }); 53 | 54 | res = getGradeForScore(-10); 55 | assert.deepEqual(res, { 56 | score: 0, 57 | grade: "F", 58 | }); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /src/analyzer/hsts.js: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "node:path"; 3 | import { fileURLToPath } from "node:url"; 4 | import { Site } from "../site.js"; 5 | 6 | const dirname = path.dirname(fileURLToPath(import.meta.url)); 7 | 8 | /** 9 | * @type {import("../types.js").Hsts | null} 10 | */ 11 | let hstsMap = null; 12 | 13 | /** 14 | * @returns {import("../types.js").Hsts} 15 | */ 16 | export function hsts() { 17 | if (!hstsMap) { 18 | const filePath = path.join( 19 | dirname, 20 | "..", 21 | "..", 22 | "conf", 23 | "hsts-preload.json" 24 | ); 25 | hstsMap = new Map( 26 | Object.entries(JSON.parse(fs.readFileSync(filePath, "utf8"))) 27 | ); 28 | } 29 | return hstsMap; 30 | } 31 | 32 | /** 33 | * 34 | * @param {Site} site 35 | * @returns {import("../types.js").Hst | null} 36 | */ 37 | export function isHstsPreloaded(site) { 38 | const h = hsts(); 39 | const hostname = site.hostname; 40 | 41 | // Check if the hostname is in the HSTS list with the right mode 42 | const existing = h.get(hostname); 43 | if (existing && existing.mode === "force-https") { 44 | return existing; 45 | } 46 | 47 | // Either the hostname is in the list *or* the TLD is and includeSubDomains is true 48 | const hostParts = hostname.split("."); 49 | 50 | // If hostname is foo.bar.baz.mozilla.org, check bar.baz.mozilla.org, 51 | // baz.mozilla.org, mozilla.org, and.org 52 | for (hostParts.shift(); hostParts.length > 0; hostParts.shift()) { 53 | const domain = hostParts.join("."); 54 | const exist = h.get(domain); 55 | if (exist && exist.mode === "force-https" && exist.includeSubDomains) { 56 | return exist; 57 | } 58 | } 59 | return null; 60 | } 61 | -------------------------------------------------------------------------------- /src/api/v2/scan/index.js: -------------------------------------------------------------------------------- 1 | import { CONFIG } from "../../../config.js"; 2 | import { selectScanLatestScanByHost as selectScanLatestScanBySite } from "../../../database/repository.js"; 3 | import { Site } from "../../../site.js"; 4 | import { SCHEMAS } from "../schemas.js"; 5 | import { checkSitename, executeScan } from "../utils.js"; 6 | 7 | /** 8 | * @typedef {import("pg").Pool} Pool 9 | */ 10 | 11 | /** 12 | * Register the API - default export 13 | * @param {import('fastify').FastifyInstance} fastify 14 | * @returns {Promise} 15 | */ 16 | export default async function (fastify) { 17 | const pool = fastify.pg.pool; 18 | fastify.post("/scan", { schema: SCHEMAS.scan }, async (request, _reply) => { 19 | const query = /** @type {import("../../v2/schemas.js").ScanQuery} */ ( 20 | request.query 21 | ); 22 | 23 | const hostname = query.host.trim().toLowerCase(); 24 | let site = Site.fromSiteString(hostname); 25 | site = await checkSitename(site); 26 | return await scanOrReturnRecent(fastify, pool, site, CONFIG.api.cooldown); 27 | }); 28 | } 29 | 30 | /** 31 | * 32 | * @param {import("fastify").FastifyInstance} _fastify 33 | * @param {Pool} pool 34 | * @param {Site} site 35 | * @param {number} age 36 | * @returns {Promise} 37 | */ 38 | async function scanOrReturnRecent(_fastify, pool, site, age) { 39 | let scanRow = await selectScanLatestScanBySite(pool, site.asSiteKey(), age); 40 | if (!scanRow) { 41 | // do a rescan 42 | scanRow = await executeScan(pool, site); 43 | } 44 | 45 | scanRow.scanned_at = scanRow.start_time; 46 | const siteLink = `https://developer.mozilla.org/en-US/observatory/analyze?host=${encodeURIComponent(site.asSiteKey())}`; 47 | return { details_url: siteLink, ...scanRow }; 48 | } 49 | -------------------------------------------------------------------------------- /.github/settings.yml: -------------------------------------------------------------------------------- 1 | repository: 2 | # See https://github.com/apps/settings for all available settings. 3 | 4 | # The name of the repository. Changing this will rename the repository 5 | name: project-template 6 | 7 | # A short description of the repository that will show up on GitHub 8 | description: MDN Web Docs project template 9 | 10 | # A URL with more information about the repository 11 | homepage: https://github.com/mdn/project-template 12 | 13 | # The branch used by default for pull requests and when the repository is cloned/viewed. 14 | default_branch: main 15 | 16 | # This repository is a template that others can use to start a new repository. 17 | is_template: true 18 | 19 | branches: 20 | - name: main 21 | protection: 22 | # Required. Require at least one approving review on a pull request, before merging. Set to null to disable. 23 | required_pull_request_reviews: 24 | # The number of approvals required. (1-6) 25 | required_approving_review_count: 1 26 | # Dismiss approved reviews automatically when a new commit is pushed. 27 | dismiss_stale_reviews: true 28 | # Blocks merge until code owners have reviewed. 29 | require_code_owner_reviews: true 30 | 31 | collaborators: 32 | - username: Rumyra 33 | permission: admin 34 | 35 | - username: fiji-flo 36 | permission: admin 37 | 38 | labels: 39 | - name: bug 40 | color: D41130 41 | description: "Something that's wrong or not working as expected" 42 | - name: chore 43 | color: 258CD3 44 | description: "A routine task" 45 | - name: "good first issue" 46 | color: 48B71D 47 | description: "Great for newcomers to start contributing" 48 | - name: "help wanted" 49 | color: 2E7A10 50 | description: "Contributions welcome" 51 | 52 | teams: 53 | - name: core 54 | permission: admin 55 | -------------------------------------------------------------------------------- /src/analyzer/tests/x-content-type-options.js: -------------------------------------------------------------------------------- 1 | import { X_CONTENT_TYPE_OPTIONS } from "../../headers.js"; 2 | import { BaseOutput, Requests } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getFirstHttpHeader } from "../utils.js"; 5 | 6 | export class XContentTypeOptionsOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | static name = "x-content-type-options"; 10 | static title = "X-Content-Type-Options"; 11 | static possibleResults = [ 12 | Expectation.XContentTypeOptionsNosniff, 13 | Expectation.XContentTypeOptionsHeaderInvalid, 14 | Expectation.XContentTypeOptionsNotImplemented, 15 | ]; 16 | 17 | /** 18 | * 19 | * @param {Expectation} expectation 20 | */ 21 | constructor(expectation) { 22 | super(expectation); 23 | } 24 | } 25 | 26 | /** 27 | * 28 | * @param {Requests} requests 29 | * @param {Expectation} expectation 30 | * @returns {XContentTypeOptionsOutput} 31 | */ 32 | export function xContentTypeOptionsTest( 33 | requests, 34 | expectation = Expectation.XContentTypeOptionsNosniff 35 | ) { 36 | const output = new XContentTypeOptionsOutput(expectation); 37 | const resp = requests.responses.auto; 38 | 39 | if (!resp) { 40 | output.result = Expectation.XContentTypeOptionsNotImplemented; 41 | return output; 42 | } 43 | 44 | const header = getFirstHttpHeader(resp, X_CONTENT_TYPE_OPTIONS); 45 | 46 | if (header) { 47 | output.data = header.slice(0, 256); 48 | if (output.data.trim().toLowerCase() === "nosniff") { 49 | output.result = Expectation.XContentTypeOptionsNosniff; 50 | } else { 51 | output.result = Expectation.XContentTypeOptionsHeaderInvalid; 52 | } 53 | } else { 54 | output.result = Expectation.XContentTypeOptionsNotImplemented; 55 | } 56 | 57 | // Check to see if the test passed or failed 58 | output.pass = output.result === expectation; 59 | return output; 60 | } 61 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | import { CookiesOutput, cookiesTest } from "./analyzer/tests/cookies.js"; 2 | import { 3 | CorsOutput, 4 | crossOriginResourceSharingTest, 5 | } from "./analyzer/tests/cors.js"; 6 | import { 7 | CrossOriginResourcePolicyOutput, 8 | crossOriginResourcePolicyTest, 9 | } from "./analyzer/tests/cross-origin-resource-policy.js"; 10 | import { contentSecurityPolicyTest, CspOutput } from "./analyzer/tests/csp.js"; 11 | import { 12 | RedirectionOutput, 13 | redirectionTest, 14 | } from "./analyzer/tests/redirection.js"; 15 | import { 16 | ReferrerOutput, 17 | referrerPolicyTest, 18 | } from "./analyzer/tests/referrer-policy.js"; 19 | import { 20 | StrictTransportSecurityOutput, 21 | strictTransportSecurityTest, 22 | } from "./analyzer/tests/strict-transport-security.js"; 23 | import { 24 | SubresourceIntegrityOutput, 25 | subresourceIntegrityTest, 26 | } from "./analyzer/tests/subresource-integrity.js"; 27 | import { 28 | XContentTypeOptionsOutput, 29 | xContentTypeOptionsTest, 30 | } from "./analyzer/tests/x-content-type-options.js"; 31 | import { 32 | XFrameOptionsOutput, 33 | xFrameOptionsTest, 34 | } from "./analyzer/tests/x-frame-options.js"; 35 | 36 | export const ALL_TESTS = [ 37 | contentSecurityPolicyTest, 38 | cookiesTest, 39 | crossOriginResourceSharingTest, 40 | redirectionTest, 41 | referrerPolicyTest, 42 | strictTransportSecurityTest, 43 | subresourceIntegrityTest, 44 | xContentTypeOptionsTest, 45 | xFrameOptionsTest, 46 | crossOriginResourcePolicyTest, 47 | ]; 48 | 49 | export const ALL_RESULTS = [ 50 | CspOutput, 51 | CookiesOutput, 52 | CorsOutput, 53 | RedirectionOutput, 54 | ReferrerOutput, 55 | StrictTransportSecurityOutput, 56 | SubresourceIntegrityOutput, 57 | XContentTypeOptionsOutput, 58 | XFrameOptionsOutput, 59 | CrossOriginResourcePolicyOutput, 60 | ]; 61 | 62 | export const NUM_TESTS = ALL_TESTS.length; 63 | 64 | export const ALGORITHM_VERSION = 5; 65 | -------------------------------------------------------------------------------- /src/api/errors.js: -------------------------------------------------------------------------------- 1 | import { STATUS_CODES } from "./utils.js"; 2 | 3 | export class AppError extends Error { 4 | // @ts-ignore 5 | constructor(...args) { 6 | super(...args); 7 | this.name = "error-unknown"; 8 | this.statusCode = STATUS_CODES.internalServerError; 9 | } 10 | } 11 | 12 | export class SiteIsDownError extends AppError { 13 | constructor() { 14 | super("Site is down"); 15 | this.name = "site-down"; 16 | this.statusCode = STATUS_CODES.badRequest; 17 | } 18 | } 19 | 20 | export class NotFoundError extends AppError { 21 | constructor() { 22 | super("Resource Not Found"); 23 | this.name = "not-found"; 24 | this.statusCode = STATUS_CODES.notFound; 25 | } 26 | } 27 | export class ScanFailedError extends AppError { 28 | /** 29 | * @param {Error} e 30 | */ 31 | constructor(e) { 32 | super("Scan Failed"); 33 | this.name = "scan-failed"; 34 | this.statusCode = STATUS_CODES.internalServerError; 35 | this.message = e.message; 36 | } 37 | } 38 | export class InvalidHostNameIpError extends AppError { 39 | constructor() { 40 | super("Cannot scan IP addresses"); 41 | this.name = "invalid-hostname-ip"; 42 | this.statusCode = STATUS_CODES.unprocessableEntity; 43 | } 44 | } 45 | 46 | export class InvalidHostNameError extends AppError { 47 | constructor() { 48 | super(`Invalid hostname`); 49 | this.name = "invalid-hostname"; 50 | this.statusCode = STATUS_CODES.unprocessableEntity; 51 | } 52 | } 53 | 54 | export class InvalidHostNameLookupError extends AppError { 55 | /** 56 | * 57 | * @param {string} hostname 58 | */ 59 | constructor(hostname) { 60 | super(`${hostname} cannot be resolved`); 61 | this.name = "invalid-hostname-lookup"; 62 | this.statusCode = STATUS_CODES.unprocessableEntity; 63 | } 64 | } 65 | 66 | export class InvalidSiteError extends AppError { 67 | /** 68 | * @param {string} siteString 69 | * @param {string} reason 70 | */ 71 | constructor(siteString, reason) { 72 | super(`${siteString} is invalid: ${reason}`); 73 | this.name = "invalid-site"; 74 | this.statusCode = STATUS_CODES.unprocessableEntity; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/grader/grader.js: -------------------------------------------------------------------------------- 1 | import { Expectation } from "../types.js"; 2 | import { GRADE_CHART, SCORE_TABLE, TEST_TOPIC_LINKS } from "./charts.js"; 3 | 4 | /** 5 | * @typedef {Object} GradeAndScore 6 | * @property {number} score 7 | * @property {string} grade 8 | */ 9 | 10 | /** 11 | * 12 | * @param {number} score - raw score based on all of the tests 13 | * @returns {GradeAndScore} - normalized score and grade 14 | */ 15 | export function getGradeForScore(score) { 16 | score = Math.max(score, 0); 17 | 18 | // If score>100, just use the grade for 100, otherwise round down to the nearest multiple of 5 19 | const scoreMapKey = Math.min(score - (score % 5), 100); 20 | const grade = GRADE_CHART.get(scoreMapKey); 21 | 22 | if (!grade) { 23 | throw new Error(`Score of ${scoreMapKey} did not map to a grade`); 24 | } 25 | 26 | return { 27 | score, 28 | grade, 29 | }; 30 | } 31 | 32 | /** 33 | * @param {Expectation} expectation 34 | * @returns {string} 35 | */ 36 | export function getScoreDescription(expectation) { 37 | return SCORE_TABLE.get(expectation)?.description ?? ""; 38 | } 39 | 40 | /** 41 | * @param {Expectation} expectation 42 | * @returns {string} 43 | */ 44 | export function getRecommendation(expectation) { 45 | return SCORE_TABLE.get(expectation)?.recommendation ?? ""; 46 | } 47 | 48 | /** 49 | * @param {string} testName 50 | * @returns {string} 51 | */ 52 | export function getTopicLink(testName) { 53 | return TEST_TOPIC_LINKS.get(testName) ?? ""; 54 | } 55 | 56 | /** 57 | * @param {Expectation} expectation 58 | * @returns {number} 59 | */ 60 | export function getScoreModifier(expectation) { 61 | return SCORE_TABLE.get(expectation)?.modifier ?? 0; 62 | } 63 | 64 | // 65 | // Helper function to line up score table and expectations 66 | export function matchScoreTableAndExpectations() { 67 | for (const exp of Object.values(Expectation)) { 68 | if (!SCORE_TABLE.get(exp)) { 69 | console.log(`No entry in SCORE_TABLE for Expectation ${exp}`); 70 | } 71 | } 72 | const eset = new Set(Object.values(Expectation)); 73 | SCORE_TABLE.forEach((_, key) => { 74 | if (!eset.has(key)) { 75 | console.log(`Expectation '${key}' has no matching entry in SCORE_TABLE`); 76 | } 77 | }); 78 | } 79 | -------------------------------------------------------------------------------- /test/scanner.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { scan } from "../src/scanner/index.js"; 3 | import { Site } from "../src/site.js"; 4 | 5 | /** @typedef {import("../src/scanner/index.js").ScanResult} ScanResult */ 6 | 7 | describe("Scanner", () => { 8 | it("returns an error on an unknown host", async function () { 9 | const domain = 10 | Array(223) 11 | .fill(0) 12 | .map(() => String.fromCharCode(Math.random() * 26 + 97)) 13 | .join("") + ".net"; 14 | const site = Site.fromSiteString(domain); 15 | try { 16 | await scan(site); 17 | throw new Error("scan should throw"); 18 | } catch (e) { 19 | if (e instanceof Error) { 20 | assert.equal(e.message, "The site seems to be down."); 21 | } else { 22 | throw new Error("Unexpected error type"); 23 | } 24 | } 25 | }); 26 | 27 | it("returns expected results on observatory.mozilla.org", async function () { 28 | const domain = "observatory.mozilla.org"; 29 | const site = Site.fromSiteString(domain); 30 | const scanResult = await scan(site); 31 | 32 | assert.equal(scanResult.scan.algorithmVersion, 5); 33 | assert.equal(scanResult.scan.grade, "A+"); 34 | assert.equal(scanResult.scan.score, 110); 35 | assert.equal(scanResult.scan.testsFailed, 0); 36 | assert.equal(scanResult.scan.testsPassed, 10); 37 | assert.equal(scanResult.scan.testsQuantity, 10); 38 | assert.equal(scanResult.scan.statusCode, 200); 39 | assert.equal(scanResult.scan.responseHeaders["content-type"], "text/html"); 40 | }).timeout(5000); 41 | 42 | it("returns expected results on mozilla.org", async function () { 43 | const domain = "mozilla.org"; 44 | const site = Site.fromSiteString(domain); 45 | const scanResult = await scan(site); 46 | assert.equal(scanResult.scan.algorithmVersion, 5); 47 | assert.equal(scanResult.scan.grade, "B+"); 48 | assert.equal(scanResult.scan.score, 80); 49 | assert.equal(scanResult.scan.testsFailed, 1); 50 | assert.equal(scanResult.scan.testsPassed, 9); 51 | assert.equal(scanResult.scan.testsQuantity, 10); 52 | assert.equal(scanResult.scan.statusCode, 200); 53 | assert.equal( 54 | // @ts-ignore 55 | scanResult.scan.responseHeaders["content-type"], 56 | "text/html; charset=utf-8" 57 | ); 58 | }).timeout(5000); 59 | }); 60 | -------------------------------------------------------------------------------- /test/cors.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { crossOriginResourceSharingTest } from "../src/analyzer/tests/cors.js"; 5 | 6 | describe("Cross Origin Resource Sharing", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests(); 11 | }); 12 | 13 | it("checks for not implemented", function () { 14 | const result = crossOriginResourceSharingTest(reqs); 15 | assert.equal( 16 | result.result, 17 | Expectation.CrossOriginResourceSharingNotImplemented 18 | ); 19 | assert.isTrue(result.pass); 20 | }); 21 | 22 | it("checks for public access", function () { 23 | assert.isNotNull(reqs.responses.cors); 24 | reqs.responses.cors.headers["access-control-allow-origin"] = "*"; 25 | const result = crossOriginResourceSharingTest(reqs); 26 | assert.equal( 27 | result.result, 28 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess 29 | ); 30 | assert.equal(result.data, "*"); 31 | assert.isTrue(result.pass); 32 | }); 33 | 34 | it("checks for restricted access", function () { 35 | assert.isNotNull(reqs.responses.cors); 36 | reqs.responses.cors.request.headers["origin"] = 37 | "https://http-observatory.security.mozilla.org"; 38 | reqs.responses.cors.headers["access-control-allow-origin"] = 39 | "https://mozilla.org"; 40 | 41 | const result = crossOriginResourceSharingTest(reqs); 42 | assert.equal( 43 | result.result, 44 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess 45 | ); 46 | assert.isTrue(result.pass); 47 | }); 48 | 49 | it("checks for universal access", function () { 50 | assert.isNotNull(reqs.responses.cors); 51 | reqs.responses.cors.request.headers["origin"] = 52 | "https://http-observatory.security.mozilla.org"; 53 | reqs.responses.cors.headers["access-control-allow-origin"] = 54 | "https://http-observatory.security.mozilla.org"; 55 | reqs.responses.cors.headers["access-control-allow-credentials"] = "true"; 56 | 57 | const result = crossOriginResourceSharingTest(reqs); 58 | assert.equal( 59 | result.result, 60 | Expectation.CrossOriginResourceSharingImplementedWithUniversalAccess 61 | ); 62 | assert.isFalse(result.pass); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /src/retriever/url.js: -------------------------------------------------------------------------------- 1 | import { SiteIsDownError } from "../api/errors.js"; 2 | import { CONFIG } from "../config.js"; 3 | import { Site } from "../site.js"; 4 | import axios from "axios"; 5 | 6 | /** 7 | * Detects if a port supports TLS by making simple test requests 8 | * @param {Site} site 9 | * @returns {Promise} true if TLS is supported, false otherwise 10 | */ 11 | export async function detectTlsSupport(site) { 12 | const httpsUrl = `https://${site.hostname}:${site.port}${site.path ?? ""}`; 13 | const httpUrl = `http://${site.hostname}:${site.port}${site.path ?? ""}`; 14 | 15 | // Simple timeout and basic config for quick checks 16 | const config = { 17 | timeout: CONFIG.retriever.clientTimeout, 18 | maxRedirects: 0, 19 | validateStatus: (/** @type {number} */ status) => status < 500, 20 | }; 21 | 22 | // Run both requests concurrently 23 | const [httpsResult, httpResult] = await Promise.allSettled([ 24 | axios.head(httpsUrl, { 25 | ...config, 26 | httpsAgent: new (await import("https")).Agent({ 27 | rejectUnauthorized: false, // Accept self-signed certs for detection 28 | }), 29 | }), 30 | axios.head(httpUrl, config), 31 | ]); 32 | 33 | // Check if HTTPS succeeded 34 | if (httpsResult.status === "fulfilled") { 35 | return true; 36 | } 37 | 38 | // Check if HTTP succeeded 39 | if (httpResult.status === "fulfilled") { 40 | return false; 41 | } 42 | 43 | // Both failed 44 | throw new SiteIsDownError(); 45 | } 46 | 47 | /** 48 | * 49 | * @param {Site} site 50 | * @param {import("../types.js").ScanOptions} [options] 51 | */ 52 | export async function urls(site, options = {}) { 53 | return { 54 | http: await url(site, false, options), 55 | https: await url(site, true, options), 56 | }; 57 | } 58 | 59 | /** 60 | * 61 | * @param {Site} site 62 | * @param {boolean} [https] 63 | * @param {import("../types.js").ScanOptions} options 64 | * @returns 65 | */ 66 | async function url(site, https = true, options = {}) { 67 | let port = (https ? options.httpsPort : options.httpPort) ?? undefined; 68 | if (site.port !== undefined) { 69 | const isTlsSecured = await detectTlsSupport(site); 70 | if (isTlsSecured && https) { 71 | port = site.port; 72 | } 73 | if (!isTlsSecured && !https) { 74 | port = site.port; 75 | } 76 | } 77 | const portString = port === undefined ? "" : `:${port}`; 78 | const url = new URL( 79 | `${https ? "https" : "http"}://${site.hostname}${portString}${site.path ?? ""}` 80 | ); 81 | return url; 82 | } 83 | -------------------------------------------------------------------------------- /test/site-utils.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { Site } from "../src/site.js"; 3 | 4 | describe("Site", () => { 5 | it("parses site strings", function () { 6 | { 7 | const site = Site.fromSiteString("example.com"); 8 | assert.equal(site.hostname, "example.com"); 9 | assert.isUndefined(site.port); 10 | assert.equal(site.path, "/"); 11 | } 12 | { 13 | const site = Site.fromSiteString("EXAMPLE.COM"); 14 | assert.equal(site.hostname, "example.com"); 15 | assert.isUndefined(site.port); 16 | assert.equal(site.path, "/"); 17 | } 18 | { 19 | const site = Site.fromSiteString("example.com:8443"); 20 | assert.equal(site.hostname, "example.com"); 21 | assert.equal(site.port, 8443); 22 | assert.equal(site.path, "/"); 23 | } 24 | { 25 | const site = Site.fromSiteString("example.com/some/path"); 26 | assert.equal(site.hostname, "example.com"); 27 | assert.isUndefined(site.port); 28 | assert.equal(site.path, "/some/path"); 29 | } 30 | { 31 | const site = Site.fromSiteString("example.com:8443/some/path"); 32 | assert.equal(site.hostname, "example.com"); 33 | assert.equal(site.port, 8443); 34 | assert.equal(site.path, "/some/path"); 35 | } 36 | { 37 | const site = Site.fromSiteString("example.com/some/path?q=bla"); 38 | assert.equal(site.hostname, "example.com"); 39 | assert.isUndefined(site.port); 40 | assert.equal(site.path, "/some/path"); 41 | } 42 | { 43 | const site = Site.fromSiteString("example.com/some/path#hash"); 44 | assert.equal(site.hostname, "example.com"); 45 | assert.isUndefined(site.port); 46 | assert.equal(site.path, "/some/path"); 47 | } 48 | { 49 | const site = Site.fromSiteString("ëxämplë.côm"); 50 | assert.equal(site.hostname, "xn--xmpl-loa4bf.xn--cm-8ja"); 51 | assert.isUndefined(site.port); 52 | assert.equal(site.path, "/"); 53 | } 54 | }); 55 | 56 | it("throws errors on invalid site strings", function () { 57 | assert.throws(() => Site.fromSiteString("")); 58 | assert.throws(() => Site.fromSiteString(".")); 59 | assert.throws(() => Site.fromSiteString(".x")); 60 | assert.throws(() => Site.fromSiteString("x.")); 61 | assert.throws(() => 62 | Site.fromSiteString( 63 | "äöüäöüöäöüöäöüöäöüäöüöäöüöäöääüöäößßäöäöüßßドメインドメインドメインkljadlkjaslドメインドメインドメインsdjkfhskdjfhhドメインドメインäöüäドメインドメインドメインäöüöäüöドメインドメインドメインドメインドメインäööüäöüドメインドメインドメインドメインäöüäölドメインドメインドメインドメインドメインドメインドメインドメインäöüääöüöäドメインa.com" 64 | ) 65 | ); 66 | }); 67 | }); 68 | -------------------------------------------------------------------------------- /src/api/v2/analyze/index.js: -------------------------------------------------------------------------------- 1 | import { CONFIG } from "../../../config.js"; 2 | import { selectScanLatestScanByHost } from "../../../database/repository.js"; 3 | import { Site } from "../../../site.js"; 4 | import { SCHEMAS } from "../schemas.js"; 5 | import { 6 | checkSitename, 7 | executeScan, 8 | historyForSite, 9 | hydrateTests, 10 | testsForScan, 11 | } from "../utils.js"; 12 | 13 | /** 14 | * @typedef {import("pg").Pool} Pool 15 | */ 16 | 17 | /** 18 | * Register the API - default export 19 | * @param {import('fastify').FastifyInstance} fastify 20 | * @returns {Promise} 21 | */ 22 | export default async function (fastify) { 23 | const pool = fastify.pg.pool; 24 | fastify.get( 25 | "/analyze", 26 | { schema: SCHEMAS.analyzeGet }, 27 | async (request, _reply) => { 28 | const query = 29 | /** @type {import("../../v2/schemas.js").AnalyzeReqQuery} */ ( 30 | request.query 31 | ); 32 | const hostname = query.host.trim().toLowerCase(); 33 | let site = Site.fromSiteString(hostname); 34 | site = await checkSitename(site); 35 | return await scanOrReturnRecent( 36 | fastify, 37 | pool, 38 | site, 39 | CONFIG.api.cacheTimeForGet 40 | ); 41 | } 42 | ); 43 | 44 | fastify.post( 45 | "/analyze", 46 | { schema: SCHEMAS.analyzePost }, 47 | async (request, _reply) => { 48 | const query = 49 | /** @type {import("../../v2/schemas.js").AnalyzeReqQuery} */ ( 50 | request.query 51 | ); 52 | const hostname = query.host.trim().toLowerCase(); 53 | let site = Site.fromSiteString(hostname); 54 | site = await checkSitename(site); 55 | return await scanOrReturnRecent(fastify, pool, site, CONFIG.api.cooldown); 56 | } 57 | ); 58 | } 59 | 60 | /** 61 | * 62 | * @param {import("fastify").FastifyInstance} _fastify 63 | * @param {Pool} pool 64 | * @param {import("../../../site.js").Site} site 65 | * @param {number} age 66 | * @returns {Promise} 67 | */ 68 | async function scanOrReturnRecent(_fastify, pool, site, age) { 69 | let scanRow = await selectScanLatestScanByHost(pool, site.asSiteKey(), age); 70 | if (!scanRow) { 71 | // do a rescan 72 | scanRow = await executeScan(pool, site); 73 | } 74 | const scanId = scanRow.id; 75 | const siteId = scanRow.site_id; 76 | 77 | const [rawTests, history] = await Promise.all([ 78 | testsForScan(pool, scanId), 79 | historyForSite(pool, siteId), 80 | ]); 81 | const tests = hydrateTests(rawTests); 82 | scanRow.scanned_at = scanRow.start_time; 83 | 84 | return { 85 | scan: scanRow, 86 | tests, 87 | history, 88 | }; 89 | } 90 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | # No GITHUB_TOKEN permissions, as we don't use it. 10 | permissions: {} 11 | 12 | jobs: 13 | test: 14 | runs-on: ubuntu-latest 15 | 16 | services: 17 | postgres: 18 | image: postgres:16 19 | env: 20 | POSTGRES_USER: observatory 21 | POSTGRES_PASSWORD: observatory 22 | POSTGRES_DB: observatory 23 | ports: 24 | - 5432:5432 25 | options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 26 | 27 | steps: 28 | - name: Checkout 29 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 30 | with: 31 | persist-credentials: false 32 | 33 | - name: Setup Node 34 | uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 35 | with: 36 | node-version-file: .nvmrc 37 | package-manager-cache: false 38 | 39 | - name: Install 40 | run: npm ci 41 | 42 | - name: Run tests 43 | run: npm test 44 | env: 45 | PGDATABASE: observatory 46 | PGHOST: localhost 47 | PGUSER: observatory 48 | PGPASSWORD: observatory 49 | 50 | build: 51 | runs-on: ubuntu-latest 52 | 53 | steps: 54 | - name: Checkout 55 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 56 | with: 57 | persist-credentials: false 58 | 59 | - name: Setup Docker 60 | uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 61 | 62 | - name: Build Docker image 63 | uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 64 | with: 65 | context: . 66 | build-args: | 67 | GIT_SHA=${{ github.sha }} 68 | push: false 69 | cache-from: type=gha 70 | cache-to: type=gha,mode=max 71 | 72 | lint: 73 | runs-on: ubuntu-latest 74 | 75 | steps: 76 | - name: Checkout 77 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 78 | with: 79 | persist-credentials: false 80 | 81 | - name: Setup Node 82 | uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 83 | with: 84 | node-version-file: ".nvmrc" 85 | package-manager-cache: false 86 | 87 | - name: Install 88 | run: npm ci 89 | 90 | - name: Run prettier 91 | run: npx prettier --check . 92 | 93 | - name: Run tsc 94 | run: npm run tsc 95 | -------------------------------------------------------------------------------- /src/analyzer/tests/x-frame-options.js: -------------------------------------------------------------------------------- 1 | import { X_FRAME_OPTIONS } from "../../headers.js"; 2 | import { BaseOutput, Requests } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getFirstHttpHeader } from "../utils.js"; 5 | import { contentSecurityPolicyTest } from "./csp.js"; 6 | 7 | export class XFrameOptionsOutput extends BaseOutput { 8 | /** @type {string | null} */ 9 | data = null; 10 | static name = "x-frame-options"; 11 | static title = "X-Frame-Options"; 12 | static possibleResults = [ 13 | Expectation.XFrameOptionsImplementedViaCsp, 14 | Expectation.XFrameOptionsSameoriginOrDeny, 15 | Expectation.XFrameOptionsAllowFromOrigin, 16 | Expectation.XFrameOptionsNotImplemented, 17 | Expectation.XFrameOptionsHeaderInvalid, 18 | ]; 19 | 20 | /** 21 | * 22 | * @param {Expectation} expectation 23 | */ 24 | constructor(expectation) { 25 | super(expectation); 26 | } 27 | } 28 | 29 | /** 30 | * 31 | * @param {Requests} requests 32 | * @param {Expectation} expectation 33 | * @returns {XFrameOptionsOutput} 34 | */ 35 | export function xFrameOptionsTest( 36 | requests, 37 | expectation = Expectation.XFrameOptionsSameoriginOrDeny 38 | ) { 39 | const output = new XFrameOptionsOutput(expectation); 40 | const resp = requests.responses.auto; 41 | 42 | if (!resp) { 43 | output.result = Expectation.XFrameOptionsNotImplemented; 44 | return output; 45 | } 46 | 47 | const header = getFirstHttpHeader(resp, X_FRAME_OPTIONS); 48 | 49 | if (header) { 50 | output.data = header.slice(0, 1024); 51 | const xfo = output.data.trim().toLowerCase(); 52 | if (["deny", "sameorigin"].includes(xfo)) { 53 | output.result = Expectation.XFrameOptionsSameoriginOrDeny; 54 | } else if (xfo.startsWith("allow-from")) { 55 | output.result = Expectation.XFrameOptionsAllowFromOrigin; 56 | } else { 57 | output.result = Expectation.XFrameOptionsHeaderInvalid; 58 | } 59 | } else { 60 | output.result = Expectation.XFrameOptionsNotImplemented; 61 | } 62 | 63 | // Check to see if frame-ancestors is implemented in CSP; if it is, then it isn't needed 64 | const csp = contentSecurityPolicyTest(requests); 65 | if (csp.data && csp.data["frame-ancestors"]) { 66 | output.result = Expectation.XFrameOptionsImplementedViaCsp; 67 | } 68 | 69 | // Check to see if the test passed or failed 70 | if ( 71 | [ 72 | Expectation.XFrameOptionsAllowFromOrigin, 73 | Expectation.XFrameOptionsSameoriginOrDeny, 74 | Expectation.XFrameOptionsImplementedViaCsp, 75 | expectation, 76 | ].includes(output.result) 77 | ) { 78 | output.pass = true; 79 | } 80 | return output; 81 | } 82 | -------------------------------------------------------------------------------- /test/analyzer-utils.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { AxiosHeaders } from "axios"; 3 | import { getFirstHttpHeader, getHttpHeaders } from "../src/analyzer/utils.js"; 4 | 5 | function emptyResponse() { 6 | return { 7 | headers: new AxiosHeaders("Content-Type: text/html"), 8 | request: { 9 | headers: new AxiosHeaders(), 10 | }, 11 | status: 200, 12 | statusText: "OK", 13 | verified: true, 14 | data: "", 15 | config: { 16 | headers: new AxiosHeaders(), 17 | }, 18 | }; 19 | } 20 | 21 | describe("getHttpHeaders", () => { 22 | it("gets all http headers for a header name", function () { 23 | const response = emptyResponse(); 24 | let headers = getHttpHeaders(response, "content-type"); 25 | assert.isArray(headers); 26 | assert.lengthOf(headers, 1); 27 | assert.equal(headers[0], "text/html"); 28 | headers = getHttpHeaders(response, "Content-Type"); 29 | assert.isArray(headers); 30 | assert.lengthOf(headers, 1); 31 | assert.equal(headers[0], "text/html"); 32 | headers = getHttpHeaders(response, "Non-Existing"); 33 | assert.isArray(headers); 34 | assert.lengthOf(headers, 0); 35 | }); 36 | 37 | it("gets headers correctly when set multiple times", function () { 38 | const response = emptyResponse(); 39 | response.headers.set("X-Test", "hello"); 40 | let headers = getHttpHeaders(response, "x-test"); 41 | assert.isArray(headers); 42 | assert.lengthOf(headers, 1); 43 | response.headers.set("X-Test", ["hello", "world", "1234"]); 44 | headers = getHttpHeaders(response, "x-test"); 45 | assert.isArray(headers); 46 | assert.lengthOf(headers, 3); 47 | assert.equal(headers[0], "hello"); 48 | assert.equal(headers[1], "world"); 49 | assert.equal(headers[2], "1234"); 50 | }); 51 | 52 | it("returns an empty array if the passed in value is `null`", function () { 53 | const headers = getHttpHeaders(null, "content-type"); 54 | assert.isArray(headers); 55 | assert.lengthOf(headers, 0); 56 | }); 57 | }); 58 | 59 | describe("getFirstHttpHeader", () => { 60 | it("gets the first header", function () { 61 | const response = emptyResponse(); 62 | const header = getFirstHttpHeader(response, "content-Type"); 63 | assert.isNotNull(header); 64 | assert.isString(header); 65 | assert.equal(header, "text/html"); 66 | }); 67 | 68 | it("gets the first header on multiple values", function () { 69 | const response = emptyResponse(); 70 | response.headers.set("X-test", ["hello", "world", "1234"]); 71 | const header = getFirstHttpHeader(response, "x-test"); 72 | assert.isNotNull(header); 73 | assert.isString(header); 74 | assert.equal(header, "hello"); 75 | }); 76 | }); 77 | -------------------------------------------------------------------------------- /test/cross-origin-resource-policy.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { crossOriginResourcePolicyTest } from "../src/analyzer/tests/cross-origin-resource-policy.js"; 5 | 6 | describe("Cross Origin Resource Policy", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests(); 11 | }); 12 | 13 | it("checks for missing", function () { 14 | const result = crossOriginResourcePolicyTest(reqs); 15 | assert.equal( 16 | result.result, 17 | Expectation.CrossOriginResourcePolicyNotImplemented 18 | ); 19 | assert.isTrue(result.pass); 20 | }); 21 | 22 | it("checks header validity", function () { 23 | const values = ["whimsy"]; 24 | assert.isNotNull(reqs.responses.auto); 25 | for (const value of values) { 26 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 27 | const result = crossOriginResourcePolicyTest(reqs); 28 | assert.equal( 29 | result.result, 30 | Expectation.CrossOriginResourcePolicyHeaderInvalid 31 | ); 32 | assert.isFalse(result.pass); 33 | } 34 | }); 35 | 36 | it("checks for same-site", function () { 37 | const values = ["same-site"]; 38 | assert.isNotNull(reqs.responses.auto); 39 | for (const value of values) { 40 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 41 | const result = crossOriginResourcePolicyTest(reqs); 42 | assert.equal( 43 | result.result, 44 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite 45 | ); 46 | assert.isTrue(result.pass); 47 | } 48 | }); 49 | it("checks for same-origin", function () { 50 | const values = ["same-origin"]; 51 | assert.isNotNull(reqs.responses.auto); 52 | for (const value of values) { 53 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 54 | const result = crossOriginResourcePolicyTest(reqs); 55 | assert.equal( 56 | result.result, 57 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin 58 | ); 59 | assert.isTrue(result.pass); 60 | } 61 | }); 62 | it("checks for cross-origin", function () { 63 | const values = ["cross-origin"]; 64 | assert.isNotNull(reqs.responses.auto); 65 | for (const value of values) { 66 | reqs.responses.auto.headers["cross-origin-resource-policy"] = value; 67 | const result = crossOriginResourcePolicyTest(reqs); 68 | assert.equal( 69 | result.result, 70 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin 71 | ); 72 | assert.isTrue(result.pass); 73 | } 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@mdn/mdn-http-observatory", 3 | "version": "1.5.1", 4 | "author": "Mozilla Developer Network", 5 | "description": "The MDN HTTP Observatory is a set of tools to analyze your website and inform you if you are utilizing the many available methods to secure it.", 6 | "main": "src/index.js", 7 | "packageManager": "npm@11.6.2", 8 | "engines": { 9 | "node": ">=24" 10 | }, 11 | "scripts": { 12 | "start": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem node src/api/index.js", 13 | "dev": "NODE_EXTRA_CA_CERTS=node_modules/node_extra_ca_certs_mozilla_bundle/ca_bundle/ca_intermediate_root_bundle.pem nodemon src/api/index.js", 14 | "test": "CONFIG_FILE=conf/config-test.json mocha", 15 | "tsc": "tsc -p jsconfig.json", 16 | "updateHsts": "node src/retrieve-hsts.js", 17 | "updateTldList": "node src/retrieve-tld-list.js", 18 | "refreshMaterializedViews": "node src/maintenance/index.js", 19 | "maintenance": "node src/maintenance/index.js", 20 | "migrate": "node -e 'import(\"./src/database/migrate.js\").then( m => m.migrateDatabase() )'", 21 | "postinstall": "npm run updateHsts && npm run updateTldList" 22 | }, 23 | "bin": { 24 | "mdn-http-observatory-scan": "bin/wrapper.js" 25 | }, 26 | "type": "module", 27 | "license": "MPL-2.0", 28 | "devDependencies": { 29 | "@faker-js/faker": "^10.0.0", 30 | "@supercharge/promise-pool": "^3.2.0", 31 | "@types/chai": "^5.2.2", 32 | "@types/convict": "^6.1.6", 33 | "@types/ip": "^1.1.3", 34 | "@types/jsdom": "^27.0.0", 35 | "@types/mocha": "^10.0.10", 36 | "@types/pg-format": "^1.0.5", 37 | "@types/tough-cookie": "^4.0.5", 38 | "chai": "^6.2.0", 39 | "json-schema-to-jsdoc": "^1.1.1", 40 | "mocha": "^11.7.1", 41 | "nodemon": "^3.1.10", 42 | "prettier-eslint": "^16.4.2", 43 | "typescript": "^5.7.2" 44 | }, 45 | "dependencies": { 46 | "@fastify/cors": "^11.0.1", 47 | "@fastify/helmet": "^13.0.0", 48 | "@fastify/postgres": "^6.0.1", 49 | "@fastify/static": "^8.2.0", 50 | "@sentry/node": "^10.1.0", 51 | "axios": "^1.10.0", 52 | "axios-cookiejar-support": "^6.0.2", 53 | "change-case": "^5.4.4", 54 | "commander": "^14.0.0", 55 | "convict": "^6.2.4", 56 | "dayjs": "^1.11.13", 57 | "fastify": "^5.4.0", 58 | "fastify-simple-form": "^3.0.0", 59 | "http-cookie-agent": "^7.0.1", 60 | "ip": "^2.0.1", 61 | "jsdom": "^27.0.0", 62 | "node_extra_ca_certs_mozilla_bundle": "^1.0.7", 63 | "pg": "^8.16.2", 64 | "pg-format": "^1.0.4", 65 | "pg-native": "^3.5.2", 66 | "pg-pool": "^3.10.1", 67 | "postgrator": "^8.0.0", 68 | "postgrator-cli": "^9.0.0", 69 | "tldts": "^7.0.12", 70 | "tough-cookie": "^5.0.0" 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/retrieve-hsts.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { writeFile } from "fs/promises"; 3 | import path from "node:path"; 4 | import { fileURLToPath } from "node:url"; 5 | 6 | const HSTS_URL = new URL( 7 | "https://raw.githubusercontent.com/chromium/chromium/main/net/http/transport_security_state_static.json" 8 | ); 9 | 10 | const SCANNER_PINNED_DOMAINS = [ 11 | "accounts.firefox.com", 12 | "addons.mozilla.org", 13 | "aus4.mozilla.org", 14 | "aus5.mozilla.org", 15 | "cdn.mozilla.org", 16 | "services.mozilla.com", 17 | ]; 18 | 19 | const dirname = path.dirname(fileURLToPath(import.meta.url)); 20 | 21 | /** 22 | * 23 | * @typedef {Object} RawData 24 | * @property {RawEntry[]} entries 25 | * @typedef {Object} RawEntry 26 | * @property {string} name 27 | * @property {string} policy 28 | * @property {string} mode 29 | * @property {string} [include_subdomains] 30 | * @property {string} [include_subdomains_for_pinning] 31 | * @typedef {Object} HstsEntry 32 | * @property {boolean} includeSubDomains 33 | * @property {boolean} includeSubDomainsForPinning 34 | * @property {string} mode 35 | * @property {boolean} pinned 36 | * @typedef {{ [key: string]: HstsEntry }} HstsMap 37 | */ 38 | 39 | /** 40 | * Download the Google HSTS preload list 41 | * @returns 42 | */ 43 | export async function retrieveAndStoreHsts() { 44 | let r; 45 | try { 46 | r = await axios.get(HSTS_URL.href); 47 | } catch (error) { 48 | console.error("Error getting data:", error); 49 | return; 50 | } 51 | const data = removeJsonComments(r.data); 52 | /** @type RawData */ 53 | const rawData = JSON.parse(data); 54 | 55 | const hstsMap = rawData.entries.reduce((acc, entry) => { 56 | const domain = entry.name.trim().toLowerCase(); 57 | acc[domain] = { 58 | includeSubDomains: !!entry.include_subdomains, 59 | includeSubDomainsForPinning: 60 | !!entry.include_subdomains || !!entry.include_subdomains_for_pinning, 61 | mode: entry.mode, 62 | // Add in the manually pinned domains 63 | pinned: SCANNER_PINNED_DOMAINS.includes(domain), 64 | }; 65 | return acc; 66 | }, /** @type {HstsMap} */ ({})); 67 | 68 | const filePath = path.join(dirname, "..", "conf", "hsts-preload.json"); 69 | try { 70 | await writeFile(filePath, JSON.stringify(hstsMap, null, 2)); 71 | console.log(`File written to ${filePath}`); 72 | } catch (error) { 73 | console.error("Error writing file:", error); 74 | return; 75 | } 76 | } 77 | 78 | /** 79 | * 80 | * @param {string} jsonString 81 | * @returns {string} 82 | */ 83 | function removeJsonComments(jsonString) { 84 | return jsonString.replace(/\/\/.*$/gm, ""); 85 | } 86 | 87 | // Execute when run directly 88 | if (import.meta.url === `file://${process.argv[1]}`) { 89 | retrieveAndStoreHsts().catch(console.error); 90 | } 91 | -------------------------------------------------------------------------------- /src/analyzer/tests/cors.js: -------------------------------------------------------------------------------- 1 | import { 2 | ACCESS_CONTROL_ALLOW_CREDENTIALS, 3 | ACCESS_CONTROL_ALLOW_ORIGIN, 4 | ORIGIN, 5 | } from "../../headers.js"; 6 | import { BaseOutput, Requests } from "../../types.js"; 7 | import { Expectation } from "../../types.js"; 8 | import { getFirstHttpHeader } from "../utils.js"; 9 | 10 | export class CorsOutput extends BaseOutput { 11 | /** @type {string | null} */ 12 | data = null; 13 | static name = "cross-origin-resource-sharing"; 14 | static title = "Cross Origin Resource Sharing (CORS)"; 15 | static possibleResults = [ 16 | Expectation.CrossOriginResourceSharingNotImplemented, 17 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess, 18 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess, 19 | Expectation.CrossOriginResourceSharingImplementedWithUniversalAccess, 20 | ]; 21 | 22 | /** 23 | * 24 | * @param {Expectation} expectation 25 | */ 26 | constructor(expectation) { 27 | super(expectation); 28 | } 29 | } 30 | 31 | /** 32 | * 33 | * @param {Requests} requests 34 | * @param {Expectation} expectation 35 | * @returns {CorsOutput} 36 | */ 37 | export function crossOriginResourceSharingTest( 38 | requests, 39 | expectation = Expectation.CrossOriginResourceSharingNotImplemented 40 | ) { 41 | const output = new CorsOutput(expectation); 42 | output.result = Expectation.CrossOriginResourceSharingNotImplemented; 43 | const accessControlAllowOrigin = requests.responses.cors; 44 | 45 | const acaoHeader = getFirstHttpHeader( 46 | accessControlAllowOrigin, 47 | ACCESS_CONTROL_ALLOW_ORIGIN 48 | ); 49 | const originHeader = getFirstHttpHeader( 50 | accessControlAllowOrigin?.request, 51 | ORIGIN 52 | ); 53 | const credentialsHeader = getFirstHttpHeader( 54 | accessControlAllowOrigin, 55 | ACCESS_CONTROL_ALLOW_CREDENTIALS 56 | ); 57 | 58 | if (accessControlAllowOrigin && acaoHeader) { 59 | output.data = acaoHeader.slice(0, 256).trim().toLowerCase(); 60 | if (output.data === "*") { 61 | output.result = 62 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess; 63 | } else if ( 64 | originHeader && 65 | acaoHeader && 66 | originHeader === acaoHeader && 67 | credentialsHeader && 68 | credentialsHeader.toLowerCase().trim() === "true" 69 | ) { 70 | output.result = 71 | Expectation.CrossOriginResourceSharingImplementedWithUniversalAccess; 72 | } else { 73 | output.result = 74 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess; 75 | } 76 | } 77 | 78 | // Check to see if the test passed or failed 79 | if ( 80 | [ 81 | Expectation.CrossOriginResourceSharingImplementedWithPublicAccess, 82 | Expectation.CrossOriginResourceSharingImplementedWithRestrictedAccess, 83 | expectation, 84 | ].includes(output.result) 85 | ) { 86 | output.pass = true; 87 | } 88 | return output; 89 | } 90 | -------------------------------------------------------------------------------- /test/helpers/db.js: -------------------------------------------------------------------------------- 1 | import { faker } from "@faker-js/faker"; 2 | import { ensureSite, ScanState } from "../../src/database/repository.js"; 3 | import { GRADE_CHART } from "../../src/grader/charts.js"; 4 | import { Expectation } from "../../src/types.js"; 5 | import { ALGORITHM_VERSION } from "../../src/constants.js"; 6 | 7 | /** 8 | * @typedef {import("pg").Pool} Pool 9 | */ 10 | 11 | /** 12 | * 13 | * @param {Pool} pool 14 | * @param {any} site 15 | */ 16 | export async function insertSite(pool, site) { 17 | await pool.query( 18 | `INSERT INTO sites (domain, creation_time, public_headers, private_headers, cookies) VALUES ($1, NOW(), $2, $3, $4)`, 19 | [site.domain, site.public_headers, site.private_headers, site.cookies] 20 | ); 21 | } 22 | 23 | /** 24 | * 25 | * @param {Pool} pool 26 | */ 27 | export async function insertSeeds(pool) { 28 | // create a bunch of sites 29 | const siteIds = await Promise.all( 30 | [...Array(10).keys()].map((i) => { 31 | if (i === 0) { 32 | return ensureSite(pool, "www.mozilla.org"); 33 | } else { 34 | return ensureSite(pool, faker.internet.domainName()); 35 | } 36 | }) 37 | ); 38 | // make some random scans for those 39 | const scanIds = ( 40 | await Promise.all( 41 | [...Array(20).keys()].map(async (i) => { 42 | let score = Math.floor(Math.random() * 120); 43 | score -= score % 5; 44 | const grade = GRADE_CHART.get(Math.min(score, 100)); 45 | const siteId = siteIds[i % siteIds.length]; 46 | return pool.query( 47 | `INSERT INTO scans (site_id, state, start_time, end_time, grade, score, tests_quantity, algorithm_version, status_code) 48 | VALUES ($1, 49 | $2, 50 | NOW() - INTERVAL '${(i + 1) * 2000} seconds', 51 | NOW() - INTERVAL '${(i + 1) * 2000} seconds', 52 | $3, 53 | $4, 54 | 9, 55 | $5, 56 | 200) RETURNING id`, 57 | [siteId, ScanState.FINISHED, grade, score, ALGORITHM_VERSION] 58 | ); 59 | }) 60 | ) 61 | ).map((r) => r.rows[0].id); 62 | 63 | await Promise.all( 64 | [...Array(100).keys()].map((i) => { 65 | const siteId = siteIds[i % siteIds.length]; 66 | const scanId = scanIds[i % scanIds.length]; 67 | const expectation = 68 | Object.values(Expectation)[ 69 | Math.floor(Math.random() * Object.values(Expectation).length) 70 | ]; 71 | return pool.query( 72 | `INSERT INTO tests (site_id, scan_id, name, expectation, result, score_modifier, pass, output) 73 | VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, 74 | [ 75 | siteId, 76 | scanId, 77 | ["redirection", "cookies", "referrer-policy"][i % 3], 78 | expectation, 79 | expectation, 80 | [-20, -10, 0, 0, 0, 5, 10][i % 7], 81 | Math.random() > 0.5, 82 | { data: "some data" }, 83 | ] 84 | ); 85 | }) 86 | ); 87 | } 88 | -------------------------------------------------------------------------------- /test/x-frame-options.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { xFrameOptionsTest } from "../src/analyzer/tests/x-frame-options.js"; 4 | import { Expectation } from "../src/types.js"; 5 | import { getScoreModifier } from "../src/grader/grader.js"; 6 | 7 | describe("X-Frame-Options", () => { 8 | /** @type {import("../src/types.js").Requests} */ 9 | let reqs; 10 | beforeEach(() => { 11 | reqs = emptyRequests("test_content_sri_no_scripts.html"); 12 | }); 13 | 14 | it("checks for missing", function () { 15 | const result = xFrameOptionsTest(reqs); 16 | assert.equal(result.result, Expectation.XFrameOptionsNotImplemented); 17 | assert.isFalse(result.pass); 18 | }); 19 | 20 | it("checks validity", function () { 21 | assert.isNotNull(reqs.responses.auto); 22 | reqs.responses.auto.headers["x-frame-options"] = "whimsy"; 23 | let result = xFrameOptionsTest(reqs); 24 | assert.equal(result.result, Expectation.XFrameOptionsHeaderInvalid); 25 | assert.isFalse(result.pass); 26 | 27 | // common to see this header sent multiple times 28 | reqs.responses.auto.headers["x-frame-options"] = "SAMEORIGIN, SAMEORIGIN"; 29 | result = xFrameOptionsTest(reqs); 30 | assert.equal(result.result, Expectation.XFrameOptionsHeaderInvalid); 31 | assert.isFalse(result.pass); 32 | }); 33 | 34 | it("checks allow from origin", function () { 35 | assert.isNotNull(reqs.responses.auto); 36 | reqs.responses.auto.headers["x-frame-options"] = 37 | "ALLOW-FROM https://mozilla.org"; 38 | const result = xFrameOptionsTest(reqs); 39 | assert.equal(result.result, Expectation.XFrameOptionsAllowFromOrigin); 40 | assert.isTrue(result.pass); 41 | }); 42 | 43 | it("checks deny", function () { 44 | assert.isNotNull(reqs.responses.auto); 45 | reqs.responses.auto.headers["x-frame-options"] = "DENY"; 46 | let result = xFrameOptionsTest(reqs); 47 | assert.equal(result.result, Expectation.XFrameOptionsSameoriginOrDeny); 48 | assert.equal(getScoreModifier(result.result || ""), 5); 49 | assert.isTrue(result.pass); 50 | 51 | reqs.responses.auto.headers["x-frame-options"] = "SAMEORIGIN"; 52 | result = xFrameOptionsTest(reqs); 53 | assert.equal(result.result, Expectation.XFrameOptionsSameoriginOrDeny); 54 | assert.equal(getScoreModifier(result.result || ""), 5); 55 | assert.isTrue(result.pass); 56 | }); 57 | 58 | it("checks implemented via CSP", function () { 59 | assert.isNotNull(reqs.responses.auto); 60 | reqs.responses.auto.headers["x-frame-options"] = "DENY"; 61 | reqs.responses.auto.headers["content-security-policy"] = 62 | "frame-ancestors https://mozilla.org"; 63 | const result = xFrameOptionsTest(reqs); 64 | assert.equal(result.result, Expectation.XFrameOptionsImplementedViaCsp); 65 | assert.isTrue(result.pass); 66 | }); 67 | 68 | it("does not obey x-frame-options in meta equiv tags", function () { 69 | reqs = emptyRequests("test_parse_http_equiv_headers_x_frame_options.html"); 70 | const result = xFrameOptionsTest(reqs); 71 | assert.equal(result.result, Expectation.XFrameOptionsNotImplemented); 72 | assert.isFalse(result.pass); 73 | }); 74 | }); 75 | -------------------------------------------------------------------------------- /test/helpers.js: -------------------------------------------------------------------------------- 1 | import fs from "node:fs"; 2 | 3 | import { AxiosHeaders } from "axios"; 4 | 5 | import { Requests } from "../src/types.js"; 6 | import { Session } from "../src/retriever/session.js"; 7 | import path from "node:path"; 8 | import { parseHttpEquivHeaders } from "../src/retriever/utils.js"; 9 | import { Site } from "../src/site.js"; 10 | 11 | /** 12 | * 13 | * @param {import("../src/types.js").Response | null} response 14 | * @param {string} header 15 | * @param {string} value 16 | */ 17 | export function setHeader(response, header, value) { 18 | if (typeof response?.headers.set === "function") { 19 | response.headers.set(header, value); 20 | } 21 | } 22 | 23 | /** 24 | * 25 | * @param {string | null} [httpEquivFile] 26 | * @returns {Requests} 27 | */ 28 | export function emptyRequests(httpEquivFile = null) { 29 | const req = new Requests(Site.fromSiteString("mozilla.org")); 30 | 31 | // Parse the HTML file for its own headers, if requested 32 | if (httpEquivFile) { 33 | const html = fs.readFileSync( 34 | path.join("test", "files", httpEquivFile), 35 | "utf8" 36 | ); 37 | 38 | // Load the HTML file into the object for content tests. 39 | req.resources.path = html; 40 | } 41 | 42 | req.responses.auto = { 43 | headers: new AxiosHeaders("Content-Type: text/html"), 44 | request: { 45 | headers: new AxiosHeaders(), 46 | }, 47 | status: 200, 48 | statusText: "OK", 49 | verified: true, 50 | data: "", 51 | config: { 52 | headers: new AxiosHeaders(), 53 | }, 54 | }; 55 | 56 | req.responses.cors = structuredClone(req.responses.auto); 57 | req.responses.http = structuredClone(req.responses.auto); 58 | req.responses.https = structuredClone(req.responses.auto); 59 | 60 | req.responses.httpRedirects = [ 61 | { 62 | url: new URL("http://mozilla.org/"), 63 | status: 301, 64 | }, 65 | { 66 | url: new URL("https://mozilla.org/"), 67 | status: 301, 68 | }, 69 | { 70 | url: new URL("https://www.mozilla.org/"), 71 | status: 200, 72 | }, 73 | ]; 74 | req.responses.httpsRedirects = [ 75 | { 76 | url: new URL("https://mozilla.org/"), 77 | status: 301, 78 | }, 79 | { 80 | url: new URL("https://www.mozilla.org/"), 81 | status: 200, 82 | }, 83 | { 84 | url: new URL("https://mozilla.org/"), 85 | status: 301, 86 | }, 87 | { 88 | url: new URL("https://www.mozilla.org/"), 89 | status: 200, 90 | }, 91 | { 92 | url: new URL("https://mozilla.org/robots.txt"), 93 | status: 301, 94 | }, 95 | { 96 | url: new URL("https://www.mozilla.org/robots.txt"), 97 | status: 200, 98 | }, 99 | ]; 100 | 101 | req.responses.auto.httpEquiv = new Map(); 102 | 103 | req.session = new Session(new URL("https://mozilla.org/")); 104 | 105 | // Parse the HTML file for its own headers, if requested 106 | if (req.resources.path) { 107 | req.responses.auto.httpEquiv = parseHttpEquivHeaders( 108 | req.resources.path, 109 | req.session.url.href 110 | ); 111 | } 112 | return req; 113 | } 114 | -------------------------------------------------------------------------------- /src/analyzer/tests/referrer-policy.js: -------------------------------------------------------------------------------- 1 | import { REFERRER_POLICY } from "../../headers.js"; 2 | import { Requests, BaseOutput } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getHttpHeaders } from "../utils.js"; 5 | 6 | export class ReferrerOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | http = false; 10 | meta = false; 11 | static name = "referrer-policy"; 12 | static title = "Referrer Policy"; 13 | static possibleResults = [ 14 | Expectation.ReferrerPolicyPrivate, 15 | Expectation.ReferrerPolicyNotImplemented, 16 | Expectation.ReferrerPolicyUnsafe, 17 | Expectation.ReferrerPolicyHeaderInvalid, 18 | ]; 19 | 20 | /** 21 | * 22 | * @param {Expectation} expectation 23 | */ 24 | constructor(expectation) { 25 | super(expectation); 26 | } 27 | } 28 | 29 | /** 30 | * 31 | * @param {Requests} requests 32 | * @param {Expectation} expectation 33 | * @returns {ReferrerOutput} 34 | */ 35 | export function referrerPolicyTest( 36 | requests, 37 | expectation = Expectation.ReferrerPolicyPrivate 38 | ) { 39 | const output = new ReferrerOutput(expectation); 40 | const goodness = [ 41 | "no-referrer", 42 | "same-origin", 43 | "strict-origin", 44 | "strict-origin-when-cross-origin", 45 | ]; 46 | const badness = [ 47 | "origin", 48 | "origin-when-cross-origin", 49 | "unsafe-url", 50 | "no-referrer-when-downgrade", 51 | ]; 52 | const valid = goodness.concat(badness); 53 | 54 | const response = requests.responses.auto; 55 | if (!response) { 56 | output.result = Expectation.ReferrerPolicyNotImplemented; 57 | return output; 58 | } 59 | 60 | const httpHeaders = getHttpHeaders(response, REFERRER_POLICY); 61 | const equivHeaders = response.httpEquiv?.get(REFERRER_POLICY) ?? []; 62 | 63 | // Store whether the header or the meta tag were present 64 | output.http = httpHeaders.length > 0; 65 | output.meta = equivHeaders ? equivHeaders?.length > 0 : false; 66 | 67 | // If it is both a header and a http-equiv, http-equiv has precedence (last value) 68 | if (output.http || output.meta) { 69 | output.data = [...httpHeaders, ...equivHeaders].join(", "); 70 | } else { 71 | output.result = Expectation.ReferrerPolicyNotImplemented; 72 | output.pass = true; 73 | return output; 74 | } 75 | 76 | // Find the last known valid policy value in the referrer policy 77 | let policy = 78 | output.data 79 | ?.split(",") 80 | .filter((e) => valid.includes(e.toLowerCase().trim())) 81 | .reverse()[0] 82 | ?.toLowerCase() 83 | .trim() ?? ""; 84 | 85 | if (goodness.includes(policy)) { 86 | output.result = Expectation.ReferrerPolicyPrivate; 87 | } else if (badness.includes(policy)) { 88 | output.result = Expectation.ReferrerPolicyUnsafe; 89 | } else { 90 | output.result = Expectation.ReferrerPolicyHeaderInvalid; 91 | } 92 | 93 | // Test if passed or fail 94 | output.pass = [ 95 | Expectation.ReferrerPolicyPrivate, 96 | Expectation.ReferrerPolicyNotImplemented, 97 | ].includes(output.result); 98 | 99 | return output; 100 | } 101 | -------------------------------------------------------------------------------- /src/scanner/index.js: -------------------------------------------------------------------------------- 1 | import { MINIMUM_SCORE_FOR_EXTRA_CREDIT } from "../grader/charts.js"; 2 | import { 3 | getGradeForScore, 4 | getScoreDescription, 5 | getScoreModifier, 6 | } from "../grader/grader.js"; 7 | import { retrieve } from "../retriever/retriever.js"; 8 | import { ALGORITHM_VERSION } from "../constants.js"; 9 | import { NUM_TESTS } from "../constants.js"; 10 | import { ALL_TESTS } from "../constants.js"; 11 | 12 | /** 13 | * @typedef {import("../types.js").ScanResult} ScanResult 14 | * @typedef {import("../types.js").Output} Output 15 | * @typedef {import("../types.js").StringMap} StringMap 16 | * @typedef {import("../types.js").TestMap} TestMap 17 | * @typedef {import("../site.js").Site} Site 18 | */ 19 | 20 | /** 21 | * @param {Site} site 22 | * @param {import("../types.js").ScanOptions} [options] 23 | * @returns {Promise} 24 | */ 25 | export async function scan(site, options) { 26 | let r = await retrieve(site, options); 27 | if (!r.responses.auto) { 28 | // We cannot connect at all, abort the test. 29 | throw new Error("The site seems to be down."); 30 | } 31 | 32 | // We allow 2xx, 3xx, 401 and 403 status codes 33 | const { status } = r.responses.auto; 34 | if (status < 200 || (status >= 400 && ![401, 403].includes(status))) { 35 | throw new Error( 36 | `Site did respond with an unexpected HTTP status code ${status}.` 37 | ); 38 | } 39 | 40 | // Run all the tests on the result 41 | /** @type {Output[]} */ 42 | const results = ALL_TESTS.map((test) => { 43 | return test(r); 44 | }); 45 | 46 | /** @type {StringMap} */ 47 | const responseHeaders = Object.entries(r.responses.auto.headers).reduce( 48 | (acc, [key, value]) => { 49 | acc[key] = value; 50 | return acc; 51 | }, 52 | /** @type {StringMap} */ ({}) 53 | ); 54 | const statusCode = r.responses.auto.status; 55 | 56 | let testsPassed = 0; 57 | let scoreWithExtraCredit = 100; 58 | let uncurvedScore = scoreWithExtraCredit; 59 | 60 | results.forEach((result) => { 61 | if (result.result) { 62 | result.scoreDescription = getScoreDescription(result.result); 63 | result.scoreModifier = getScoreModifier(result.result); 64 | testsPassed += result.pass ? 1 : 0; 65 | scoreWithExtraCredit += result.scoreModifier; 66 | if (result.scoreModifier < 0) { 67 | uncurvedScore += result.scoreModifier; 68 | } 69 | } 70 | }); 71 | 72 | // Only record the full score if the uncurved score already receives an A 73 | const score = 74 | uncurvedScore >= MINIMUM_SCORE_FOR_EXTRA_CREDIT 75 | ? scoreWithExtraCredit 76 | : uncurvedScore; 77 | 78 | const final = getGradeForScore(score); 79 | 80 | const tests = results.reduce((obj, result) => { 81 | const name = result.constructor.name; 82 | obj[name] = result; 83 | return obj; 84 | }, /** @type {TestMap} */ ({})); 85 | 86 | return { 87 | scan: { 88 | algorithmVersion: ALGORITHM_VERSION, 89 | grade: final.grade, 90 | error: null, 91 | score: final.score, 92 | statusCode: statusCode, 93 | testsFailed: NUM_TESTS - testsPassed, 94 | testsPassed: testsPassed, 95 | testsQuantity: NUM_TESTS, 96 | responseHeaders: responseHeaders, 97 | }, 98 | tests, 99 | }; 100 | } 101 | -------------------------------------------------------------------------------- /.github/workflows/_build.yml: -------------------------------------------------------------------------------- 1 | name: Build (reusable) 2 | description: Builds and pushes a Docker image. 3 | 4 | on: 5 | workflow_call: 6 | inputs: 7 | environment: 8 | type: string 9 | required: true 10 | 11 | ref: 12 | description: "Branch to build (default: main)" 13 | type: string 14 | 15 | tag: 16 | description: "Additional tag for the Docker image" 17 | type: string 18 | required: false 19 | 20 | env: 21 | IMAGE: mdn-observatory 22 | REGISTRY: us-docker.pkg.dev 23 | 24 | concurrency: 25 | group: build-${{ inputs.environment }} 26 | 27 | jobs: 28 | docker-build-push: 29 | environment: build 30 | runs-on: ubuntu-latest 31 | 32 | permissions: 33 | # Read/write GHA cache. 34 | actions: write 35 | # Checkout. 36 | contents: read 37 | # Authenticate with GCP. 38 | id-token: write 39 | 40 | steps: 41 | - name: Validate tag format 42 | if: inputs.tag 43 | env: 44 | TAG: ${{ inputs.tag }} 45 | run: | 46 | if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 47 | echo "❌ Invalid tag: $TAG does not match format vX.Y.Z (e.g., v1.2.3)" 48 | exit 1 49 | fi 50 | echo "✅ Valid tag: $TAG" 51 | 52 | - name: Checkout 53 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 54 | with: 55 | ref: ${{ inputs.ref || github.event.repository.default_branch }} 56 | persist-credentials: false 57 | 58 | - name: Docker setup 59 | uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 60 | 61 | - name: GCP auth 62 | id: gcp-auth 63 | uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 64 | with: 65 | token_format: access_token 66 | service_account: artifact-writer@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com 67 | workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions 68 | 69 | - name: Gather GIT_SHA and DOCKER_TAGS 70 | id: gather-build-metadata 71 | run: | 72 | IMAGE_PREFIX="${{ env.REGISTRY }}/${{ secrets.GCP_PROJECT_NAME }}/${{ secrets.GAR_REPOSITORY }}/${{ env.IMAGE }}" 73 | 74 | GIT_SHA="$(git rev-parse HEAD)" 75 | DOCKER_TAGS="$IMAGE_PREFIX:$GIT_SHA" 76 | 77 | if [ -n "${{ inputs.tag }}" ]; then 78 | DOCKER_TAGS="$DOCKER_TAGS,$IMAGE_PREFIX:${{ inputs.tag }}" 79 | fi 80 | 81 | echo "GIT_SHA=$GIT_SHA" >> "$GITHUB_OUTPUT" 82 | echo "DOCKER_TAGS=$DOCKER_TAGS" >> "$GITHUB_OUTPUT" 83 | 84 | - name: Docker login 85 | uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 86 | with: 87 | registry: ${{ env.REGISTRY }} 88 | username: oauth2accesstoken 89 | password: ${{ steps.gcp-auth.outputs.access_token }} 90 | 91 | - name: Build and push 92 | uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 93 | with: 94 | context: . 95 | build-args: | 96 | GIT_SHA=${{ steps.gather-build-metadata.outputs.GIT_SHA }} 97 | tags: ${{ steps.gather-build-metadata.outputs.DOCKER_TAGS }} 98 | push: true 99 | cache-from: type=gha 100 | cache-to: type=gha,mode=max 101 | -------------------------------------------------------------------------------- /src/retriever/retriever.js: -------------------------------------------------------------------------------- 1 | import { AxiosHeaders } from "axios"; 2 | import { CONFIG } from "../config.js"; 3 | import { HTML_TYPES, Requests } from "../types.js"; 4 | import { Session, getPageText } from "./session.js"; 5 | import { urls } from "./url.js"; 6 | import { parseHttpEquivHeaders } from "./utils.js"; 7 | 8 | const STANDARD_HEADERS = [ 9 | "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 10 | ]; 11 | const ROBOTS_HEADERS = ["Accept: text/plain,*/*;q=0.8"]; 12 | 13 | /** 14 | * 15 | * @param {import("../scanner/index.js").Site} site 16 | * @param {import("../types.js").ScanOptions} options 17 | * @returns {Promise} 18 | */ 19 | export async function retrieve(site, options = {}) { 20 | const retrievals = new Requests(site); 21 | const { http: httpUrl, https: httpsUrl } = await urls(site, options); 22 | 23 | const [httpSession, httpsSession] = await Promise.all([ 24 | Session.fromUrl(httpUrl, { headers: STANDARD_HEADERS, ...options.headers }), 25 | Session.fromUrl(httpsUrl, { 26 | headers: STANDARD_HEADERS, 27 | ...options.headers, 28 | }), 29 | ]); 30 | 31 | if (!httpSession && !httpsSession) { 32 | return retrievals; 33 | } 34 | 35 | retrievals.responses.http = httpSession.response; 36 | retrievals.responses.https = httpsSession.response; 37 | 38 | // use the http redirect chain 39 | retrievals.responses.httpRedirects = httpSession.redirectHistory; 40 | retrievals.responses.httpsRedirects = httpSession.redirectHistory; 41 | 42 | if (httpsSession.clientInstanceRecordingRedirects) { 43 | retrievals.responses.auto = httpsSession.response; 44 | retrievals.session = httpsSession; 45 | } else { 46 | retrievals.responses.auto = httpSession.response; 47 | retrievals.session = httpSession; 48 | } 49 | 50 | // Store the contents of the "base" page 51 | retrievals.resources.path = getPageText(retrievals.responses.auto, true); 52 | 53 | // Get robots.txt to gather additional cookies, if any. 54 | // robots.txt has to live at the root of the server, hence the leading slash 55 | await retrievals.session?.get({ 56 | path: "/robots.txt", 57 | headers: new AxiosHeaders(ROBOTS_HEADERS.join("\n")), 58 | }); 59 | 60 | // Do a CORS preflight request 61 | const corsUrl = retrievals.session.redirectHistory[ 62 | retrievals.session.redirectHistory.length - 1 63 | ] 64 | ? retrievals.session.redirectHistory[ 65 | retrievals.session.redirectHistory.length - 1 66 | ]?.url.href 67 | : retrievals.session.url.href; 68 | const cors_resp = 69 | (await retrievals.session?.options({ 70 | url: corsUrl, 71 | headers: { 72 | "Access-Control-Request-Method": "GET", 73 | Origin: CONFIG.retriever.corsOrigin, 74 | }, 75 | })) || null; 76 | 77 | if (cors_resp) { 78 | retrievals.responses.cors = { 79 | ...cors_resp, 80 | verified: retrievals.session.response?.verified ?? false, 81 | }; 82 | } else { 83 | retrievals.responses.cors = null; 84 | } 85 | 86 | if (retrievals.responses.auto) { 87 | if ( 88 | HTML_TYPES.has( 89 | retrievals.responses.auto.headers["content-type"]?.split(";")[0] 90 | ) && 91 | retrievals.resources.path 92 | ) { 93 | retrievals.responses.auto.httpEquiv = parseHttpEquivHeaders( 94 | retrievals.resources.path, 95 | retrievals.session.url.href 96 | ); 97 | } else { 98 | retrievals.responses.auto.httpEquiv = new Map(); 99 | } 100 | } 101 | 102 | return retrievals; 103 | } 104 | -------------------------------------------------------------------------------- /src/analyzer/tests/cross-origin-resource-policy.js: -------------------------------------------------------------------------------- 1 | import { CROSS_ORIGIN_RESOURCE_POLICY } from "../../headers.js"; 2 | import { BaseOutput, Requests } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getFirstHttpHeader } from "../utils.js"; 5 | 6 | export class CrossOriginResourcePolicyOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | http = false; 10 | meta = false; 11 | static name = "cross-origin-resource-policy"; 12 | static title = "Cross Origin Resource Policy"; 13 | static possibleResults = [ 14 | Expectation.CrossOriginResourcePolicyNotImplemented, 15 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin, 16 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite, 17 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin, 18 | Expectation.CrossOriginResourcePolicyHeaderInvalid, 19 | ]; 20 | 21 | /** 22 | * 23 | * @param {Expectation} expectation 24 | */ 25 | constructor(expectation) { 26 | super(expectation); 27 | } 28 | } 29 | 30 | /** 31 | * 32 | * @param {Requests} requests 33 | * @param {Expectation} expectation 34 | * @returns {CrossOriginResourcePolicyOutput} 35 | */ 36 | export function crossOriginResourcePolicyTest( 37 | requests, 38 | expectation = Expectation.CrossOriginResourcePolicyImplementedWithSameSite 39 | ) { 40 | const output = new CrossOriginResourcePolicyOutput(expectation); 41 | output.result = Expectation.CrossOriginResourcePolicyNotImplemented; 42 | 43 | const resp = requests.responses.auto; 44 | if (!resp) { 45 | return output; 46 | } 47 | 48 | const httpHeader = getFirstHttpHeader(resp, CROSS_ORIGIN_RESOURCE_POLICY); 49 | const equivHeaders = 50 | resp.httpEquiv?.get(CROSS_ORIGIN_RESOURCE_POLICY) ?? null; 51 | 52 | // Store whether the header or the meta tag were present 53 | output.http = !!httpHeader; 54 | output.meta = equivHeaders ? equivHeaders.length > 0 : false; 55 | 56 | // If it is both a header and a http-equiv, http-equiv has precedence (last value) 57 | /** @type {string | undefined} */ 58 | let corpHeader; 59 | if (output.http && httpHeader) { 60 | corpHeader = httpHeader.slice(0, 256).trim().toLowerCase(); 61 | } else if (output.meta) { 62 | if ( 63 | equivHeaders && 64 | Array.isArray(equivHeaders) && 65 | equivHeaders.length > 0 66 | ) { 67 | const h = equivHeaders[equivHeaders.length - 1]; 68 | if (h) { 69 | corpHeader = h.slice(0, 256).trim().toLowerCase(); 70 | } 71 | } 72 | } 73 | 74 | if (corpHeader) { 75 | output.data = corpHeader; 76 | if (corpHeader === "same-site") { 77 | output.result = 78 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite; 79 | } else if (corpHeader === "same-origin") { 80 | output.result = 81 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin; 82 | } else if (corpHeader === "cross-origin") { 83 | output.result = 84 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin; 85 | } else { 86 | output.result = Expectation.CrossOriginResourcePolicyHeaderInvalid; 87 | } 88 | } 89 | 90 | // Check to see if the test passed or failed 91 | output.pass = [ 92 | expectation, 93 | Expectation.CrossOriginResourcePolicyNotImplemented, 94 | Expectation.CrossOriginResourcePolicyImplementedWithSameSite, 95 | Expectation.CrossOriginResourcePolicyImplementedWithSameOrigin, 96 | Expectation.CrossOriginResourcePolicyImplementedWithCrossOrigin, 97 | ].includes(output.result ?? ""); 98 | 99 | return output; 100 | } 101 | -------------------------------------------------------------------------------- /test/retriever.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | 3 | import { retrieve } from "../src/retriever/retriever.js"; 4 | import { Session } from "../src/retriever/session.js"; 5 | import { Resources } from "../src/types.js"; 6 | import { Site } from "../src/site.js"; 7 | import { detectTlsSupport } from "../src/retriever/url.js"; 8 | import { CONFIG } from "../src/config.js"; 9 | 10 | describe("TestRetriever", () => { 11 | if (CONFIG.tests.hostForPortAndPathChecks !== "") { 12 | it("detects tls on a custom port", async () => { 13 | let site = Site.fromSiteString( 14 | `${CONFIG.tests.hostForPortAndPathChecks}:8443` 15 | ); 16 | let res = await detectTlsSupport(site); 17 | assert.isTrue(res); 18 | site = Site.fromSiteString( 19 | `${CONFIG.tests.hostForPortAndPathChecks}:8080` 20 | ); 21 | res = await detectTlsSupport(site); 22 | assert.isFalse(res); 23 | site = Site.fromSiteString( 24 | `${CONFIG.tests.hostForPortAndPathChecks}:8684` 25 | ); 26 | try { 27 | await detectTlsSupport(site); 28 | throw new Error("scan should throw"); 29 | } catch (e) { 30 | if (e instanceof Error) { 31 | assert.equal(e.name, "site-down"); 32 | } else { 33 | throw new Error("Unexpected error type"); 34 | } 35 | } 36 | }).timeout(10000); 37 | } 38 | 39 | it("correctly uses port and path on retrieving", async () => { 40 | let site = Site.fromSiteString("generalmagic.space:8443/test"); 41 | const requests = await retrieve(site); 42 | assert(requests.responses.auto); 43 | assert(requests.responses.auto.verified); 44 | assert.equal(requests.responses.httpRedirects.length, 3); 45 | }).timeout(10000); 46 | 47 | it("test retrieve mdn", async () => { 48 | const site = Site.fromSiteString("developer.mozilla.org/en-US"); 49 | const requests = await retrieve(site); 50 | // console.log("REQUESTS", requests); 51 | assert.isNotNull(requests.resources.path); 52 | assert.isNotNull(requests.responses.auto); 53 | assert.isNotNull(requests.responses.http); 54 | assert.isNotNull(requests.responses.https); 55 | assert.isNumber(requests.responses.http.status); 56 | assert.isNumber(requests.responses.https.status); 57 | assert.instanceOf(requests.session, Session); 58 | assert.equal(requests.site.hostname, "developer.mozilla.org"); 59 | assert.equal(requests.responses.httpRedirects.length, 3); 60 | assert.equal( 61 | "text/html", 62 | requests.responses.auto.headers["content-type"].substring(0, 9) 63 | ); 64 | assert.equal(200, requests.responses.auto.status); 65 | assert.equal( 66 | "https://developer.mozilla.org/en-US/", 67 | requests.responses.httpRedirects[ 68 | requests.responses.httpRedirects.length - 1 69 | ]?.url.href 70 | ); 71 | }).timeout(10000); 72 | 73 | it("test retrieve non-existent domain", async function () { 74 | const domain = 75 | Array(223) 76 | .fill(0) 77 | .map(() => String.fromCharCode(Math.random() * 26 + 97)) 78 | .join("") + ".net"; 79 | const site = Site.fromSiteString(domain); 80 | const requests = await retrieve(site); 81 | assert.isNull(requests.responses.auto); 82 | assert.isNull(requests.responses.cors); 83 | assert.isNull(requests.responses.http); 84 | assert.isNull(requests.responses.https); 85 | assert.isNotNull(requests.session); 86 | assert.isNull(requests.session.response); 87 | assert.equal(domain, requests.site.hostname); 88 | assert.deepEqual(new Resources(), requests.resources); 89 | }).timeout(10000); 90 | 91 | // test site seems to have outage from time to time, disable for now 92 | it("test_retrieve_invalid_cert", async function () { 93 | const site = Site.fromSiteString("expired.badssl.com"); 94 | const reqs = await retrieve(site); 95 | assert.isNotNull(reqs.responses.auto); 96 | assert.isFalse(reqs.responses.auto.verified); 97 | }).timeout(10000); 98 | }); 99 | -------------------------------------------------------------------------------- /migrations/007.undo.history-mat-view.sql: -------------------------------------------------------------------------------- 1 | DROP MATERIALIZED VIEW IF EXISTS grade_distribution; 2 | 3 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_scans 4 | AS SELECT latest_scans.site_id, latest_scans.scan_id, s.domain, latest_scans.state, 5 | latest_scans.start_time, latest_scans.end_time, latest_scans.tests_failed, latest_scans.tests_passed, 6 | latest_scans.grade, latest_scans.score, latest_scans.error 7 | FROM sites s, 8 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 9 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time DESC LIMIT 1 ) latest_scans; 10 | CREATE UNIQUE INDEX IF NOT EXISTS latest_scans_scan_id_idx ON latest_scans (scan_id); 11 | COMMENT ON MATERIALIZED VIEW latest_scans IS 'Most recently completed scan for a given website'; 12 | 13 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_tests 14 | AS SELECT latest_scans.domain, tests.site_id, tests.scan_id, name, result, pass, output 15 | FROM tests 16 | INNER JOIN latest_scans 17 | ON (latest_scans.scan_id = tests.scan_id); 18 | COMMENT ON MATERIALIZED VIEW latest_tests IS 'Test results from all the most recent scans'; 19 | 20 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution 21 | AS SELECT grade, count(*) 22 | FROM latest_scans 23 | GROUP BY grade; 24 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_grade_idx ON grade_distribution (grade); 25 | COMMENT ON MATERIALIZED VIEW grade_distribution IS 'The grades and how many latest scans have that score'; 26 | 27 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution_all_scans 28 | AS SELECT grade, count(*) 29 | FROM scans 30 | WHERE state = 'FINISHED' 31 | GROUP BY grade; 32 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_all_scans_grade_idx ON grade_distribution_all_scans (grade); 33 | COMMENT ON MATERIALIZED VIEW grade_distribution_all_scans IS 'The grades and how many scans have that score'; 34 | 35 | 36 | CREATE MATERIALIZED VIEW IF NOT EXISTS earliest_scans 37 | AS SELECT earliest_scans.site_id, earliest_scans.scan_id, s.domain, earliest_scans.state, 38 | earliest_scans.start_time, earliest_scans.end_time, earliest_scans.tests_failed, earliest_scans.tests_passed, 39 | earliest_scans.grade, earliest_scans.score, earliest_scans.error 40 | FROM sites s, 41 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 42 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time ASC LIMIT 1 ) earliest_scans; 43 | CREATE UNIQUE INDEX IF NOT EXISTS earliest_scans_scan_id_idx ON earliest_scans (scan_id); 44 | COMMENT ON MATERIALIZED VIEW earliest_scans IS 'Oldest completed scan for a given website'; 45 | 46 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution 47 | AS SELECT earliest_scans.site_id, earliest_scans.domain, earliest_scans.score AS before, latest_scans.score AS after, 48 | (latest_scans.score - earliest_scans.score) AS difference 49 | FROM earliest_scans, latest_scans 50 | WHERE earliest_scans.site_id = latest_scans.site_id; 51 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution IS 'How much score has changed since first scan'; 52 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_site_id_idx ON scan_score_difference_distribution (site_id); 53 | CREATE INDEX scan_score_difference_difference_distribution_idx ON scan_score_difference_distribution (difference); 54 | 55 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution_summation 56 | AS SELECT DISTINCT difference, COUNT(difference) AS num_sites 57 | FROM scan_score_difference_distribution 58 | GROUP BY difference 59 | ORDER BY difference DESC; 60 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_summation_difference_idx ON scan_score_difference_distribution_summation (difference); 61 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution_summation IS 'How many sites have improved by how many points'; 62 | 63 | -------------------------------------------------------------------------------- /src/config.js: -------------------------------------------------------------------------------- 1 | import convict from "convict"; 2 | 3 | const SCHEMA = { 4 | retriever: { 5 | retrieverUserAgent: { 6 | doc: "The user agent to use for retriever requests.", 7 | format: "String", 8 | default: 9 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:129.0) Gecko/20100101 Firefox/129.0 Observatory/129.0", 10 | env: "RETRIEVER_USER_AGENT", 11 | }, 12 | corsOrigin: { 13 | doc: "The CORS origin to use for CORS origin retriever requests.", 14 | format: "String", 15 | default: "https://http-observatory.security.mozilla.org", 16 | env: "CORS_ORIGIN", 17 | }, 18 | abortTimeout: { 19 | doc: "The overall timeout for a request, in ms", 20 | format: "Number", 21 | default: 10000, 22 | env: "ABORT_TIMEOUT", 23 | }, 24 | clientTimeout: { 25 | doc: "The timeout once the request has been sent, in ms", 26 | format: "Number", 27 | default: 9000, 28 | env: "CLIENT_TIMEOUT", 29 | }, 30 | }, 31 | database: { 32 | database: { 33 | doc: "The name of the database to use", 34 | format: "String", 35 | default: "httpobservatory", 36 | env: "PGDATABASE", 37 | }, 38 | host: { 39 | doc: "The database server hostname", 40 | format: "String", 41 | default: "localhost", 42 | env: "PGHOST", 43 | }, 44 | user: { 45 | doc: "Database username", 46 | format: "String", 47 | default: "postgres", 48 | env: "PGUSER", 49 | }, 50 | pass: { 51 | doc: "Database password", 52 | format: "String", 53 | default: "", 54 | sensitive: true, 55 | env: "PGPASSWORD", 56 | }, 57 | port: { 58 | doc: "The port of the database service", 59 | format: "port", 60 | default: 5432, 61 | env: "PGPORT", 62 | }, 63 | sslmode: { 64 | doc: "Database SSL mode", 65 | format: "Boolean", 66 | default: false, 67 | env: "PGSSLMODE", 68 | }, 69 | }, 70 | api: { 71 | cooldown: { 72 | doc: "Cached result time for API V2, in Seconds. Defaults to 1 minute", 73 | format: "nat", 74 | default: 60, 75 | env: "HTTPOBS_API_COOLDOWN", 76 | }, 77 | cacheTimeForGet: { 78 | doc: "Maximum scan age a GET request returns before initiating a new scan, in seconds. Defaults to 24 hours.", 79 | format: "nat", 80 | default: 86400, 81 | env: "HTTPOBS_API_GET_CACHE", 82 | }, 83 | port: { 84 | doc: "The port to bind to", 85 | format: "Number", 86 | default: 8080, 87 | env: "HTTPOBS_API_PORT", 88 | }, 89 | enableLogging: { 90 | doc: "Enable server logging", 91 | format: "Boolean", 92 | default: true, 93 | env: "HTTPOBS_ENABLE_LOGGING", 94 | }, 95 | }, 96 | sentry: { 97 | dsn: { 98 | doc: "The Sentry data source name (DSN) to use for error reporting.", 99 | format: "String", 100 | default: "", 101 | env: "SENTRY_DSN", 102 | }, 103 | }, 104 | tests: { 105 | enableDBTests: { 106 | doc: "Enable database tests", 107 | format: "Boolean", 108 | default: false, 109 | env: "HTTPOBS_TESTS_ENABLE_DB_TESTS", 110 | }, 111 | hostForPortAndPathChecks: { 112 | doc: "Host to use for custom port and path checks", 113 | format: "String", 114 | default: "", 115 | env: "HTTPOBS_TESTS_HOST_FOR_PORT_AND_PATH_CHECKS", 116 | }, 117 | }, 118 | }; 119 | 120 | /** 121 | * 122 | * @param {string | undefined} configFile 123 | * @returns 124 | */ 125 | export function load(configFile) { 126 | const configuration = convict(SCHEMA); 127 | try { 128 | if (configFile) { 129 | configuration.loadFile(configFile); 130 | } 131 | configuration.validate({ allowed: "strict" }); 132 | return configuration.getProperties(); 133 | } catch (e) { 134 | throw new Error(`error reading config: ${e}`); 135 | } 136 | } 137 | 138 | export const CONFIG = load(process.env["CONFIG_FILE"]); 139 | -------------------------------------------------------------------------------- /src/analyzer/tests/strict-transport-security.js: -------------------------------------------------------------------------------- 1 | import { STRICT_TRANSPORT_SECURITY } from "../../headers.js"; 2 | import { Requests, BaseOutput } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { isHstsPreloaded } from "../hsts.js"; 5 | import { getHttpHeaders } from "../utils.js"; 6 | export class StrictTransportSecurityOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | data = null; 9 | includeSubDomains = false; 10 | /** @type {number | null} */ 11 | maxAge = null; 12 | preload = false; 13 | preloaded = false; 14 | static name = "strict-transport-security"; 15 | static title = "Strict Transport Security (HSTS)"; 16 | static possibleResults = [ 17 | Expectation.HstsPreloaded, 18 | Expectation.HstsImplementedMaxAgeAtLeastSixMonths, 19 | Expectation.HstsImplementedMaxAgeLessThanSixMonths, 20 | Expectation.HstsNotImplemented, 21 | Expectation.HstsHeaderInvalid, 22 | Expectation.HstsNotImplementedNoHttps, 23 | Expectation.HstsInvalidCert, 24 | ]; 25 | /** 26 | * 27 | * @param {Expectation} expectation 28 | */ 29 | constructor(expectation) { 30 | super(expectation); 31 | } 32 | } 33 | 34 | // 15768000 is six months, but a lot of sites use 15552000, so a white lie is in order 35 | const SIX_MONTHS = 15552000; 36 | 37 | /** 38 | * 39 | * @param {Requests} requests 40 | * @param {Expectation} expectation 41 | * @returns {StrictTransportSecurityOutput} 42 | */ 43 | export function strictTransportSecurityTest( 44 | requests, 45 | expectation = Expectation.HstsImplementedMaxAgeAtLeastSixMonths 46 | ) { 47 | const output = new StrictTransportSecurityOutput(expectation); 48 | output.result = Expectation.HstsNotImplemented; 49 | const response = requests.responses.https; 50 | if (!response) { 51 | // If there's no HTTPS, we can't have HSTS 52 | output.result = Expectation.HstsNotImplementedNoHttps; 53 | } else if (!response.verified) { 54 | // Also need a valid certificate chain for HSTS 55 | output.result = Expectation.HstsInvalidCert; 56 | } else if (getHttpHeaders(response, STRICT_TRANSPORT_SECURITY).length > 0) { 57 | const header = getHttpHeaders(response, STRICT_TRANSPORT_SECURITY)[0] || ""; 58 | output.data = header.slice(0, 1024); // code against malicious headers 59 | 60 | try { 61 | let sts = output.data.split(";").map((i) => i.trim().toLowerCase()); 62 | // Throw an error if the header is set twice 63 | if (output.data.includes(",")) { 64 | throw new Error("Header set multiple times"); 65 | } 66 | sts.forEach((parameter) => { 67 | if (parameter.startsWith("max-age=")) { 68 | // Use slice to get the part of the string after 'max-age=' 69 | // Parse it to an integer. We're slicing up to 128 characters as a defense mechanism. 70 | output.maxAge = parseInt(parameter.slice(8, 128), 10); 71 | } else if (parameter === "includesubdomains") { 72 | output.includeSubDomains = true; 73 | } else if (parameter === "preload") { 74 | output.preload = true; 75 | } 76 | }); 77 | if (output.maxAge !== null) { 78 | if (output.maxAge < SIX_MONTHS) { 79 | output.result = Expectation.HstsImplementedMaxAgeLessThanSixMonths; 80 | } else { 81 | output.result = Expectation.HstsImplementedMaxAgeAtLeastSixMonths; 82 | } 83 | } else { 84 | throw new Error("MaxAge missing"); 85 | } 86 | } catch (e) { 87 | output.result = Expectation.HstsHeaderInvalid; 88 | } 89 | } 90 | 91 | // If they're in the preloaded list, this overrides most anything else 92 | if (response) { 93 | const preloaded = isHstsPreloaded(requests.site); 94 | if (preloaded) { 95 | output.result = Expectation.HstsPreloaded; 96 | output.includeSubDomains = preloaded.includeSubDomains; 97 | output.preloaded = true; 98 | } 99 | } 100 | // Check to see if the test passed or failed 101 | if ( 102 | [ 103 | Expectation.HstsImplementedMaxAgeAtLeastSixMonths, 104 | Expectation.HstsPreloaded, 105 | expectation, 106 | ].includes(output.result) 107 | ) { 108 | output.pass = true; 109 | } 110 | return output; 111 | } 112 | -------------------------------------------------------------------------------- /src/analyzer/tests/redirection.js: -------------------------------------------------------------------------------- 1 | import { Site } from "../../site.js"; 2 | import { BaseOutput, Requests } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { isHstsPreloaded } from "../hsts.js"; 5 | 6 | export class RedirectionOutput extends BaseOutput { 7 | /** @type {string | null} */ 8 | destination = null; 9 | redirects = true; 10 | /** @type {string[]} */ 11 | route = []; 12 | /** @type {number | null} */ 13 | statusCode = null; 14 | static name = "redirection"; 15 | static title = "Redirection"; 16 | static possibleResults = [ 17 | Expectation.RedirectionAllRedirectsPreloaded, 18 | Expectation.RedirectionToHttps, 19 | Expectation.RedirectionNotNeededNoHttp, 20 | Expectation.RedirectionOffHostFromHttp, 21 | Expectation.RedirectionNotToHttpsOnInitialRedirection, 22 | Expectation.RedirectionNotToHttps, 23 | Expectation.RedirectionMissing, 24 | Expectation.RedirectionInvalidCert, 25 | ]; 26 | 27 | /** 28 | * 29 | * @param {Expectation} expectation 30 | */ 31 | constructor(expectation) { 32 | super(expectation); 33 | } 34 | } 35 | 36 | /** 37 | * 38 | * @param {Requests} requests 39 | * @param {Expectation} expectation 40 | * @returns {RedirectionOutput} 41 | */ 42 | export function redirectionTest( 43 | requests, 44 | expectation = Expectation.RedirectionToHttps 45 | ) { 46 | const output = new RedirectionOutput(expectation); 47 | const response = requests.responses.http; 48 | 49 | if (requests.responses.httpRedirects.length > 0) { 50 | output.destination = 51 | requests.responses.httpRedirects[ 52 | requests.responses.httpRedirects.length - 1 53 | ]?.url?.href || null; 54 | } else if (requests.responses.httpsRedirects.length > 0) { 55 | output.destination = 56 | requests.responses.httpsRedirects[ 57 | requests.responses.httpsRedirects.length - 1 58 | ]?.url?.href || null; 59 | } 60 | output.statusCode = response ? response.status : null; 61 | 62 | if (!response) { 63 | output.result = Expectation.RedirectionNotNeededNoHttp; 64 | } else if (!response.verified) { 65 | output.result = Expectation.RedirectionInvalidCert; 66 | } else { 67 | const route = requests.responses.httpRedirects; 68 | output.route = route.map((r) => r.url.href); 69 | 70 | // Check to see if every redirection was covered by the preload list 71 | const allRedirectsPreloaded = route.every((re) => 72 | isHstsPreloaded(Site.fromSiteString(re.url.hostname)) 73 | ); 74 | if (allRedirectsPreloaded) { 75 | output.result = Expectation.RedirectionAllRedirectsPreloaded; 76 | } else if (route.length === 1) { 77 | // No redirection, so you just stayed on the http website 78 | output.result = Expectation.RedirectionMissing; 79 | output.redirects = false; 80 | } else if (route[route.length - 1]?.url.protocol !== "https:") { 81 | // Final destination wasn't an https website 82 | output.result = Expectation.RedirectionNotToHttps; 83 | } else if (route[1]?.url.protocol === "http:") { 84 | // http should never redirect to another http location -- should always go to https first 85 | output.result = Expectation.RedirectionNotToHttpsOnInitialRedirection; 86 | output.statusCode = route[route.length - 1]?.status || null; 87 | } else if ( 88 | route[0]?.url.protocol === "http:" && 89 | route[1]?.url.protocol === "https:" && 90 | route[0]?.url.hostname !== route[1]?.url.hostname 91 | ) { 92 | output.result = Expectation.RedirectionOffHostFromHttp; 93 | } else { 94 | // Yeah, you're good 95 | output.result = Expectation.RedirectionToHttps; 96 | } 97 | } 98 | // Code defensively against infinite routing loops and other shenanigans 99 | output.route = JSON.stringify(output.route).length > 8192 ? [] : output.route; 100 | output.statusCode = 101 | `${output.statusCode}`.length < 5 ? output.statusCode : null; 102 | 103 | // Check to see if the test passed or failed 104 | if ( 105 | [ 106 | Expectation.RedirectionNotNeededNoHttp, 107 | Expectation.RedirectionAllRedirectsPreloaded, 108 | expectation, 109 | ].includes(output.result) 110 | ) { 111 | output.pass = true; 112 | } 113 | 114 | return output; 115 | } 116 | -------------------------------------------------------------------------------- /test/strict-transport-security.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { strictTransportSecurityTest } from "../src/analyzer/tests/strict-transport-security.js"; 4 | import { Expectation } from "../src/types.js"; 5 | import { Site } from "../src/site.js"; 6 | 7 | describe("Strict Transport Security", () => { 8 | /** 9 | * @type {import("../src/types.js").Requests} 10 | */ 11 | let reqs; 12 | beforeEach(() => { 13 | reqs = emptyRequests(); 14 | }); 15 | 16 | it("missing", function () { 17 | const result = strictTransportSecurityTest(reqs); 18 | assert.equal(result.result, Expectation.HstsNotImplemented); 19 | assert.isFalse(result.pass); 20 | }); 21 | 22 | it("header invalid", function () { 23 | assert.isNotNull(reqs.responses.https); 24 | reqs.responses.https.headers["strict-transport-security"] = 25 | "includeSubDomains; preload"; 26 | let result = strictTransportSecurityTest(reqs); 27 | assert.equal(result.result, Expectation.HstsHeaderInvalid); 28 | assert.isFalse(result.pass); 29 | 30 | reqs.responses.https.headers["strict-transport-security"] = 31 | "max-age=15768000; includeSubDomains, max-age=15768000; includeSubDomains"; 32 | result = strictTransportSecurityTest(reqs); 33 | assert.equal(result.result, Expectation.HstsHeaderInvalid); 34 | assert.isFalse(result.pass); 35 | }); 36 | 37 | it("no https", function () { 38 | assert.isNotNull(reqs.responses.auto); 39 | reqs.responses.auto.headers["strict-transport-security"] = 40 | "max-age=15768000"; 41 | assert.isNotNull(reqs.responses.http); 42 | reqs.responses.http.headers["strict-transport-security"] = 43 | "max-age=15768000"; 44 | reqs.responses.https = null; 45 | 46 | const result = strictTransportSecurityTest(reqs); 47 | assert.equal(result.result, Expectation.HstsNotImplementedNoHttps); 48 | assert.isFalse(result.pass); 49 | }); 50 | 51 | it("invalid cert", function () { 52 | assert.isNotNull(reqs.responses.https); 53 | reqs.responses.https.headers["strict-transport-security"] = 54 | "max-age=15768000; includeSubDomains; preload"; 55 | reqs.responses.https.verified = false; 56 | 57 | const result = strictTransportSecurityTest(reqs); 58 | assert.equal(result.result, Expectation.HstsInvalidCert); 59 | assert.isFalse(result.pass); 60 | }); 61 | 62 | it("max age too low", function () { 63 | assert.isNotNull(reqs.responses.https); 64 | reqs.responses.https.headers["Strict-Transport-Security"] = "max-age=86400"; 65 | 66 | const result = strictTransportSecurityTest(reqs); 67 | assert.equal( 68 | result.result, 69 | Expectation.HstsImplementedMaxAgeLessThanSixMonths 70 | ); 71 | assert.isFalse(result.pass); 72 | }); 73 | 74 | it("implemented", function () { 75 | assert.isNotNull(reqs.responses.https); 76 | reqs.responses.https.headers["strict-transport-security"] = 77 | "max-age=15768000; includeSubDomains; preload"; 78 | 79 | const result = strictTransportSecurityTest(reqs); 80 | assert.equal( 81 | result.result, 82 | Expectation.HstsImplementedMaxAgeAtLeastSixMonths 83 | ); 84 | assert.equal(result.maxAge, 15768000); 85 | assert.isTrue(result.includeSubDomains); 86 | assert.isTrue(result.preload); 87 | assert.isTrue(result.pass); 88 | }); 89 | 90 | it("preloaded", function () { 91 | reqs.site = Site.fromSiteString("bugzilla.mozilla.org"); 92 | let result = strictTransportSecurityTest(reqs); 93 | assert.equal(result.result, Expectation.HstsPreloaded); 94 | assert.isTrue(result.includeSubDomains); 95 | assert.isTrue(result.pass); 96 | assert.isTrue(result.preloaded); 97 | 98 | reqs.site = Site.fromSiteString("facebook.com"); 99 | result = strictTransportSecurityTest(reqs); 100 | assert.equal(result.result, Expectation.HstsPreloaded); 101 | assert.isFalse(result.includeSubDomains); 102 | assert.isTrue(result.pass); 103 | assert.isTrue(result.preloaded); 104 | 105 | reqs.site = Site.fromSiteString("dropboxusercontent.com"); 106 | result = strictTransportSecurityTest(reqs); 107 | assert.equal(result.result, Expectation.HstsNotImplemented); 108 | assert.isFalse(result.includeSubDomains); 109 | assert.isFalse(result.pass); 110 | assert.isFalse(result.preloaded); 111 | }); 112 | }); 113 | -------------------------------------------------------------------------------- /src/api/server.js: -------------------------------------------------------------------------------- 1 | import Fastify from "fastify"; 2 | import simpleFormPlugin from "fastify-simple-form"; 3 | import cors from "@fastify/cors"; 4 | import helmet from "@fastify/helmet"; 5 | import * as Sentry from "@sentry/node"; 6 | 7 | // import analyzeApiV1 from "./v1/analyze/index.js"; 8 | import analyzeApiV2 from "./v2/analyze/index.js"; 9 | import scanApiV2 from "./v2/scan/index.js"; 10 | import statsApiV2 from "./v2/stats/index.js"; 11 | import recommendationMatrixApiV2 from "./v2/recommendations/index.js"; 12 | import version from "./version/index.js"; 13 | import globalErrorHandler from "./global-error-handler.js"; 14 | import pool from "@fastify/postgres"; 15 | import { poolOptions } from "../database/repository.js"; 16 | import { CONFIG } from "../config.js"; 17 | 18 | const FILTERED_ERROR_TYPES = [ 19 | "invalid-hostname", 20 | "invalid-hostname-lookup", 21 | "invalid-hostname-ip", 22 | "scan-failed", 23 | "site-down", 24 | ]; 25 | const FILTERED_ERROR_CODES = [ 26 | "FST_ERR_VALIDATION", 27 | "FST_ERR_CTP_INVALID_MEDIA_TYPE", 28 | ]; 29 | const FILTERED_STATUS_CODES = [422]; 30 | 31 | if (CONFIG.sentry.dsn) { 32 | Sentry.init({ 33 | dsn: CONFIG.sentry.dsn, 34 | beforeSend(event, hint) { 35 | // Filter all 422 status codes 36 | const originalError = hint.originalException; 37 | if ( 38 | // @ts-expect-error 39 | FILTERED_STATUS_CODES.includes(originalError?.statusCode) || 40 | // @ts-expect-error 41 | FILTERED_STATUS_CODES.includes(originalError?.originalError?.status) 42 | ) { 43 | return null; 44 | } 45 | // Also check event tags for HTTP status 46 | if ( 47 | FILTERED_STATUS_CODES.includes( 48 | Number(event.tags?.["http.status_code"] || 0) 49 | ) 50 | ) { 51 | return null; 52 | } 53 | // Filter out common user errors 54 | // @ts-expect-error 55 | const errorType = originalError?.name || ""; 56 | if (FILTERED_ERROR_TYPES.includes(errorType)) { 57 | return null; 58 | } 59 | // Filter out errors from query schema validation 60 | // @ts-ignore 61 | const errorMessage = originalError?.code || ""; 62 | if (FILTERED_ERROR_CODES.includes(errorMessage)) { 63 | return null; 64 | } 65 | return event; 66 | }, 67 | }); 68 | } 69 | 70 | /** 71 | * Creates a Fastify server instance 72 | * @returns {Promise} 73 | */ 74 | export async function createServer() { 75 | const server = Fastify({ 76 | logger: CONFIG.api.enableLogging, 77 | }); 78 | 79 | if (CONFIG.sentry.dsn) { 80 | server.log.error("Sentry enabled"); 81 | Sentry.setupFastifyErrorHandler(server); 82 | } 83 | 84 | // @ts-ignore 85 | server.register(simpleFormPlugin); 86 | await server.register(cors, { 87 | origin: "*", 88 | methods: ["GET", "OPTIONS", "HEAD", "POST"], 89 | maxAge: 86400, 90 | }); 91 | await server.register(helmet, { 92 | contentSecurityPolicy: { 93 | useDefaults: false, 94 | directives: { 95 | defaultSrc: ["'none'"], 96 | baseUri: ["'none'"], 97 | formAction: ["'none'"], 98 | frameAnchestors: ["'none'"], 99 | }, 100 | }, 101 | 102 | hsts: { 103 | maxAge: 63072000, 104 | includeSubDomains: false, 105 | }, 106 | frameguard: { 107 | action: "deny", 108 | }, 109 | xXssProtection: true, 110 | referrerPolicy: { 111 | policy: "no-referrer", 112 | }, 113 | }); 114 | server.register(pool, poolOptions); 115 | server.setErrorHandler(globalErrorHandler); 116 | 117 | server.get("/", {}, async (_request, _reply) => { 118 | return "Welcome to the MDN Observatory!"; 119 | }); 120 | 121 | // await Promise.all([server.register(analyzeApiV1, { prefix: "/api/v1" })]); 122 | await Promise.all([ 123 | server.register(analyzeApiV2, { prefix: "/api/v2" }), 124 | server.register(scanApiV2, { prefix: "/api/v2" }), 125 | server.register(statsApiV2, { prefix: "/api/v2" }), 126 | server.register(recommendationMatrixApiV2, { prefix: "/api/v2" }), 127 | server.register(version, { prefix: "/api/v2" }), 128 | ]); 129 | 130 | ["SIGINT", "SIGTERM"].forEach((signal) => { 131 | process.on(signal, async () => { 132 | await server.close(); 133 | process.exit(0); 134 | }); 135 | }); 136 | 137 | return server; 138 | } 139 | -------------------------------------------------------------------------------- /migrations/006.do.mat-views.sql: -------------------------------------------------------------------------------- 1 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_scans 2 | AS SELECT latest_scans.site_id, latest_scans.scan_id, s.domain, latest_scans.state, 3 | latest_scans.start_time, latest_scans.end_time, latest_scans.tests_failed, latest_scans.tests_passed, 4 | latest_scans.grade, latest_scans.score, latest_scans.error 5 | FROM sites s, 6 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 7 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time DESC LIMIT 1 ) latest_scans; 8 | CREATE UNIQUE INDEX IF NOT EXISTS latest_scans_scan_id_idx ON latest_scans (scan_id); 9 | COMMENT ON MATERIALIZED VIEW latest_scans IS 'Most recently completed scan for a given website'; 10 | GRANT SELECT ON latest_scans TO httpobsapi; 11 | 12 | CREATE MATERIALIZED VIEW IF NOT EXISTS latest_tests 13 | AS SELECT latest_scans.domain, tests.site_id, tests.scan_id, name, result, pass, output 14 | FROM tests 15 | INNER JOIN latest_scans 16 | ON (latest_scans.scan_id = tests.scan_id); 17 | COMMENT ON MATERIALIZED VIEW latest_tests IS 'Test results from all the most recent scans'; 18 | 19 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution 20 | AS SELECT grade, count(*) 21 | FROM latest_scans 22 | GROUP BY grade; 23 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_grade_idx ON grade_distribution (grade); 24 | COMMENT ON MATERIALIZED VIEW grade_distribution IS 'The grades and how many latest scans have that score'; 25 | GRANT SELECT ON grade_distribution TO httpobsapi; 26 | 27 | CREATE MATERIALIZED VIEW IF NOT EXISTS grade_distribution_all_scans 28 | AS SELECT grade, count(*) 29 | FROM scans 30 | WHERE state = 'FINISHED' 31 | GROUP BY grade; 32 | CREATE UNIQUE INDEX IF NOT EXISTS grade_distribution_all_scans_grade_idx ON grade_distribution_all_scans (grade); 33 | COMMENT ON MATERIALIZED VIEW grade_distribution_all_scans IS 'The grades and how many scans have that score'; 34 | GRANT SELECT ON grade_distribution_all_scans TO httpobsapi; 35 | 36 | 37 | CREATE MATERIALIZED VIEW IF NOT EXISTS earliest_scans 38 | AS SELECT earliest_scans.site_id, earliest_scans.scan_id, s.domain, earliest_scans.state, 39 | earliest_scans.start_time, earliest_scans.end_time, earliest_scans.tests_failed, earliest_scans.tests_passed, 40 | earliest_scans.grade, earliest_scans.score, earliest_scans.error 41 | FROM sites s, 42 | LATERAL ( SELECT id AS scan_id, site_id, state, start_time, end_time, tests_failed, tests_passed, grade, score, error 43 | FROM scans WHERE site_id = s.id AND state = 'FINISHED' ORDER BY end_time ASC LIMIT 1 ) earliest_scans; 44 | CREATE UNIQUE INDEX IF NOT EXISTS earliest_scans_scan_id_idx ON earliest_scans (scan_id); 45 | COMMENT ON MATERIALIZED VIEW earliest_scans IS 'Oldest completed scan for a given website'; 46 | GRANT SELECT ON earliest_scans TO httpobsapi; 47 | 48 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution 49 | AS SELECT earliest_scans.site_id, earliest_scans.domain, earliest_scans.score AS before, latest_scans.score AS after, 50 | (latest_scans.score - earliest_scans.score) AS difference 51 | FROM earliest_scans, latest_scans 52 | WHERE earliest_scans.site_id = latest_scans.site_id; 53 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution IS 'How much score has changed since first scan'; 54 | GRANT SELECT ON scan_score_difference_distribution TO httpobsapi; 55 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_site_id_idx ON scan_score_difference_distribution (site_id); 56 | CREATE INDEX scan_score_difference_difference_distribution_idx ON scan_score_difference_distribution (difference); 57 | 58 | CREATE MATERIALIZED VIEW IF NOT EXISTS scan_score_difference_distribution_summation 59 | AS SELECT DISTINCT difference, COUNT(difference) AS num_sites 60 | FROM scan_score_difference_distribution 61 | GROUP BY difference 62 | ORDER BY difference DESC; 63 | CREATE UNIQUE INDEX IF NOT EXISTS scan_score_difference_distribution_summation_difference_idx ON scan_score_difference_distribution_summation (difference); 64 | COMMENT ON MATERIALIZED VIEW scan_score_difference_distribution_summation IS 'How many sites have improved by how many points'; 65 | GRANT SELECT ON scan_score_difference_distribution_summation TO httpobsapi; 66 | 67 | ALTER MATERIALIZED VIEW grade_distribution OWNER TO httpobsapi; /* so it can refresh */ 68 | ALTER MATERIALIZED VIEW grade_distribution_all_scans OWNER TO httpobsapi; /* so it can refresh */ 69 | ALTER MATERIALIZED VIEW latest_scans OWNER TO httpobsapi; 70 | ALTER MATERIALIZED VIEW earliest_scans OWNER TO httpobsapi; 71 | ALTER MATERIALIZED VIEW scan_score_difference_distribution OWNER TO httpobsapi; 72 | ALTER MATERIALIZED VIEW scan_score_difference_distribution_summation OWNER TO httpobsapi; 73 | ALTER MATERIALIZED VIEW latest_tests OWNER TO httpobsapi; -------------------------------------------------------------------------------- /test/referrer-policy.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { referrerPolicyTest } from "../src/analyzer/tests/referrer-policy.js"; 5 | 6 | describe("ReferrerPolicy", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | 10 | beforeEach(() => { 11 | reqs = emptyRequests(); 12 | }); 13 | 14 | it("checks for private referrer header", function () { 15 | const privValues = [ 16 | "no-referrer", 17 | "same-origin", 18 | "strict-origin", 19 | "STRICT-ORIGIN", 20 | "strict-origin-when-cross-origin", 21 | ]; 22 | assert.isNotNull(reqs.responses.auto); 23 | for (const v of privValues) { 24 | reqs.responses.auto.headers["Referrer-Policy"] = v; 25 | const result = referrerPolicyTest(reqs); 26 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 27 | assert.isTrue(result.http); 28 | assert.isFalse(result.meta); 29 | assert.isTrue(result.pass); 30 | } 31 | 32 | { 33 | // Test with a meta/http-equiv header 34 | reqs = emptyRequests("test_parse_http_equiv_headers_referrer1.html"); 35 | const result = referrerPolicyTest(reqs); 36 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 37 | assert.equal(result.data, "no-referrer, same-origin"); 38 | assert.isFalse(result.http); 39 | assert.isTrue(result.meta); 40 | assert.isTrue(result.pass); 41 | } 42 | 43 | { 44 | // The meta/http-equiv header has precendence over the http header 45 | assert.isNotNull(reqs.responses.auto); 46 | reqs.responses.auto.headers["referrer-policy"] = "unsafe-url"; 47 | const result = referrerPolicyTest(reqs); 48 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 49 | assert.equal(result.data, "unsafe-url, no-referrer, same-origin"); 50 | assert.isTrue(result.http); 51 | assert.isTrue(result.meta); 52 | assert.isTrue(result.pass); 53 | } 54 | }); 55 | 56 | it("checks for missing referrer header", function () { 57 | const result = referrerPolicyTest(reqs); 58 | assert.equal(result.result, Expectation.ReferrerPolicyNotImplemented); 59 | assert.isTrue(result.pass); 60 | }); 61 | 62 | it("checks for invalid referrer header", function () { 63 | assert.isNotNull(reqs.responses.auto); 64 | reqs.responses.auto.headers["referrer-policy"] = "whimsy"; 65 | const result = referrerPolicyTest(reqs); 66 | assert.equal(result.result, Expectation.ReferrerPolicyHeaderInvalid); 67 | assert.isFalse(result.pass); 68 | }); 69 | 70 | it("checks for unsafe referrer header", function () { 71 | const policies = ["origin", "origin-when-cross-origin", "unsafe-url"]; 72 | for (const policy of policies) { 73 | assert.isNotNull(reqs.responses.auto); 74 | reqs.responses.auto.headers["referrer-policy"] = policy; 75 | const result = referrerPolicyTest(reqs); 76 | assert.equal(result.result, Expectation.ReferrerPolicyUnsafe); 77 | assert.isFalse(result.pass); 78 | } 79 | }); 80 | 81 | it("checks for multiple valid referrer headers", function () { 82 | const valid_but_unsafe_policies = [ 83 | "origin-when-cross-origin, no-referrer, unsafe-url", // safe middle value 84 | "no-referrer, unsafe-url", // safe first value 85 | ]; 86 | for (const policy of valid_but_unsafe_policies) { 87 | assert.isNotNull(reqs.responses.auto); 88 | reqs.responses.auto.headers["referrer-policy"] = policy; 89 | const result = referrerPolicyTest(reqs); 90 | assert.equal(result.result, Expectation.ReferrerPolicyUnsafe); 91 | assert.isFalse(result.pass); 92 | } 93 | }); 94 | 95 | it("checks for multiple referrer headers with valid and invalid mixed", function () { 96 | const mixed_valid_invalid_policies = ["no-referrer, whimsy"]; 97 | for (const policy of mixed_valid_invalid_policies) { 98 | assert.isNotNull(reqs.responses.auto); 99 | reqs.responses.auto.headers["referrer-policy"] = policy; 100 | const result = referrerPolicyTest(reqs); 101 | assert.equal(result.result, Expectation.ReferrerPolicyPrivate); 102 | assert.isTrue(result.pass); 103 | } 104 | }); 105 | 106 | it("checks for multiple invalid referrer headers", function () { 107 | const invalid_policies = ["whimsy, whimsy1, whimsy2"]; 108 | for (const policy of invalid_policies) { 109 | assert.isNotNull(reqs.responses.auto); 110 | reqs.responses.auto.headers["referrer-policy"] = policy; 111 | const result = referrerPolicyTest(reqs); 112 | assert.equal(result.result, Expectation.ReferrerPolicyHeaderInvalid); 113 | assert.isFalse(result.pass); 114 | } 115 | }); 116 | }); 117 | -------------------------------------------------------------------------------- /src/site.js: -------------------------------------------------------------------------------- 1 | import { domainToASCII } from "url"; 2 | import { InvalidSiteError } from "./api/errors.js"; 3 | 4 | /** 5 | * A string representing a site that can be: 6 | * - A simple hostname: "example.com" 7 | * - A hostname with port: "example.com:8443" 8 | * - A hostname with path: "example.com/path/to/resource" 9 | * - A hostname with port and path: "example.com:8443/path/to/resource" 10 | * - Combinations of the above 11 | * 12 | * @typedef {string} SiteString 13 | */ 14 | 15 | /** 16 | * Represents a parsed site with hostname, optional port, and optional path components. 17 | * 18 | * This class provides a structured way to work with site information that may include 19 | * different combinations of hostname, port, and path. It can parse site strings in 20 | * various formats and extract the individual components. 21 | * 22 | * @example 23 | * // Create a site with just hostname 24 | * const site1 = new Site("example.com", undefined, undefined); 25 | * 26 | * @example 27 | * // Create a site with hostname and port 28 | * const site2 = new Site("example.com", 8443, undefined); 29 | * 30 | * @example 31 | * // Create a site with hostname and path 32 | * const site3 = new Site("example.com", undefined, "path/to/resource"); 33 | * 34 | * @example 35 | * // Parse from a site string 36 | * const site4 = Site.fromSiteString("example.com:8443/api/v1"); 37 | * console.log(site4.hostname); // "example.com" 38 | * console.log(site4.port); // 8443 39 | * console.log(site4.path); // "api/v1" 40 | */ 41 | export class Site { 42 | /** @type {string} The hostname component of the site */ 43 | hostname; 44 | /** @type {number | undefined} The port number, if specified */ 45 | port; 46 | /** @type {string | undefined} The path component, if specified */ 47 | path; 48 | 49 | /** 50 | * @param {string} hostname 51 | * @param {number | undefined} port 52 | * @param {string} path 53 | */ 54 | constructor(hostname, port, path) { 55 | this.hostname = hostname; 56 | this.port = port; 57 | this.path = path; 58 | } 59 | 60 | /** 61 | * 62 | * @returns a string suitable for the site field in the database 63 | */ 64 | asSiteKey() { 65 | return `${this.hostname}${this.port ? `:${this.port}` : ""}${this.path === "/" ? "" : this.path || ""}`; 66 | } 67 | 68 | /** 69 | * Parses a site string and creates a Site instance. 70 | * 71 | * This method can parse various site string formats: 72 | * - Simple hostname: "example.com" 73 | * - Hostname with port: "example.com:8443" 74 | * - Hostname with path: "example.com/path/to/resource" 75 | * - Hostname with port and path: "example.com:8443/path/to/resource" 76 | * 77 | * @param {string} siteString - The site string to parse 78 | * @returns {Site} A new Site instance with parsed components 79 | * @throws {Error} Throws an error if the site string is invalid or empty 80 | * 81 | * @example 82 | * const site = Site.fromSiteString("api.example.com:3000/v1/users"); 83 | * // Returns: Site { hostname: "api.example.com", port: 3000, path: "v1/users" } 84 | * 85 | * @example 86 | * const site = Site.fromSiteString("localhost:8080"); 87 | * // Returns: Site { hostname: "localhost", port: 8080, path: undefined } 88 | */ 89 | static fromSiteString(siteString) { 90 | try { 91 | // Add a protocol to make it a valid URL for parsing 92 | const url = new URL(`https://${siteString}`); 93 | 94 | const hostname = url.hostname; 95 | if (!hostname) { 96 | throw new InvalidSiteError(hostname, "hostname cannot be empty"); 97 | } 98 | 99 | // 253 bytes is the practical limit for DNS hostnames. 100 | // Take IDN notation into account. 101 | if (domainToASCII(hostname).length > 253) { 102 | throw new InvalidSiteError(hostname, "hostname is too long"); 103 | } 104 | 105 | // Ensure hostname has proper structure: at least one dot 106 | // with non-empty parts. 107 | const parts = hostname.split("."); 108 | if (parts.length < 2 || parts.some((part) => part.length === 0)) { 109 | throw new InvalidSiteError( 110 | hostname, 111 | "hostname must have at least a host and TLD (e.g., site.com)" 112 | ); 113 | } 114 | 115 | // URL.port returns empty string if no port, convert to 116 | // number or undefined. 117 | const port = url.port ? Number(url.port) : undefined; 118 | 119 | // URL.pathname is "/" by default 120 | const path = url.pathname; 121 | 122 | return new Site(hostname.toLowerCase(), port, path); 123 | } catch (error) { 124 | if (error instanceof InvalidSiteError) { 125 | throw error; 126 | } else { 127 | throw new InvalidSiteError(siteString, "invalid site string"); 128 | } 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /test/redirection.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { Expectation } from "../src/types.js"; 4 | import { redirectionTest } from "../src/analyzer/tests/redirection.js"; 5 | 6 | describe("Redirections", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests(); 11 | }); 12 | 13 | it("checks for no http but does have https", function () { 14 | reqs.responses.http = null; 15 | reqs.responses.httpRedirects = []; 16 | const res = redirectionTest(reqs); 17 | assert.equal(res.result, Expectation.RedirectionNotNeededNoHttp); 18 | assert.isTrue(res.pass); 19 | }); 20 | 21 | it("checks for redirection missing", function () { 22 | // the requests object has a single, non-redirecting successful 23 | // http request for this test. 24 | reqs.responses.httpRedirects = [ 25 | { 26 | url: new URL("http://mozilla.org"), 27 | status: 200, 28 | }, 29 | ]; 30 | 31 | const res = redirectionTest(reqs); 32 | 33 | assert.equal(res.result, Expectation.RedirectionMissing); 34 | assert.isFalse(res.pass); 35 | }); 36 | 37 | it("checks for redirection not to https", function () { 38 | // The requests object has only non-https redirects from http 39 | reqs.responses.httpRedirects = [ 40 | { 41 | url: new URL("http://mozilla.org"), 42 | status: 301, 43 | }, 44 | { 45 | url: new URL("http://www.mozilla.org"), 46 | status: 200, 47 | }, 48 | ]; 49 | 50 | const res = redirectionTest(reqs); 51 | assert.equal(res.result, Expectation.RedirectionNotToHttps); 52 | assert.isFalse(res.pass); 53 | 54 | // Longer redirect chains should "work" as well 55 | reqs.responses.httpRedirects = [ 56 | { 57 | url: new URL("http://mozilla.org"), 58 | status: 301, 59 | }, 60 | { 61 | url: new URL("http://www.mozilla.org"), 62 | status: 302, 63 | }, 64 | { 65 | url: new URL("http://www.mozilla.org/en/"), 66 | status: 200, 67 | }, 68 | ]; 69 | 70 | const res2 = redirectionTest(reqs); 71 | assert.equal(res2.result, Expectation.RedirectionNotToHttps); 72 | assert.isFalse(res2.pass); 73 | }); 74 | 75 | it("checks for proper redirection to https", function () { 76 | const res = redirectionTest(reqs); 77 | assert.equal(res.result, Expectation.RedirectionToHttps); 78 | assert.isTrue(res.pass); 79 | }); 80 | 81 | it("checks for proper redirection to https with port number", function () { 82 | reqs.responses.httpRedirects = [ 83 | { 84 | url: new URL("http://mozilla.org/"), 85 | status: 301, 86 | }, 87 | { 88 | url: new URL("https://mozilla.org:8443/"), 89 | status: 200, 90 | }, 91 | ]; 92 | 93 | const res = redirectionTest(reqs); 94 | assert.equal(res.result, Expectation.RedirectionToHttps); 95 | assert.isTrue(res.pass); 96 | assert.deepEqual(res.route, [ 97 | "http://mozilla.org/", 98 | "https://mozilla.org:8443/", 99 | ]); 100 | }); 101 | 102 | it("checks for first redirection to http", function () { 103 | reqs.responses.httpRedirects = [ 104 | { 105 | url: new URL("http://mozilla.org/"), 106 | status: 301, 107 | }, 108 | { 109 | url: new URL("http://www.mozilla.org/"), 110 | status: 301, 111 | }, 112 | { 113 | url: new URL("https://www.mozilla.org/"), 114 | status: 200, 115 | }, 116 | ]; 117 | 118 | const res = redirectionTest(reqs); 119 | assert.equal( 120 | res.result, 121 | Expectation.RedirectionNotToHttpsOnInitialRedirection 122 | ); 123 | assert.isFalse(res.pass); 124 | }); 125 | 126 | it("checks for first redirection off host", function () { 127 | reqs.responses.httpRedirects = [ 128 | { 129 | url: new URL("http://mozilla.org/"), 130 | status: 301, 131 | }, 132 | { 133 | url: new URL("https://www.mozilla.org/"), 134 | status: 200, 135 | }, 136 | ]; 137 | 138 | const res = redirectionTest(reqs); 139 | assert.equal(res.result, Expectation.RedirectionOffHostFromHttp); 140 | assert.isFalse(res.pass); 141 | }); 142 | 143 | it("checks for all redirections preloaded", function () { 144 | reqs.responses.httpRedirects = [ 145 | { 146 | url: new URL("http://pokeinthe.io/"), 147 | status: 301, 148 | }, 149 | { 150 | url: new URL("https://pokeinthe.io/"), 151 | status: 302, 152 | }, 153 | { 154 | url: new URL("https://www.pokeinthe.io/"), 155 | status: 302, 156 | }, 157 | { 158 | url: new URL("https://baz.pokeinthe.io/foo"), 159 | status: 200, 160 | }, 161 | ]; 162 | 163 | const res = redirectionTest(reqs); 164 | assert.equal(res.result, Expectation.RedirectionAllRedirectsPreloaded); 165 | assert.isTrue(res.pass); 166 | }); 167 | }); 168 | -------------------------------------------------------------------------------- /test/subresource-integrity.test.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import { emptyRequests } from "./helpers.js"; 3 | import { subresourceIntegrityTest } from "../src/analyzer/tests/subresource-integrity.js"; 4 | import { Expectation } from "../src/types.js"; 5 | 6 | describe("Subresource Integrity", () => { 7 | /** @type {import("../src/types.js").Requests} */ 8 | let reqs; 9 | beforeEach(() => { 10 | reqs = emptyRequests("test_content_sri_no_scripts.html"); 11 | }); 12 | 13 | it("checks for no scripts", function () { 14 | reqs = emptyRequests("test_content_sri_no_scripts.html"); 15 | const result = subresourceIntegrityTest(reqs); 16 | assert.equal(result.result, "sri-not-implemented-but-no-scripts-loaded"); 17 | }); 18 | 19 | it("checks for not html", function () { 20 | reqs.resources.path = `{"foo": "bar"}`; 21 | assert.isNotNull(reqs.responses.auto); 22 | reqs.responses.auto.headers["content-type"] = "application/json"; 23 | const result = subresourceIntegrityTest(reqs); 24 | assert.equal(result.result, Expectation.SriNotImplementedResponseNotHtml); 25 | assert.isTrue(result.pass); 26 | }); 27 | 28 | it("checks for same origin", function () { 29 | reqs = emptyRequests("test_content_sri_sameorigin1.html"); 30 | let result = subresourceIntegrityTest(reqs); 31 | assert.equal( 32 | result.result, 33 | Expectation.SriNotImplementedButAllScriptsLoadedFromSecureOrigin 34 | ); 35 | assert.isTrue(result.pass); 36 | 37 | // On the same second-level domain, but without a protocol 38 | reqs = emptyRequests("test_content_sri_sameorigin3.html"); 39 | result = subresourceIntegrityTest(reqs); 40 | assert.equal( 41 | result.result, 42 | Expectation.SriNotImplementedAndExternalScriptsNotLoadedSecurely 43 | ); 44 | assert.isFalse(result.pass); 45 | 46 | // On the same second-level domain, but with https:// specified 47 | reqs = emptyRequests("test_content_sri_sameorigin2.html"); 48 | result = subresourceIntegrityTest(reqs); 49 | assert.equal( 50 | result.result, 51 | Expectation.SriNotImplementedButAllScriptsLoadedFromSecureOrigin 52 | ); 53 | assert.isTrue(result.pass); 54 | 55 | // And the same, but with a 404 status code 56 | assert.isNotNull(reqs.responses.auto); 57 | reqs.responses.auto.status = 404; 58 | result = subresourceIntegrityTest(reqs); 59 | assert.equal( 60 | result.result, 61 | Expectation.SriNotImplementedButAllScriptsLoadedFromSecureOrigin 62 | ); 63 | assert.isTrue(result.pass); 64 | }); 65 | 66 | it("checks if implemented with external scripts and https", function () { 67 | // load from a remote site 68 | reqs = emptyRequests("test_content_sri_impl_external_https1.html"); 69 | let result = subresourceIntegrityTest(reqs); 70 | assert.equal( 71 | result.result, 72 | Expectation.SriImplementedAndExternalScriptsLoadedSecurely 73 | ); 74 | assert.isTrue(result.pass); 75 | 76 | // load from an intranet / localhost 77 | reqs = emptyRequests("test_content_sri_impl_external_https2.html"); 78 | result = subresourceIntegrityTest(reqs); 79 | assert.equal( 80 | result.result, 81 | Expectation.SriImplementedAndExternalScriptsLoadedSecurely 82 | ); 83 | assert.isTrue(result.pass); 84 | }); 85 | 86 | it("checks if implemented with same origin", function () { 87 | reqs = emptyRequests("test_content_sri_impl_sameorigin.html"); 88 | let result = subresourceIntegrityTest(reqs); 89 | assert.equal( 90 | result.result, 91 | Expectation.SriImplementedAndAllScriptsLoadedSecurely 92 | ); 93 | assert.isTrue(result.pass); 94 | }); 95 | 96 | it("checks if not implemented with external scripts and https", function () { 97 | reqs = emptyRequests("test_content_sri_notimpl_external_https.html"); 98 | let result = subresourceIntegrityTest(reqs); 99 | assert.equal( 100 | result.result, 101 | Expectation.SriNotImplementedButExternalScriptsLoadedSecurely 102 | ); 103 | assert.isFalse(result.pass); 104 | }); 105 | 106 | it("checks if implemented with external scripts and http", function () { 107 | reqs = emptyRequests("test_content_sri_impl_external_http.html"); 108 | let result = subresourceIntegrityTest(reqs); 109 | assert.equal( 110 | result.result, 111 | Expectation.SriImplementedButExternalScriptsNotLoadedSecurely 112 | ); 113 | assert.isFalse(result.pass); 114 | }); 115 | 116 | it("checks if implemented with external scripts and no protocol", function () { 117 | reqs = emptyRequests("test_content_sri_impl_external_noproto.html"); 118 | let result = subresourceIntegrityTest(reqs); 119 | assert.equal( 120 | result.result, 121 | Expectation.SriImplementedButExternalScriptsNotLoadedSecurely 122 | ); 123 | assert.isFalse(result.pass); 124 | }); 125 | 126 | it("checks if not implemented with external scripts and http", function () { 127 | reqs = emptyRequests("test_content_sri_notimpl_external_http.html"); 128 | let result = subresourceIntegrityTest(reqs); 129 | assert.equal( 130 | result.result, 131 | Expectation.SriNotImplementedAndExternalScriptsNotLoadedSecurely 132 | ); 133 | assert.isFalse(result.pass); 134 | }); 135 | 136 | it("checks if not implemented with external scripts and no protocol", function () { 137 | reqs = emptyRequests("test_content_sri_notimpl_external_noproto.html"); 138 | let result = subresourceIntegrityTest(reqs); 139 | assert.equal( 140 | result.result, 141 | Expectation.SriNotImplementedAndExternalScriptsNotLoadedSecurely 142 | ); 143 | assert.isFalse(result.pass); 144 | }); 145 | }); 146 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution guide 2 | 3 | ![github-profile](https://user-images.githubusercontent.com/10350960/166113119-629295f6-c282-42c9-9379-af2de5ad4338.png) 4 | 5 | - [Ways to contribute](#ways-to-contribute) 6 | - [Finding an issue](#finding-an-issue) 7 | - [Asking for help](#asking-for-help) 8 | - [Pull request process](#pull-request-process) 9 | - [Setting up the development environment](#setting-up-the-development-environment) 10 | - [Forking and cloning the project](#forking-and-cloning-the-project) 11 | - [Prerequisites](#prerequisites) 12 | - [Signing commits](#signing-commits) 13 | 14 | Welcome 👋 Thank you for your interest in contributing to MDN Web Docs. We are happy to have you join us! 💖 15 | 16 | As you get started, you are in the best position to give us feedback on project areas we might have forgotten about or assumed to work well. 17 | These include, but are not limited to: 18 | 19 | - Problems found while setting up a new developer environment 20 | - Gaps in our documentation 21 | - Bugs in our automation scripts 22 | 23 | If anything doesn't make sense or work as expected, please open an issue and let us know! 24 | 25 | ## Ways to contribute 26 | 27 | We welcome many different types of contributions including: 28 | 29 | 30 | 31 | - New features and content suggestions. 32 | - Identifying and filing issues. 33 | - Providing feedback on existing issues. 34 | - Engaging with the community and answering questions. 35 | - Contributing documentation or code. 36 | - Promoting the project in personal circles and social media. 37 | 38 | ## Finding an issue 39 | 40 | We have issues labeled `good first issue` for new contributors and `help wanted` suitable for any contributor. 41 | Good first issues have extra information to help you make your first contribution a success. 42 | Help wanted issues are ideal when you feel a bit more comfortable with the project details. 43 | 44 | Sometimes there won't be any issues with these labels, but there is likely still something for you to work on. 45 | If you want to contribute but don't know where to start or can't find a suitable issue, speak to us on [Matrix](https://matrix.to/#/#mdn:mozilla.org), and we will be happy to help. 46 | 47 | Once you find an issue you'd like to work on, please post a comment saying you want to work on it. 48 | Something like "I want to work on this" is fine. 49 | Also, mention the community team using the `@mdn/community` handle to ensure someone will get back to you. 50 | 51 | ## Asking for help 52 | 53 | The best way to reach us with a question when contributing is to use the following channels in the following order of precedence: 54 | 55 | - [Start a discussion](https://github.com/orgs/mdn/discussions) 56 | - Ask your question or highlight your discussion on [Matrix](https://matrix.to/#/#mdn:mozilla.org). 57 | - File an issue and tag the community team using the `@mdn/community` handle. 58 | 59 | ## Pull request process 60 | 61 | The MDN Web Docs project has a well-defined pull request process which is documented in the [Pull request guidelines](https://developer.mozilla.org/en-US/docs/MDN/Community/Pull_requests). 62 | Make sure you read and understand this process before you start working on a pull request. 63 | 64 | 69 | 70 | ## Setting up the development environment 71 | 72 | 75 | 76 | ### Forking and cloning the project 77 | 78 | The first step in setting up your development environment is to [fork the repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo) and [clone](https://docs.github.com/en/get-started/quickstart/fork-a-repo#cloning-your-forked-repository) the repository to your local machine. 79 | 80 | ### Prerequisites 81 | 82 | 92 | 93 | ### Building the project 94 | 95 | 120 | 121 | ## Signing commits 122 | 123 | We require all commits to be signed to verify the author's identity. 124 | GitHub has a detailed guide on [setting up signed commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits). 125 | If you get stuck, please [ask for help](#asking-for-help). 126 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Welcome to Mozilla's HTTP Observatory 2 | 3 | [HTTP Observatory](https://developer.mozilla.org/en-US/observatory/) is a service that checks web sites for security-relevant headers. It is hosted by [MDN Web Docs](https://github.com/mdn). 4 | 5 | ## Getting Started 6 | 7 | If you just want to scan a host, please head over to . If you want to 8 | run the code locally or on your premises, continue reading. 9 | 10 | ### Running a simple scan from the command line 11 | 12 | Using npx to install the package, simply run 13 | 14 | ```sh 15 | npx @mdn/mdn-http-observatory mdn.dev 16 | ``` 17 | 18 | Subpaths like `example.com/path` and port numbers like `example.com:8080/path` are suported. 19 | 20 | If you want to install the package first, use npm to install it globally 21 | 22 | ```sh 23 | npm install --global @mdn/mdn-http-observatory 24 | ``` 25 | 26 | After that, the `mdn-http-observatory-scan` command should be available in your shell. To scan a host, run 27 | 28 | ```sh 29 | mdn-http-observatory-scan mdn.dev 30 | ``` 31 | 32 | Both methods return a JSON response of the following form: 33 | 34 | ```json 35 | { 36 | "scan": { 37 | "algorithmVersion": 4, 38 | "grade": "A+", 39 | "error": null, 40 | "score": 105, 41 | "statusCode": 200, 42 | "testsFailed": 0, 43 | "testsPassed": 10, 44 | "testsQuantity": 10, 45 | "responseHeaders": { 46 | ... 47 | } 48 | }, 49 | "tests": { 50 | "cross-origin-resource-sharing": { 51 | "expectation": "cross-origin-resource-sharing-not-implemented", 52 | "pass": true, 53 | "result": "cross-origin-resource-sharing-not-implemented", 54 | "scoreModifier": 0, 55 | "data": null 56 | }, 57 | ... 58 | } 59 | } 60 | ``` 61 | 62 | ### Running a local API server 63 | 64 | This needs a [postgres](https://www.postgresql.org/) database for the API to use as a persistence layer. All scans and results initiated via the API are stored in the database. 65 | 66 | #### Configuration 67 | 68 | Default configuration is read from a default `config/config.json` file. See [this file](src/config.js) for a list of possible configuration options. 69 | 70 | Create a configuration file by copying the [`config/config-example.json`](conf/config-example.json) to `config/config.json`. 71 | Put in your database credentials into `config/config.json`: 72 | 73 | ```json 74 | { 75 | "database": { 76 | "database": "observatory", 77 | "user": "postgres" 78 | } 79 | } 80 | ``` 81 | 82 | To initialize the database with the proper tables, use this command to migrate. This is a one-time action, but future code changes 83 | might need further database changes, so run this migration every time the code is updated from the repository. 84 | 85 | ```sh 86 | npm run migrate 87 | ``` 88 | 89 | Finally, start the server by running 90 | 91 | ```sh 92 | npm start 93 | ``` 94 | 95 | The server is listening on your local interface on port `8080`. You can check the root path by opening in your browser or `curl` the URL. The server should respond with `Welcome to the MDN Observatory!`. 96 | 97 | ## JSON API 98 | 99 | **Note:** We provide these endpoints on our public deployment of HTTP Observatory at 100 | 101 | ### POST `/api/v2/scan` 102 | 103 | For integration in CI pipelines or similar applications, a JSON API endpoint is provided. The request rate is limited to one scan per host per `api.cooldown` (default: One minute) seconds. If exceeded, a cached result will be returned. 104 | 105 | #### Query parameters 106 | 107 | - `host` hostname (required) 108 | 109 | #### Examples 110 | 111 | - `POST /api/v2/scan?host=mdn.dev` 112 | - `POST /api/v2/scan?host=google.com` 113 | 114 | #### Result 115 | 116 | On success, a JSON object is returned, structured like this example response: 117 | 118 | ```json 119 | { 120 | "id": 77666718, 121 | "details_url": "https://developer.mozilla.org/en-US/observatory/analyze?host=mdn.dev", 122 | "algorithm_version": 4, 123 | "scanned_at": "2024-08-12T08:20:18.926Z", 124 | "error": null, 125 | "grade": "A+", 126 | "score": 105, 127 | "status_code": 200, 128 | "tests_failed": 0, 129 | "tests_passed": 10, 130 | "tests_quantity": 10 131 | } 132 | ``` 133 | 134 | **Note:** For a full set of details about the host, use the provided link in the `details_url` field. 135 | 136 | If an error occurred, an object like this is returned: 137 | 138 | ```json 139 | { 140 | "error": "invalid-hostname-lookup", 141 | "message": "some.invalid.hostname.dev cannot be resolved" 142 | } 143 | ``` 144 | 145 | ## Migrating from the public V1 API to the V2 API 146 | 147 | ### Sunset of the V1 API 148 | 149 | The previous iteration of the Observatory JSON API has been deprecated and shut down on October 31, 2024. 150 | 151 | ### Migrating your application 152 | 153 | If you previously used the Observatory API with some automation or a CI context, the switch from the old `/api/v1/analyze` endpoint to the new `/api/v2/scan` endpoint should be painless: 154 | 155 | - Replace all API calls to `POST https://http-observatory.security.mozilla.org/api/v1/analyze?host=` with `POST https://observatory-api.mdn.mozilla.net/api/v2/scan?host=` 156 | - Be aware that the complete list of headers has been removed from the response. 157 | - The POST parameters `rescan` and `hidden` in the POST body have been removed. 158 | - Remove all other requests from your application, if any. If you need any additional information about your scan, open the URL from the `detail_url` field of the response in your browser. 159 | - Note that scans are still limited to one every minute per host, otherwise a cached response is returned. 160 | 161 | ## Contributing 162 | 163 | Our project welcomes contributions from any member of our community. 164 | To get started contributing, please see our [Contributor Guide](CONTRIBUTING.md). 165 | 166 | By participating in and contributing to our projects and discussions, you acknowledge that you have read and agree to our [Code of Conduct](CODE_OF_CONDUCT.md). 167 | 168 | ## Communications 169 | 170 | If you have any questions, please reach out to us on [Mozilla Developer Network](https://developer.mozilla.org). 171 | 172 | ## License 173 | 174 | This project is licensed under the [Mozilla Public License 2.0](LICENSE). 175 | -------------------------------------------------------------------------------- /src/analyzer/tests/subresource-integrity.js: -------------------------------------------------------------------------------- 1 | import { BaseOutput, HTML_TYPES, Requests } from "../../types.js"; 2 | import { Expectation } from "../../types.js"; 3 | import { JSDOM } from "jsdom"; 4 | import { parse } from "tldts"; 5 | import { getFirstHttpHeader, onlyIfWorse } from "../utils.js"; 6 | import { CONTENT_TYPE } from "../../headers.js"; 7 | 8 | export class SubresourceIntegrityOutput extends BaseOutput { 9 | /** @type {import("../../types.js").ScriptMap} */ 10 | data; 11 | static name = "subresource-integrity"; 12 | static title = "Subresource Integrity"; 13 | static possibleResults = [ 14 | Expectation.SriImplementedAndAllScriptsLoadedSecurely, 15 | Expectation.SriImplementedAndExternalScriptsLoadedSecurely, 16 | Expectation.SriNotImplementedResponseNotHtml, 17 | Expectation.SriNotImplementedButNoScriptsLoaded, 18 | Expectation.SriNotImplementedButAllScriptsLoadedFromSecureOrigin, 19 | Expectation.SriNotImplementedButExternalScriptsLoadedSecurely, 20 | Expectation.SriImplementedButExternalScriptsNotLoadedSecurely, 21 | Expectation.SriNotImplementedAndExternalScriptsNotLoadedSecurely, 22 | ]; 23 | 24 | /** 25 | * 26 | * @param {Expectation} expectation 27 | */ 28 | constructor(expectation) { 29 | super(expectation); 30 | this.data = {}; 31 | } 32 | } 33 | 34 | /** 35 | * 36 | * @param {Requests} requests 37 | * @param {Expectation} expectation 38 | * @returns {SubresourceIntegrityOutput} 39 | */ 40 | export function subresourceIntegrityTest( 41 | requests, 42 | expectation = Expectation.SriImplementedAndExternalScriptsLoadedSecurely 43 | ) { 44 | const output = new SubresourceIntegrityOutput(expectation); 45 | const goodness = [ 46 | Expectation.SriImplementedAndAllScriptsLoadedSecurely, 47 | Expectation.SriImplementedAndExternalScriptsLoadedSecurely, 48 | Expectation.SriImplementedButExternalScriptsNotLoadedSecurely, 49 | Expectation.SriNotImplementedButExternalScriptsLoadedSecurely, 50 | Expectation.SriNotImplementedAndExternalScriptsNotLoadedSecurely, 51 | Expectation.SriNotImplementedResponseNotHtml, 52 | ]; 53 | 54 | const resp = requests.responses.auto; 55 | 56 | if (!resp) { 57 | output.result = Expectation.SriNotImplementedButNoScriptsLoaded; 58 | return output; 59 | } 60 | 61 | const mime = (getFirstHttpHeader(resp, CONTENT_TYPE) ?? "").split(";")[0]; 62 | if (mime && !HTML_TYPES.has(mime)) { 63 | // If the content isn't HTML, there's no scripts to load; this is okay 64 | output.result = Expectation.SriNotImplementedResponseNotHtml; 65 | } else { 66 | // Try to parse the HTML 67 | let dom; 68 | try { 69 | dom = new JSDOM(requests.resources.path || ""); 70 | } catch (e) { 71 | // severe parser error 72 | output.result = Expectation.HtmlNotParseable; 73 | return output; 74 | } 75 | // Track to see if any scripts were on foreign TLDs. 76 | let scriptsOnForeignOrigin = false; 77 | const scripts = dom.window.document.querySelectorAll("script"); 78 | for (const script of scripts) { 79 | if (script.src) { 80 | const src = parse(script.src); 81 | const integrity = script.getAttribute("integrity"); 82 | const crossorigin = script.crossOrigin; 83 | 84 | let relativeOrigin = false; 85 | let relativeProtocol = false; 86 | let sameSecondLevelDomain = false; 87 | 88 | const relativeProtocolRegex = /^(\/\/)[^\/]/; 89 | const fullUrlRegex = /^https?:\/\//; 90 | 91 | if (relativeProtocolRegex.test(script.src)) { 92 | // relative protocol(src="//example.com/script.js") 93 | relativeProtocol = true; 94 | sameSecondLevelDomain = true; 95 | } else if (fullUrlRegex.test(script.src)) { 96 | // full URL (src="https://example.com/script.js") 97 | sameSecondLevelDomain = 98 | src.domain === parse(requests.site.hostname).domain; 99 | } else { 100 | // relative URL (src="/path" etc.) 101 | relativeOrigin = true; 102 | sameSecondLevelDomain = true; 103 | } 104 | 105 | // Check to see if it is the same origin or second level domain 106 | let secureOrigin = false; 107 | if (relativeOrigin || (sameSecondLevelDomain && !relativeProtocol)) { 108 | secureOrigin = true; 109 | } else { 110 | secureOrigin = false; 111 | scriptsOnForeignOrigin = true; 112 | } 113 | 114 | // Check if it is a secure scheme 115 | let scheme = null; 116 | if (!relativeProtocol && !relativeOrigin) { 117 | scheme = new URL(script.src).protocol; 118 | } 119 | let secureScheme = false; 120 | if ( 121 | scheme === "https:" || 122 | (relativeOrigin && requests.session?.url.protocol === "https:") 123 | ) { 124 | secureScheme = true; 125 | } 126 | 127 | // Add it to the scripts data result, if it's not a relative URI 128 | if (!secureOrigin) { 129 | output.data[script.src] = { crossorigin, integrity }; 130 | 131 | if (integrity && !secureScheme) { 132 | output.result = onlyIfWorse( 133 | Expectation.SriImplementedButExternalScriptsNotLoadedSecurely, 134 | output.result, 135 | goodness 136 | ); 137 | } else if (!integrity && secureScheme) { 138 | output.result = onlyIfWorse( 139 | Expectation.SriNotImplementedButExternalScriptsLoadedSecurely, 140 | output.result, 141 | goodness 142 | ); 143 | } else if (!integrity && !secureScheme && sameSecondLevelDomain) { 144 | output.result = onlyIfWorse( 145 | Expectation.SriNotImplementedAndExternalScriptsNotLoadedSecurely, 146 | output.result, 147 | goodness 148 | ); 149 | } else if (!integrity && !secureScheme) { 150 | output.result = onlyIfWorse( 151 | Expectation.SriNotImplementedAndExternalScriptsNotLoadedSecurely, 152 | output.result, 153 | goodness 154 | ); 155 | } 156 | } else { 157 | // Grant bonus even if they use SRI on the same origin 158 | if (integrity && secureScheme && !output.result) { 159 | output.result = 160 | Expectation.SriImplementedAndAllScriptsLoadedSecurely; 161 | } 162 | } 163 | } 164 | } 165 | 166 | if (scripts.length === 0) { 167 | output.result = Expectation.SriNotImplementedButNoScriptsLoaded; 168 | } else { 169 | if (!output.result) { 170 | if (scriptsOnForeignOrigin) { 171 | output.result = 172 | Expectation.SriImplementedAndExternalScriptsLoadedSecurely; 173 | } else { 174 | output.result = 175 | Expectation.SriNotImplementedButAllScriptsLoadedFromSecureOrigin; 176 | } 177 | } 178 | } 179 | } 180 | 181 | // Code defensively on the size of the data 182 | output.data = JSON.stringify(output.data).length < 32768 ? output.data : {}; 183 | // Check to see if the test passed or failed 184 | if ( 185 | [ 186 | Expectation.SriImplementedAndAllScriptsLoadedSecurely, 187 | Expectation.SriImplementedAndExternalScriptsLoadedSecurely, 188 | Expectation.SriNotImplementedResponseNotHtml, 189 | Expectation.SriNotImplementedButAllScriptsLoadedFromSecureOrigin, 190 | Expectation.SriNotImplementedButNoScriptsLoaded, 191 | expectation, 192 | ].includes(output.result) 193 | ) { 194 | output.pass = true; 195 | } 196 | return output; 197 | } 198 | -------------------------------------------------------------------------------- /src/analyzer/cspParser.js: -------------------------------------------------------------------------------- 1 | const compareString = Intl.Collator("en").compare; 2 | 3 | const SHORTEST_DIRECTIVE = "img-src"; 4 | const SHORTEST_DIRECTIVE_LENGTH = SHORTEST_DIRECTIVE.length - 1; // the shortest policy accepted by the CSP test 5 | const DIRECTIVES_DISALLOWED_IN_META = [ 6 | "frame-ancestors", 7 | "report-uri", 8 | "sandbox", 9 | ]; 10 | const ALLOWED_DUPLICATE_KEYS = new Set(["report-uri", "report-to"]); 11 | export const DUPLICATE_WARNINGS_KEY = "_observatory_duplicate_key_warnings"; 12 | 13 | /** 14 | * Parse CSP from meta tags, weeding out directives 15 | * only allowed in headers. 16 | * See https://html.spec.whatwg.org/#attr-meta-http-equiv-content-security-policy 17 | * @param {string[]} cspList 18 | * @returns {Map>} 19 | */ 20 | export function parseCspMeta(cspList) { 21 | const ret = parseCsp(cspList); 22 | for (const directive of DIRECTIVES_DISALLOWED_IN_META) { 23 | ret.delete(directive); 24 | } 25 | return ret; 26 | } 27 | 28 | /** 29 | * The returned Map has the directive as the key and a Set of sources as the value. 30 | * If there are allowed duplicates detected, the first one is kept and the rest are discarded, 31 | * and an entry in the final Map is added with the key "_observatory_duplicate_key_warnings" 32 | * and the directive's name as the value. 33 | * 34 | * @param {string[]} cspList 35 | * @returns {Map>} 36 | */ 37 | export function parseCsp(cspList) { 38 | const cleanCspList = cspList.map((cspString) => 39 | cspString.replaceAll(/[\r\n]/g, "").trim() 40 | ); 41 | if (cleanCspList.length === 0) { 42 | return new Map(); 43 | } 44 | 45 | for (const cspString of cleanCspList) { 46 | if (!cspString || cspString.length < SHORTEST_DIRECTIVE_LENGTH) { 47 | throw new Error(`Invalid policy: ${cspString}`); 48 | } 49 | } 50 | 51 | /** @type {Map} */ 52 | const csp = new Map(); 53 | /** @type {Set} */ 54 | const duplicate_warnings = new Set(); 55 | 56 | for (const [policyIndex, policy] of cleanCspList.entries()) { 57 | const directiveSeenBeforeThisPolicy = new Set(); 58 | 59 | // since we can have multiple policies, we need to iterate through each policy 60 | for (const [directiveEntry, ...valueEntries] of policy 61 | .split(";") 62 | .map((entry) => entry.trim().split(/\s+/))) { 63 | if (!directiveEntry) { 64 | continue; 65 | } 66 | // Using lower due to directives being case insensitive after CSP3 67 | const directive = directiveEntry.toLowerCase(); 68 | 69 | // While technically valid in that you just use the first entry, we are saying that repeated 70 | // directives are invalid so that people notice it. The exception are duplicate report-uri 71 | // and report-to directives, which we allow. 72 | if (directiveSeenBeforeThisPolicy.has(directive)) { 73 | if (ALLOWED_DUPLICATE_KEYS.has(directive)) { 74 | duplicate_warnings.add(directive); 75 | } else { 76 | throw new Error( 77 | `Duplicate directive ${directive} in policy ${policyIndex}` 78 | ); 79 | } 80 | } else { 81 | directiveSeenBeforeThisPolicy.add(directive); 82 | } 83 | 84 | const values = []; 85 | const keep = policyIndex === 0; 86 | if (valueEntries.length) { 87 | values.push( 88 | ...valueEntries.map((rawSource) => { 89 | const source = rawSource.trim().toLocaleLowerCase(); 90 | return { 91 | source, 92 | index: policyIndex, 93 | keep, 94 | }; 95 | }) 96 | ); 97 | } else if (valueEntries.length === 0 && directive.endsWith("-src")) { 98 | // if it's a source list with no values, it's 'none' 99 | values.push({ 100 | source: "'none'", 101 | index: policyIndex, 102 | keep, 103 | }); 104 | } 105 | const combinedSources = 106 | policyIndex === 0 107 | ? [...values] 108 | : [...(csp.get(directive) || []), ...values]; 109 | combinedSources.sort((a, b) => compareString(a.source, b.source)); 110 | 111 | if (combinedSources.length > 1) { 112 | for (let index = 1; index < combinedSources.length; index++) { 113 | const source = combinedSources[index]; 114 | // convenience variable pointing to previous entry in the combined list 115 | const prev = combinedSources[index - 1]; 116 | 117 | // Skip if either source or prev is undefined 118 | if (!source || !prev) { 119 | continue; 120 | } 121 | 122 | // if it's from the same policy and they start with the same thing, the longer one is 123 | // superfluous, e.g. https://example.com/foo and https://example.com/foobar 124 | if ( 125 | source.index === prev.index && 126 | source.source.startsWith(prev.source) 127 | ) { 128 | source.keep = false; 129 | } 130 | 131 | // a source _has_ to exist in both policies for it to count 132 | if ( 133 | source.index !== prev.index && 134 | pathPartMatch(prev.source, source.source) 135 | ) { 136 | source.keep = true; 137 | } 138 | } 139 | } 140 | // now we need to purge anything that's not necessary and store it into the policy 141 | csp.set( 142 | directive, 143 | combinedSources.filter((source) => source.keep) 144 | ); 145 | 146 | // the first time through the loop is special case -- everything is marked as True to keep, 147 | // and only purged if it has a shorter match. however, if we are going to have more loops through 148 | // due to having multiple CSP policies, then everything needs to be marked False to keep and 149 | // then forcibly kept in future loops 150 | if (policyIndex === 0 && cspList.length > 1) { 151 | for (const source of csp.get(directive) || []) { 152 | source.keep = false; 153 | } 154 | } 155 | } 156 | } 157 | // now we need to flatten out all the CSP directives (e.g. (source, index, False) back into actual values 158 | // if they had defined a directive and didn't have a value remaining, then force it to none 159 | const finalCsp = new Map( 160 | [...csp.entries()].map(([directive, sources]) => [ 161 | directive, 162 | sources.length 163 | ? new Set([...sources.values()].map((source) => source.source)) 164 | : new Set(["'none'"]), 165 | ]) 166 | ); 167 | if (duplicate_warnings.size) { 168 | finalCsp.set(DUPLICATE_WARNINGS_KEY, duplicate_warnings); 169 | } 170 | return finalCsp; 171 | } 172 | 173 | /** 174 | * 175 | * @param {string} pathA 176 | * @param {string} pathB 177 | * @returns 178 | */ 179 | function pathPartMatch(pathA, pathB) { 180 | if (pathA.length === 0) { 181 | return true; 182 | } 183 | if (pathA === "/" && pathB.length === 0) { 184 | return true; 185 | } 186 | const exactMatch = !pathA.endsWith("/"); 187 | const pathListA = pathA.split("/"); 188 | const pathListB = pathB.split("/"); 189 | if (pathListA.length > pathListB.length) { 190 | return false; 191 | } 192 | if (exactMatch && pathListA.length !== pathListB.length) { 193 | return false; 194 | } 195 | if (!exactMatch) { 196 | pathListA.pop(); 197 | } 198 | for (let i = 0; i < pathListA.length; i++) { 199 | const pathAElement = pathListA[i]; 200 | const pathBElement = pathListB[i]; 201 | if (pathAElement === undefined || pathBElement === undefined) { 202 | return false; 203 | } 204 | 205 | const pieceA = decodeURIComponent(pathAElement); 206 | const pieceB = decodeURIComponent(pathBElement); 207 | if (pieceA !== pieceB) { 208 | return false; 209 | } 210 | } 211 | return true; 212 | } 213 | -------------------------------------------------------------------------------- /src/api/v2/utils.js: -------------------------------------------------------------------------------- 1 | import ip from "ip"; 2 | import dns from "node:dns"; 3 | import fs from "fs"; 4 | import { fileURLToPath } from "node:url"; 5 | import path from "node:path"; 6 | import { 7 | InvalidHostNameError, 8 | InvalidHostNameIpError, 9 | InvalidHostNameLookupError, 10 | ScanFailedError, 11 | } from "../errors.js"; 12 | import { 13 | ensureSite, 14 | insertScan, 15 | insertTestResults, 16 | ScanState, 17 | selectScanHostHistory, 18 | selectTestResults, 19 | updateScanState, 20 | } from "../../database/repository.js"; 21 | import { 22 | getRecommendation, 23 | getScoreDescription, 24 | getTopicLink, 25 | } from "../../grader/grader.js"; 26 | import { snakeCase } from "change-case"; 27 | import { PolicyResponse } from "./schemas.js"; 28 | import { Expectation } from "../../types.js"; 29 | import { TEST_TITLES } from "../../grader/charts.js"; 30 | import { scan } from "../../scanner/index.js"; 31 | 32 | /** 33 | * 34 | * @param {string} hostname 35 | * @returns {boolean} 36 | */ 37 | export function isIp(hostname) { 38 | if (ip.isV4Format(hostname)) return true; 39 | if (ip.isV6Format(hostname)) return true; 40 | return false; 41 | } 42 | 43 | /** 44 | * @typedef {Object} ValidHostnameResult 45 | * @property {string} [hostname] 46 | * @property {boolean} [isIpAddress] 47 | */ 48 | 49 | /** 50 | * @type {Set | null} 51 | */ 52 | let tldSet = null; 53 | const dirname = path.dirname(fileURLToPath(import.meta.url)); 54 | 55 | /** 56 | * Get the cached set of top level domains. 57 | * @returns {Set} 58 | */ 59 | function tlds() { 60 | if (!tldSet) { 61 | const filePath = path.join( 62 | dirname, 63 | "..", 64 | "..", 65 | "..", 66 | "conf", 67 | "tld-list.json" 68 | ); 69 | tldSet = new Set(JSON.parse(fs.readFileSync(filePath, "utf8"))); 70 | } 71 | return tldSet; 72 | } 73 | 74 | /** 75 | * 76 | * @param {string} hostname 77 | * @returns {Promise} - The valid hostname, maybe prefixed with 'www.' 78 | * @throws {InvalidHostNameIpError | InvalidHostNameError} 79 | */ 80 | export async function validHostname(hostname) { 81 | // remove any trailing dot 82 | hostname = hostname.replace(/\.$/, ""); 83 | const tld = hostname.split(".").pop()?.toLowerCase(); 84 | if ( 85 | !hostname || 86 | !hostname.includes(".") || 87 | !tlds().has(tld || "") || 88 | hostname === "" 89 | ) { 90 | throw new InvalidHostNameError(); 91 | } 92 | 93 | // Check if we can look up the host 94 | await /** @type {Promise} */ ( 95 | new Promise((resolve, reject) => { 96 | dns.lookup(hostname, (err, _address, _family) => { 97 | if (err) { 98 | reject(new InvalidHostNameLookupError(hostname)); 99 | } 100 | resolve(); 101 | }); 102 | }) 103 | ); 104 | 105 | return hostname; 106 | } 107 | 108 | /** 109 | * 110 | * @param {import("../../site.js").Site} site 111 | * @returns {Promise} 112 | */ 113 | export async function checkSitename(site) { 114 | // first, divide the site string into its components: hostname, port (optional) and path (optional) 115 | // then lowercase the hostname. 116 | // look up the hostname 117 | // if lookup fails, try again with "www." prefix 118 | // finally, return the sanitized site string. 119 | 120 | if (isIp(site.hostname)) { 121 | throw new InvalidHostNameIpError(); 122 | } 123 | 124 | // Try prefixing with `www.` if it fails on first try 125 | try { 126 | site.hostname = await validHostname(site.hostname); 127 | } catch (e) { 128 | if (e instanceof InvalidHostNameLookupError) { 129 | site.hostname = await validHostname(`www.${site.hostname}`); 130 | } else { 131 | throw e; 132 | } 133 | } 134 | return site; 135 | } 136 | 137 | /** 138 | * @typedef {import("pg").Pool} Pool 139 | */ 140 | 141 | /** 142 | * Return API-formatted test results for a single scan. 143 | * @param {number} scanId 144 | * @param {Pool} pool 145 | */ 146 | export async function testsForScan(pool, scanId) { 147 | const testRows = await selectTestResults(pool, scanId); 148 | const tests = testRows.reduce((acc, test) => { 149 | /** @type {any} */ 150 | const value = { 151 | expectation: test.expectation, 152 | name: test.name, 153 | link: getTopicLink(test.name), 154 | title: getTitle(test.name), 155 | pass: test.pass, 156 | result: test.result, 157 | score_description: getScoreDescription(test.result), 158 | recommendation: getRecommendation(test.result), 159 | score_modifier: test.score_modifier, 160 | }; 161 | // lift fields from the output JSON one level 162 | for (const [k, v] of Object.entries(test.output)) { 163 | // fix camelization 164 | const key = snakeCase(k); 165 | value[key] = v; 166 | } 167 | acc[test.name] = value; 168 | return acc; 169 | }, /** @type {any} */ ({})); 170 | return tests; 171 | } 172 | 173 | /** 174 | * 175 | * @param {string} name 176 | * @returns {string} 177 | */ 178 | function getTitle(name) { 179 | return TEST_TITLES[name] || name; 180 | } 181 | 182 | /** 183 | * @param {Pool} pool 184 | * @param {number} siteId 185 | */ 186 | export async function historyForSite(pool, siteId) { 187 | const historyRows = await selectScanHostHistory(pool, siteId); 188 | const history = historyRows.map((h) => { 189 | return { 190 | id: h.id, 191 | grade: h.grade, 192 | score: h.score, 193 | scanned_at: h.end_time, 194 | }; 195 | }); 196 | return history; 197 | } 198 | 199 | /** 200 | * Massage test results for API responses, i.e. CSP info, 201 | * null ("not applicable") values for pass on some results. 202 | * @param {any} tests 203 | * @returns {any} 204 | */ 205 | export function hydrateTests(tests) { 206 | // hydrate the csp test policy with descriptions and info fields 207 | if (tests["content-security-policy"]?.policy) { 208 | const fatPolicy = new PolicyResponse( 209 | tests["content-security-policy"].policy 210 | ); 211 | // dynamicStrict exception, set pass=null if false 212 | if (fatPolicy.strictDynamic.pass === false) { 213 | fatPolicy.strictDynamic.pass = null; 214 | } 215 | 216 | tests["content-security-policy"].policy = fatPolicy; 217 | } 218 | // For some tests whose pass flag is "not applicable", we 219 | // return null on the pass field. 220 | 221 | const noneResults = [ 222 | Expectation.ReferrerPolicyNotImplemented, 223 | Expectation.SriNotImplementedResponseNotHtml, 224 | Expectation.SriNotImplementedButNoScriptsLoaded, 225 | Expectation.SriNotImplementedButAllScriptsLoadedFromSecureOrigin, 226 | Expectation.CookiesNotFound, 227 | Expectation.CrossOriginResourcePolicyNotImplemented, 228 | ]; 229 | 230 | for (const [k, v] of Object.entries(tests)) { 231 | if (v.result && noneResults.includes(v.result)) { 232 | tests[k].pass = null; 233 | } 234 | } 235 | 236 | return tests; 237 | } 238 | 239 | /** 240 | * 241 | * @param {Pool} pool 242 | * @param {import("../../site.js").Site} site 243 | * @returns {Promise} 244 | */ 245 | export async function executeScan(pool, site) { 246 | const siteId = await ensureSite(pool, site.asSiteKey()); 247 | let scanRow = await insertScan(pool, siteId); 248 | const scanId = scanRow.id; 249 | let scanResult; 250 | try { 251 | scanResult = await scan(site); 252 | } catch (e) { 253 | if (e instanceof Error) { 254 | await updateScanState(pool, scanId, ScanState.FAILED, e.message); 255 | throw new ScanFailedError(e); 256 | } else { 257 | const unknownError = new Error("Unknown error occurred"); 258 | await updateScanState( 259 | pool, 260 | scanId, 261 | ScanState.FAILED, 262 | unknownError.message 263 | ); 264 | throw new ScanFailedError(unknownError); 265 | } 266 | } 267 | scanRow = await insertTestResults(pool, siteId, scanId, scanResult); 268 | return scanRow; 269 | } 270 | -------------------------------------------------------------------------------- /src/analyzer/tests/cookies.js: -------------------------------------------------------------------------------- 1 | import { SET_COOKIE } from "../../headers.js"; 2 | import { Requests, BaseOutput } from "../../types.js"; 3 | import { Expectation } from "../../types.js"; 4 | import { getHttpHeaders, onlyIfWorse } from "../utils.js"; 5 | import { strictTransportSecurityTest } from "./strict-transport-security.js"; 6 | 7 | // See: https://github.com/mozilla/http-observatory/issues/282 for the heroku-session-affinity insanity 8 | const COOKIES_TO_DELETE = ["heroku-session-affinity"]; 9 | 10 | /** 11 | * @typedef {{ [key: string]: CookieDataItem }} CookieMap 12 | * / 13 | 14 | /** 15 | * @typedef {object} CookieDataItem 16 | * @property {string} domain 17 | * @property {number} expires 18 | * @property {boolean} httponly 19 | * @property {number | "Infinity" | "-Infinity"} `max-age`` 20 | * @property {string} path 21 | * @property {null} port 22 | * @property {string} samesite 23 | * @property {boolean} secure 24 | */ 25 | 26 | export class CookiesOutput extends BaseOutput { 27 | /** @type {CookieMap | null} */ 28 | data = null; 29 | // Store whether or not we saw SameSite cookies, if cookies were set 30 | /** @type {boolean | null} */ 31 | sameSite = null; 32 | static name = "cookies"; 33 | static title = "Cookies"; 34 | static possibleResults = [ 35 | Expectation.CookiesSecureWithHttponlySessionsAndSamesite, 36 | Expectation.CookiesSecureWithHttponlySessions, 37 | Expectation.CookiesNotFound, 38 | Expectation.CookiesWithoutSecureFlagButProtectedByHsts, 39 | Expectation.CookiesSessionWithoutSecureFlagButProtectedByHsts, 40 | Expectation.CookiesWithoutSecureFlag, 41 | Expectation.CookiesSamesiteFlagInvalid, 42 | Expectation.CookiesAnticsrfWithoutSamesiteFlag, 43 | Expectation.CookiesSessionWithoutHttponlyFlag, 44 | Expectation.CookiesSessionWithoutSecureFlag, 45 | ]; 46 | 47 | /** @param {Expectation} expectation */ 48 | constructor(expectation) { 49 | super(expectation); 50 | } 51 | } 52 | 53 | /** 54 | * 55 | * @param {Requests} requests 56 | * @param {Expectation} expectation 57 | * @returns {CookiesOutput} 58 | */ 59 | export function cookiesTest( 60 | requests, 61 | expectation = Expectation.CookiesSecureWithHttponlySessions 62 | ) { 63 | const output = new CookiesOutput(expectation); 64 | const goodness = [ 65 | Expectation.CookiesWithoutSecureFlagButProtectedByHsts, 66 | Expectation.CookiesWithoutSecureFlag, 67 | Expectation.CookiesSessionWithoutSecureFlagButProtectedByHsts, 68 | Expectation.CookiesSamesiteFlagInvalid, 69 | Expectation.CookiesAnticsrfWithoutSamesiteFlag, 70 | Expectation.CookiesSessionWithoutHttponlyFlag, 71 | Expectation.CookiesSessionWithoutSecureFlag, 72 | ]; 73 | 74 | const hsts = strictTransportSecurityTest(requests)["pass"]; 75 | 76 | output.sameSite = false; 77 | let hasMissingSameSite = false; 78 | 79 | // Check if we got a malformed SameSite on the raw headers 80 | if (requests.responses.auto) { 81 | const rawCookies = getHttpHeaders(requests.responses.auto, SET_COOKIE); 82 | if (rawCookies) { 83 | for (const rawCookie of rawCookies) { 84 | if (containsInvalidSameSiteCookie(rawCookie)) { 85 | output.result = onlyIfWorse( 86 | Expectation.CookiesSamesiteFlagInvalid, 87 | output.result, 88 | goodness 89 | ); 90 | } 91 | } 92 | } 93 | } 94 | 95 | // get ALL the cookies from the store with serializeSync instead of using getCookiesSync 96 | const allCookies = 97 | requests.session?.jar?.serializeSync()?.cookies.filter(filterCookies) ?? []; 98 | 99 | if (!allCookies.length) { 100 | output.result = Expectation.CookiesNotFound; 101 | output.data = null; 102 | } else { 103 | // Now loop through all remaining cookies in the jar 104 | // and do the checks 105 | for (const cookie of allCookies) { 106 | // Is it a session identifier or an anti-csrf token? 107 | const sessionId = ["login", "sess"].some((i) => 108 | cookie.key?.toLowerCase().includes(i) 109 | ); 110 | const anticsrf = cookie.key?.toLowerCase().includes("csrf"); 111 | 112 | if ( 113 | !cookie.secure && 114 | typeof cookie.sameSite === "string" && 115 | cookie.sameSite.toLowerCase() === "none" 116 | ) { 117 | output.result = onlyIfWorse( 118 | Expectation.CookiesSamesiteFlagInvalid, 119 | output.result, 120 | goodness 121 | ); 122 | } 123 | 124 | if (!cookie.secure && hsts) { 125 | output.result = onlyIfWorse( 126 | Expectation.CookiesWithoutSecureFlagButProtectedByHsts, 127 | output.result, 128 | goodness 129 | ); 130 | } else if (!cookie.secure) { 131 | output.result = onlyIfWorse( 132 | Expectation.CookiesWithoutSecureFlag, 133 | output.result, 134 | goodness 135 | ); 136 | } 137 | 138 | // Anti-CSRF tokens should be set using the SameSite option. 139 | if (anticsrf && !cookie.sameSite) { 140 | output.result = onlyIfWorse( 141 | Expectation.CookiesAnticsrfWithoutSamesiteFlag, 142 | output.result, 143 | goodness 144 | ); 145 | } 146 | 147 | // Login and session cookies should be set with Secure. 148 | if (sessionId && !cookie.secure && hsts) { 149 | output.result = onlyIfWorse( 150 | Expectation.CookiesSessionWithoutSecureFlagButProtectedByHsts, 151 | output.result, 152 | goodness 153 | ); 154 | } else if (sessionId && !cookie.secure) { 155 | output.result = onlyIfWorse( 156 | Expectation.CookiesSessionWithoutSecureFlag, 157 | output.result, 158 | goodness 159 | ); 160 | } 161 | 162 | // Login and session cookies should be set with HttpOnly. 163 | if (sessionId && !cookie.httpOnly) { 164 | output.result = onlyIfWorse( 165 | Expectation.CookiesSessionWithoutHttponlyFlag, 166 | output.result, 167 | goodness 168 | ); 169 | } 170 | if (!cookie.sameSite && !hasMissingSameSite) { 171 | hasMissingSameSite = true; 172 | } 173 | } 174 | 175 | if (!output.result) { 176 | if (hasMissingSameSite) { 177 | output.result = Expectation.CookiesSecureWithHttponlySessions; 178 | output.sameSite = false; 179 | } else { 180 | output.result = 181 | Expectation.CookiesSecureWithHttponlySessionsAndSamesite; 182 | output.sameSite = true; 183 | } 184 | } 185 | 186 | const cookieSize = allCookies.join("").length; 187 | if (cookieSize < 32768) { 188 | /** @type {cookieData} */ 189 | let cookieData = {}; 190 | cookieData = allCookies.reduce((acc, cookie) => { 191 | acc[cookie.key] = { 192 | domain: cookie.domain, 193 | expires: cookie.expires, 194 | httponly: cookie.httpOnly, 195 | "max-age": cookie.maxAge, 196 | path: cookie.path, 197 | port: null, // do we support ports? no. 198 | samesite: cookie.sameSite, 199 | secure: cookie.secure, 200 | }; 201 | return acc; 202 | }, cookieData); 203 | output.data = cookieData; 204 | } 205 | } 206 | 207 | // Check to see if the test passed or failed 208 | if ( 209 | [ 210 | Expectation.CookiesNotFound, 211 | Expectation.CookiesSecureWithHttponlySessionsAndSamesite, 212 | expectation, 213 | ].includes(output.result) 214 | ) { 215 | output.pass = true; 216 | } 217 | return output; 218 | } 219 | 220 | /** 221 | * 222 | * @param {string} cookieString 223 | */ 224 | function containsInvalidSameSiteCookie(cookieString) { 225 | const parts = cookieString.trim().split(";"); 226 | for (const p of parts) { 227 | const splitResult = p.trim().split("="); 228 | const key = splitResult[0]; 229 | const value = splitResult[1]; 230 | if (key && key.trim().toLowerCase() === "samesite") { 231 | if (!value) { 232 | return true; 233 | } 234 | if (!["lax", "strict", "none"].includes(value.trim().toLowerCase())) { 235 | return true; 236 | } 237 | } 238 | } 239 | return false; 240 | } 241 | 242 | /** 243 | * 244 | * @param {import("tough-cookie").SerializedCookie} cookie 245 | */ 246 | function filterCookies(cookie) { 247 | const key = cookie.key; 248 | return key && !COOKIES_TO_DELETE.includes(key); 249 | } 250 | --------------------------------------------------------------------------------