├── .gitignore ├── w3c.json ├── .github ├── dependabot.yml ├── candidate-specs.md ├── workflows │ ├── check-base-url.yml │ ├── request-pr-review.yml │ ├── monitor-specs.yml │ ├── lint.yml │ ├── release-package.yml │ ├── report-new-specs.yml │ └── build.yml └── incorrect-base-url.md ├── packages ├── web-specs │ ├── package.json │ └── README.md └── browser-specs │ ├── package.json │ └── README.md ├── src ├── extract-pages.js ├── throttle.js ├── octokit.js ├── check-base-url.js ├── determine-filename.js ├── compute-prevnext.js ├── compute-currentlevel.js ├── compute-shorttitle.js ├── monitor-specs.js ├── parse-spec-url.js ├── compute-categories.js ├── compute-series-urls.js ├── prepare-packages.js ├── request-pr-review.js ├── build-diff.js ├── bump-packages-minor.js ├── lint.js ├── release-package.js ├── determine-testpath.js ├── compute-repository.js ├── fetch-groups.js ├── compute-shortname.js ├── find-specs.js └── fetch-info.js ├── package.json ├── test ├── determine-filename.js ├── extract-pages.js ├── fetch-groups.js ├── data.js ├── compute-categories.js ├── compute-shorttitle.js ├── compute-repository.js ├── fetch-info.js ├── compute-currentlevel.js ├── fetch-groups-w3c.js ├── compute-prevnext.js ├── fetch-info-w3c.js ├── compute-series-urls.js ├── lint.js ├── compute-shortname.js ├── index.js └── specs.js ├── index.js ├── schema ├── specs.json ├── index.json ├── data.json └── definitions.json ├── LICENSE.md └── CONTRIBUTING.md /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | node_modules/ 3 | config.json 4 | packages/**/index.json -------------------------------------------------------------------------------- /w3c.json: -------------------------------------------------------------------------------- 1 | { 2 | "contacts": ["dontcallmedom", "tidoust"], 3 | "repo-type": "tool" 4 | } 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: '10:00' 8 | open-pull-requests-limit: 10 -------------------------------------------------------------------------------- /packages/web-specs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web-specs", 3 | "version": "2.35.1", 4 | "description": "Curated list of technical Web specifications", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/w3c/browser-specs.git" 8 | }, 9 | "bugs": { 10 | "url": "https://github.com/w3c/browser-specs/issues" 11 | }, 12 | "license": "CC0-1.0", 13 | "files": [ 14 | "index.json" 15 | ], 16 | "main": "index.json" 17 | } -------------------------------------------------------------------------------- /packages/browser-specs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "browser-specs", 3 | "version": "3.33.1", 4 | "description": "Curated list of technical Web specifications that are directly implemented or that will be implemented by Web browsers.", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/w3c/browser-specs.git" 8 | }, 9 | "bugs": { 10 | "url": "https://github.com/w3c/browser-specs/issues" 11 | }, 12 | "license": "CC0-1.0", 13 | "files": [ 14 | "index.json" 15 | ], 16 | "main": "index.json" 17 | } -------------------------------------------------------------------------------- /.github/candidate-specs.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: New specs for review 3 | assignees: tidoust, dontcallmedom, sideshowbarker 4 | labels: enhancement 5 | --- 6 | [find-specs](../blob/main/src/find-specs.js) has identified the following candidates as potential new specs to consider: 7 | 8 | {{ env.candidate_list }} 9 | 10 | Please review if they match the inclusion criteria. Those that don't and never will should be added to [ignore.json](../blob/main/src/data/ignore.json), those that don't match yet but may in the future can be added to [monitor-repo.json](../blob/main/src/data/monitor-repos.json), and those that do match should be brought as a pull request on [specs.json](../blob/main/specs.json). 11 | -------------------------------------------------------------------------------- /.github/workflows/check-base-url.yml: -------------------------------------------------------------------------------- 1 | name: Check base URL 2 | 3 | on: 4 | schedule: 5 | - cron: '30 0 * * 1' 6 | workflow_dispatch: 7 | 8 | jobs: 9 | find-specs: 10 | name: Check base URL 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Use Node.js 16.x 15 | uses: actions/setup-node@v1 16 | with: 17 | node-version: 16.x 18 | - run: npm ci 19 | - run: node src/check-base-url.js # sets check_list env variable 20 | - uses: JasonEtco/create-an-issue@v2 21 | if: ${{ env.check_list }} 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | with: 25 | filename: .github/incorrect-base-url.md -------------------------------------------------------------------------------- /.github/incorrect-base-url.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Base URL mismatch 3 | assignees: tidoust, dontcallmedom 4 | labels: bug 5 | --- 6 | [check-base-url](../blob/main/src/check-base-url.js) has detected that the base URL (i.e. the one that appears in the root `url` property in `index.json`) of the following specifications does not match the `release` URL or the `nightly` URL: 7 | 8 | {{ env.check_list }} 9 | 10 | Please review the above list. For each specification, consider updating the URL in [specs.json](../blob/main/specs.json) or fixing the info at the source (the W3C API, Specref, or the spec itself). If the discrepancy seems warranted, the specification should be hardcoded as an exception to the rule in the [check-base-url](../blob/main/src/check-base-url.js) script. -------------------------------------------------------------------------------- /.github/workflows/request-pr-review.yml: -------------------------------------------------------------------------------- 1 | name: "NPM release: Request review of pre-release PR" 2 | 3 | on: 4 | schedule: 5 | - cron: '0 5 * * 4' 6 | workflow_dispatch: 7 | 8 | jobs: 9 | review: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Setup node.js 13 | uses: actions/setup-node@v2 14 | with: 15 | node-version: 16.x 16 | 17 | - name: Checkout webref 18 | uses: actions/checkout@v2 19 | 20 | - name: Install dependencies 21 | run: | 22 | npm install --no-save @octokit/rest 23 | npm install --no-save @octokit/plugin-throttling 24 | 25 | - name: Request review of pre-release PR 26 | run: node src/request-pr-review.js 27 | env: 28 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /src/extract-pages.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes the URL of the index page of a 3 | * multi-page spec as input and that returns the list of pages referenced in 4 | * the table of contents, in document order, excluding the index page. 5 | */ 6 | 7 | const { JSDOM } = require("jsdom"); 8 | 9 | module.exports = async function (url) { 10 | try { 11 | const dom = await JSDOM.fromURL(url); 12 | const window = dom.window; 13 | const document = window.document; 14 | 15 | const allPages = [...document.querySelectorAll('.toc a[href]')] 16 | .map(link => link.href) 17 | .map(url => url.split('#')[0]) 18 | .filter(url => url !== window.location.href); 19 | const pageSet = new Set(allPages); 20 | return [...pageSet]; 21 | } 22 | catch (err) { 23 | throw new Error(`Could not extract pages from ${url} with JSDOM: ${err.message}`); 24 | } 25 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "browser-specs", 3 | "version": "2.27.0", 4 | "repository": { 5 | "type": "git", 6 | "url": "git+https://github.com/w3c/browser-specs.git" 7 | }, 8 | "files": [ 9 | "index.json" 10 | ], 11 | "license": "CC0-1.0", 12 | "main": "index.json", 13 | "scripts": { 14 | "build": "node src/build-index.js", 15 | "lint": "node src/lint.js", 16 | "lint-fix": "node src/lint.js --fix", 17 | "test": "mocha", 18 | "test-pr": "mocha --exclude test/fetch-info-w3c.js --exclude test/fetch-groups-w3c.js" 19 | }, 20 | "devDependencies": { 21 | "@actions/core": "^1.10.0", 22 | "@jsdevtools/npm-publish": "^1.4.3", 23 | "@octokit/plugin-throttling": "^4.3.2", 24 | "@octokit/rest": "^19.0.5", 25 | "ajv": "^8.11.2", 26 | "ajv-formats": "^2.1.1", 27 | "jsdom": "^20.0.2", 28 | "mocha": "^10.1.0", 29 | "node-fetch": "^2.6.7", 30 | "rimraf": "^3.0.2" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /test/determine-filename.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const determineFilename = require("../src/determine-filename.js"); 3 | 4 | describe("determine-filename module", function () { 5 | // Tests need to send network requests 6 | this.slow(5000); 7 | this.timeout(30000); 8 | 9 | it("extracts filename from URL", async () => { 10 | const url = "https://example.org/spec/filename.html"; 11 | const filename = await determineFilename(url); 12 | assert.equal(filename, "filename.html"); 13 | }); 14 | 15 | it("finds index.html filenames", async () => { 16 | const url = "https://w3c.github.io/presentation-api/"; 17 | const filename = await determineFilename(url); 18 | assert.equal(filename, "index.html"); 19 | }); 20 | 21 | it("finds Overview.html filenames", async () => { 22 | const url = "https://www.w3.org/TR/presentation-api/"; 23 | const filename = await determineFilename(url); 24 | assert.equal(filename, "Overview.html"); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/throttle.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Provides a throttling wrapper to a function 3 | */ 4 | 5 | module.exports = function (fn, max) { 6 | let ongoing = 0; 7 | const pending = []; 8 | 9 | return async function throttled(...args) { 10 | if (ongoing >= max) { 11 | // Too many tasks in parallel, need to throttle 12 | return new Promise((resolve, reject) => { 13 | pending.push({ params: [...args], resolve, reject }); 14 | }); 15 | } 16 | else { 17 | // Task can run immediately 18 | ongoing += 1; 19 | const res = await fn.call(null, ...args); 20 | ongoing -= 1; 21 | 22 | // Done with current task, trigger next pending task in the background 23 | setTimeout(() => { 24 | if (pending.length && ongoing < max) { 25 | const next = pending.shift(); 26 | throttled.apply(null, next.params) 27 | .then(res => next.resolve(res)) 28 | .catch(err => next.reject(err)); 29 | } 30 | }, 0); 31 | 32 | return res; 33 | } 34 | }; 35 | } 36 | -------------------------------------------------------------------------------- /.github/workflows/monitor-specs.yml: -------------------------------------------------------------------------------- 1 | name: Monitor specs 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 1 */2 *' 6 | workflow_dispatch: 7 | 8 | jobs: 9 | find-specs: 10 | name: Update the list of monitored specs and highlights those that have changed 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Use Node.js 16.x 15 | uses: actions/setup-node@v1 16 | with: 17 | node-version: 16.x 18 | - run: npm ci 19 | - run: node src/monitor-specs.js # sets review_list env variable 20 | - uses: peter-evans/create-pull-request@v3 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | with: 24 | title: Update review of monitored specs 25 | commit-message: "Update review of monitored specs" 26 | body: | 27 | The following specs have been updated since the last review: 28 | ${{env.review_list}} 29 | assignees: tidoust, dontcallmedom, sideshowbarker 30 | branch: monitor-update 31 | branch-suffix: timestamp -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Test and lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | workflow_dispatch: 11 | 12 | jobs: 13 | lint: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout repo contents 17 | uses: actions/checkout@v1 18 | 19 | - name: Setup node.js 20 | uses: actions/setup-node@v1 21 | with: 22 | node-version: 16.x 23 | 24 | - name: Install dependencies 25 | run: npm ci 26 | 27 | - name: Test (including W3C API tests) 28 | env: 29 | CONFIG_JSON: ${{ secrets.CONFIG_JSON }} 30 | if: ${{ env.CONFIG_JSON }} 31 | run: | 32 | echo "${{ secrets.CONFIG_JSON }}" | base64 --decode > config.json 33 | npm run test 34 | 35 | - name: Test (without W3C API tests) 36 | env: 37 | CONFIG_JSON: ${{ secrets.CONFIG_JSON }} 38 | if: ${{ !env.CONFIG_JSON }} 39 | run: npm run test-pr 40 | 41 | - name: Lint 42 | run: npm run lint -------------------------------------------------------------------------------- /.github/workflows/release-package.yml: -------------------------------------------------------------------------------- 1 | # Publish a new package when a pre-release PR is merged. 2 | # 3 | # Job does nothing if PR that was merged is not a pre-release PR. 4 | 5 | name: "Publish NPM package if needed" 6 | 7 | on: 8 | pull_request: 9 | branches: 10 | - main 11 | types: 12 | - closed 13 | 14 | jobs: 15 | release: 16 | if: startsWith(github.head_ref, 'release-') && github.event.pull_request.merged == true 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Setup node.js 20 | uses: actions/setup-node@v2 21 | with: 22 | node-version: 16.x 23 | 24 | - name: Checkout latest version of release script 25 | uses: actions/checkout@v2 26 | with: 27 | ref: main 28 | 29 | - name: Install dependencies 30 | run: | 31 | npm install --no-save @octokit/rest 32 | npm install --no-save @octokit/plugin-throttling 33 | npm install --no-save rimraf 34 | npm install --no-save @jsdevtools/npm-publish 35 | 36 | - name: Release package if needed 37 | run: node src/release-package.js ${{ github.event.pull_request.number }} 38 | env: 39 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 41 | -------------------------------------------------------------------------------- /.github/workflows/report-new-specs.yml: -------------------------------------------------------------------------------- 1 | name: Report new specs 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * 1' 6 | workflow_dispatch: 7 | 8 | jobs: 9 | find-specs: 10 | name: Find potential new specs 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Use Node.js 16.x 15 | uses: actions/setup-node@v1 16 | with: 17 | node-version: 16.x 18 | - run: npm ci 19 | - run: node src/find-specs.js # sets candidate_list env variable 20 | - uses: JasonEtco/create-an-issue@v2 21 | if: ${{ env.candidate_list }} 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | with: 25 | filename: .github/candidate-specs.md 26 | - uses: peter-evans/create-pull-request@v3 27 | if: ${{ env.monitor_list }} 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | with: 31 | title: Identify new monitored repos 32 | commit-message: "List new repos to be monitored" 33 | body: | 34 | The following repos have been identified as possibly relevant: 35 | ${{env.monitor_list}} 36 | assignees: tidoust, dontcallmedom, sideshowbarker 37 | branch: new-monitor 38 | branch-suffix: timestamp -------------------------------------------------------------------------------- /src/octokit.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Wrapper around Octokit to add throttling and avoid hitting rate limits 3 | */ 4 | 5 | const { throttling } = require("@octokit/plugin-throttling"); 6 | const Octokit = require("@octokit/rest").Octokit.plugin(throttling); 7 | 8 | const MAX_RETRIES = 3; 9 | 10 | module.exports = function (params) { 11 | params = params || {}; 12 | 13 | const octoParams = Object.assign({ 14 | throttle: { 15 | onRateLimit: (retryAfter, options) => { 16 | if (options.request.retryCount < MAX_RETRIES) { 17 | console.warn(`Rate limit exceeded, retrying after ${retryAfter} seconds`) 18 | return true; 19 | } else { 20 | console.error(`Rate limit exceeded, giving up after ${MAX_RETRIES} retries`); 21 | return false; 22 | } 23 | }, 24 | onSecondaryRateLimit: (retryAfter, options) => { 25 | if (options.request.retryCount < MAX_RETRIES) { 26 | console.warn(`Abuse detection triggered, retrying after ${retryAfter} seconds`) 27 | return true; 28 | } else { 29 | console.error(`Abuse detection triggered, giving up after ${MAX_RETRIES} retries`); 30 | return false; 31 | } 32 | } 33 | } 34 | }, params); 35 | 36 | return new Octokit(octoParams); 37 | } 38 | -------------------------------------------------------------------------------- /src/check-base-url.js: -------------------------------------------------------------------------------- 1 | /** 2 | * CLI tool that parses the generated index of specifications to make sure that 3 | * the base URL either matches the release URL if there is one, or the nightly 4 | * URL otherwise. 5 | * 6 | * The CLI tool returns Markdown that can typically be used to create an issue. 7 | * It also sets a check_list environment variable that can be used in GitHub 8 | * actions. 9 | * 10 | * No content is returned when everything looks good. 11 | */ 12 | 13 | const core = require("@actions/core"); 14 | const specs = require("../index.json"); 15 | 16 | const problems = specs 17 | // A subset of the IETF RFCs are crawled from their httpwg.org rendering 18 | // see https://github.com/tobie/specref/issues/672 and 19 | // https://github.com/w3c/browser-specs/issues/280 20 | .filter(s => !s.nightly || !s.nightly.url.startsWith('https://httpwg.org')) 21 | .filter(s => (s.release && s.url !== s.release.url) || (!s.release && s.url !== s.nightly.url)) 22 | .map(s => { 23 | const expected = s.release ? "release" : "nightly"; 24 | const expectedUrl = s.release ? s.release.url : s.nightly.url; 25 | return `- [ ] [${s.title}](${s.url}): expected ${expected} URL ${expectedUrl} to match base URL ${s.url}`; 26 | }); 27 | 28 | if (problems.length > 0) { 29 | const res = problems.join("\n"); 30 | core.exportVariable("check_list", res); 31 | console.log(res); 32 | } 33 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const specs = require("./index.json"); 4 | 5 | 6 | /** 7 | * Return the list of specs that match the specified filter. 8 | * 9 | * - If the filter is an integer, return the spec at that index in the list 10 | * - If the filter is full or delta, return specs with same level composition 11 | * - If the filter is empty, return the whole list 12 | * - return specs that have the same URL, name, shortname, or source otherwise 13 | */ 14 | function getSpecs(filter) { 15 | if (filter) { 16 | const res = filter.match(/^\d+$/) ? 17 | [specs[parseInt(filter, 10)]] : 18 | specs.filter(s => 19 | s.url === filter || 20 | s.name === filter || 21 | s.seriesComposition === filter || 22 | s.source === filter || 23 | s.title === filter || 24 | (s.series && s.series.shortname === filter) || 25 | (s.release && s.release.url === filter) || 26 | (s.nightly && s.nightly.url === filter)); 27 | return res; 28 | } 29 | else { 30 | return specs; 31 | } 32 | } 33 | 34 | 35 | if (require.main === module) { 36 | // Code used as command-line interface (CLI), output info about known specs. 37 | const res = getSpecs(process.argv[2]); 38 | console.log(JSON.stringify(res.length === 1 ? res[0] : res, null, 2)); 39 | } 40 | else { 41 | // Code referenced from another JS module, export 42 | module.exports = { getSpecs }; 43 | } -------------------------------------------------------------------------------- /src/determine-filename.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that takes the URL of the index page of a spec as input, possibly 3 | * without a filename, and that tries to determine the underlying filename. 4 | * 5 | * For instance: 6 | * - given "https://w3c.github.io/webrtc-identity/identity.html", the function 7 | * would return "identity.html" 8 | * - given "https://compat.spec.whatwg.org/", the function would determine that 9 | * the filename is "index.html". 10 | */ 11 | 12 | const fetch = require("node-fetch"); 13 | 14 | module.exports = async function (url) { 15 | // Extract filename directly from the URL when possible 16 | const match = url.match(/\/([^/]+\.html)$/); 17 | if (match) { 18 | return match[1]; 19 | } 20 | 21 | // RFC-editor HTML rendering 22 | const rfcMatch = url.match(/\/rfc\/(rfc[0-9]+)$/); 23 | if (rfcMatch) { 24 | return rfcMatch[1] + '.html'; 25 | } 26 | 27 | // Make sure that url ends with a "/" 28 | const urlWithSlash = url.endsWith("/") ? url : url + "/"; 29 | 30 | // Check common candidates 31 | const candidates = [ 32 | "Overview.html", 33 | "index.html" 34 | ]; 35 | 36 | for (const candidate of candidates) { 37 | const res = await fetch(urlWithSlash + candidate, { method: "HEAD" }); 38 | if (res.status === 200) { 39 | return candidate; 40 | } 41 | } 42 | 43 | // Not found? Look at Content-Location header 44 | const res = await fetch(url, { method: "HEAD" }); 45 | const filename = res.headers.get("Content-Location"); 46 | return filename; 47 | } 48 | -------------------------------------------------------------------------------- /schema/specs.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/schema#", 3 | "$id": "https://w3c.github.io/browser-specs/schema/specs.json", 4 | 5 | "type": "array", 6 | "items": { 7 | "oneOf": [ 8 | { 9 | "type": "string", 10 | "pattern": "^https://[^\\s]+(\\s(delta|current|multipage))?$" 11 | }, 12 | { 13 | "type": "object", 14 | "properties": { 15 | "url": { "$ref": "definitions.json#/$defs/url" }, 16 | "shortname": { "$ref": "definitions.json#/$defs/shortname" }, 17 | "forkOf": { "$ref": "definitions.json#/$defs/shortname" }, 18 | "series": { "$ref": "definitions.json#/$defs/series" }, 19 | "seriesVersion": { "$ref": "definitions.json#/$defs/seriesVersion" }, 20 | "seriesComposition": { "$ref": "definitions.json#/$defs/seriesComposition" }, 21 | "nightly": { "$ref": "definitions.json#/$defs/nightly" }, 22 | "tests": { "$ref": "definitions.json#/$defs/tests" }, 23 | "title": { "$ref": "definitions.json#/$defs/title" }, 24 | "shortTitle": { "$ref": "definitions.json#/$defs/title" }, 25 | "organization": { "$ref": "definitions.json#/$defs/organization" }, 26 | "groups": { "$ref": "definitions.json#/$defs/groups" }, 27 | "categories": { "$ref": "definitions.json#/$defs/categories-specs" }, 28 | "forceCurrent": { "type": "boolean" }, 29 | "multipage": { "type": "boolean" } 30 | }, 31 | "required": ["url"], 32 | "additionalProperties": false 33 | } 34 | ] 35 | }, 36 | "minItems": 1 37 | } 38 | -------------------------------------------------------------------------------- /test/extract-pages.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const extractPages = require("../src/extract-pages.js"); 3 | 4 | describe("extract-pages module", function () { 5 | // Tests need to send network requests 6 | this.slow(5000); 7 | this.timeout(30000); 8 | 9 | it("extracts pages from the SVG2 spec", async () => { 10 | const url = "https://svgwg.org/svg2-draft/" 11 | const pages = await extractPages(url); 12 | assert.ok(pages.length > 20); 13 | }); 14 | 15 | it("extracts pages from the HTML spec", async () => { 16 | const url = "https://html.spec.whatwg.org/multipage/"; 17 | const pages = await extractPages(url); 18 | assert.ok(pages.length > 20); 19 | }); 20 | 21 | it("extracts pages from the CSS 2.1 spec", async () => { 22 | const url = "https://www.w3.org/TR/CSS21/"; 23 | const pages = await extractPages(url); 24 | assert.ok(pages.length > 20); 25 | }); 26 | 27 | it("does not include the index page as first page", async () => { 28 | const url = "https://svgwg.org/svg2-draft/" 29 | const pages = await extractPages(url); 30 | assert.ok(!pages.find(page => page.url)); 31 | }); 32 | 33 | it("does not get lost when given a single-page ReSpec spec", async () => { 34 | const url = "https://w3c.github.io/presentation-api/"; 35 | const pages = await extractPages(url); 36 | assert.deepStrictEqual(pages, []); 37 | }); 38 | 39 | it("does not get lost when given a single-page Bikeshed spec", async () => { 40 | const url = "https://w3c.github.io/mediasession/"; 41 | const pages = await extractPages(url); 42 | assert.deepStrictEqual(pages, []); 43 | }); 44 | }); 45 | -------------------------------------------------------------------------------- /src/compute-prevnext.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a spec object that already has a 3 | * "shortname", "series" and "level" properties (if needed) as input along with 4 | * a list of specs with the same info for each spec, and that returns an object 5 | * with "seriesPrevious" and "seriesNext" properties as needed, that point 6 | * to the "shortname" of the spec object that describes the previous and next 7 | * level for the spec in the list. 8 | */ 9 | 10 | /** 11 | * Exports main function that takes a spec object and a list of specs (which 12 | * may contain the spec object itself) and returns an object with properties 13 | * "seriesPrevious" and/or "seriesNext" set. Function only sets the 14 | * properties when needed, so returned object may be empty. 15 | */ 16 | module.exports = function (spec, list) { 17 | if (!spec || !spec.shortname || !spec.series || !spec.series.shortname) { 18 | throw "Invalid spec object passed as parameter"; 19 | } 20 | 21 | list = list || []; 22 | const level = spec.seriesVersion || "0"; 23 | 24 | return list 25 | .filter(s => s.series.shortname === spec.series.shortname && s.seriesComposition !== "fork") 26 | .sort((a, b) => (a.seriesVersion || "0").localeCompare(b.seriesVersion || "0")) 27 | .reduce((res, s) => { 28 | if ((s.seriesVersion || "0") < level) { 29 | // Previous level is the last spec with a lower level 30 | res.seriesPrevious = s.shortname; 31 | } 32 | else if ((s.seriesVersion || "0") > level) { 33 | // Next level is the first spec with a greater level 34 | if (!res.seriesNext) { 35 | res.seriesNext = s.shortname; 36 | } 37 | } 38 | return res; 39 | }, {}); 40 | } -------------------------------------------------------------------------------- /schema/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/schema#", 3 | "$id": "https://w3c.github.io/browser-specs/schema/index.json", 4 | 5 | "type": "array", 6 | "items": { 7 | "type": "object", 8 | "properties": { 9 | "url": { "$ref": "definitions.json#/$defs/url" }, 10 | "shortname": { "$ref": "definitions.json#/$defs/shortname" }, 11 | "forkOf": { "$ref": "definitions.json#/$defs/shortname" }, 12 | "forks": { "$ref": "definitions.json#/$defs/forks" }, 13 | "series": { "$ref": "definitions.json#/$defs/series" }, 14 | "seriesVersion": { "$ref": "definitions.json#/$defs/seriesVersion" }, 15 | "seriesComposition": { "$ref": "definitions.json#/$defs/seriesComposition" }, 16 | "seriesPrevious": { "$ref": "definitions.json#/$defs/shortname" }, 17 | "seriesNext": { "$ref": "definitions.json#/$defs/shortname" }, 18 | "nightly": { "$ref": "definitions.json#/$defs/nightly" }, 19 | "tests": { "$ref": "definitions.json#/$defs/tests" }, 20 | "release": { "$ref": "definitions.json#/$defs/release" }, 21 | "title": { "$ref": "definitions.json#/$defs/title" }, 22 | "shortTitle": { "$ref": "definitions.json#/$defs/title" }, 23 | "source": { "$ref": "definitions.json#/$defs/source" }, 24 | "organization": { "$ref": "definitions.json#/$defs/organization" }, 25 | "groups": { "$ref": "definitions.json#/$defs/groups" }, 26 | "categories": { "$ref": "definitions.json#/$defs/categories" } 27 | }, 28 | "required": [ 29 | "url", "shortname", "series", "seriesComposition", "nightly", 30 | "title", "shortTitle", "source", "organization", "groups", "categories" 31 | ], 32 | "additionalProperties": false 33 | }, 34 | "minItems": 1 35 | } 36 | -------------------------------------------------------------------------------- /schema/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/schema#", 3 | "$id": "https://w3c.github.io/browser-specs/schema/data.json", 4 | 5 | "type": "object", 6 | "properties": { 7 | "groups": { 8 | "type": "object", 9 | "propertyNames": { 10 | "type": "string" 11 | }, 12 | "additionalProperties": { 13 | "type": "object", 14 | "properties": { 15 | "comment": { 16 | "type": "string" 17 | } 18 | }, 19 | "required": ["comment"], 20 | "additionalProperties": false 21 | } 22 | }, 23 | "repos": { 24 | "type": "object", 25 | "propertyNames": { 26 | "pattern": "^[\\w\\-\\.]+\\/[\\w\\-\\.]+$" 27 | }, 28 | "additionalProperties": { 29 | "type": "object", 30 | "properties": { 31 | "comment": { 32 | "type": "string" 33 | }, 34 | "lastreviewed": { 35 | "type": "string", 36 | "pattern": "^\\d{4}-\\d{2}-\\d{2}$" 37 | } 38 | }, 39 | "required": ["comment"], 40 | "additionalProperties": false 41 | } 42 | }, 43 | "specs": { 44 | "type": "object", 45 | "propertyNames": { 46 | "$ref": "definitions.json#/$defs/url" 47 | }, 48 | "additionalProperties": { 49 | "type": "object", 50 | "properties": { 51 | "comment": { 52 | "type": "string" 53 | }, 54 | "lastreviewed": { 55 | "type": "string", 56 | "pattern": "^\\d{4}-\\d{2}-\\d{2}$" 57 | } 58 | }, 59 | "required": ["comment"], 60 | "additionalProperties": false 61 | } 62 | } 63 | }, 64 | "additionalProperties": false 65 | } 66 | -------------------------------------------------------------------------------- /src/compute-currentlevel.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a spec object and a list of specs 3 | * that contains it, and that returns an object with a "currentSpecification" 4 | * property set to the "shortname" of the spec object that should be seen as 5 | * the current level for the set of specs with the same series' shortname in 6 | * the list. 7 | * 8 | * Each spec in the list must have "shortname", "series" and "seriesVersion" 9 | * (if needed) properties. 10 | * 11 | * By default, the current level is defined as the last level that is not a 12 | * delta/fork spec, unless a level is explicitly flagged with a "forceCurrent" 13 | * property in the list of specs. 14 | */ 15 | 16 | /** 17 | * Exports main function that takes a spec object and a list of specs (which 18 | * must contain the spec object itself) and returns an object with a 19 | * "currentSpecification" property. Function always sets the property (value is 20 | * the name of the spec itself when it is the current level) 21 | */ 22 | module.exports = function (spec, list) { 23 | list = list || []; 24 | if (!spec) { 25 | throw "Invalid spec object passed as parameter"; 26 | } 27 | 28 | const current = list.reduce((candidate, curr) => { 29 | if (curr.series.shortname === candidate.series.shortname && 30 | !candidate.forceCurrent && 31 | curr.seriesComposition !== "fork" && 32 | curr.seriesComposition !== "delta" && 33 | (curr.forceCurrent || 34 | candidate.seriesComposition === "delta" || 35 | candidate.seriesComposition === "fork" || 36 | (curr.seriesVersion || "0") > (candidate.seriesVersion || "0"))) { 37 | return curr; 38 | } 39 | else { 40 | return candidate; 41 | } 42 | }, spec); 43 | 44 | return { 45 | currentSpecification: current.shortname, 46 | forceCurrent: current.forceCurrent 47 | }; 48 | }; -------------------------------------------------------------------------------- /src/compute-shorttitle.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a title as input and returns a 3 | * meaningful short title out of it, or the full title if it cannot be 4 | * abbreviated. 5 | * 6 | * For instance, given "CSS Conditional Rules Module Level 3" as a title, the 7 | * function would return "CSS Conditional 3" 8 | */ 9 | 10 | 11 | /** 12 | * Internal function that takes a URL as input and returns a name for it 13 | * if the URL matches well-known patterns, or if the given parameter is actually 14 | * already a name (meaning that it does not contains any "/"). 15 | * 16 | * The function throws if it cannot compute a meaningful name from the URL. 17 | */ 18 | module.exports = function (title) { 19 | if (!title) { 20 | return title; 21 | } 22 | 23 | // Handle HTTP/1.1 specs separately to preserve feature name after "HTTP/1.1" 24 | const httpStart = 'Hypertext Transfer Protocol (HTTP/1.1): '; 25 | if (title.startsWith(httpStart)) { 26 | return 'HTTP/1.1 ' + title.substring(httpStart.length); 27 | } 28 | 29 | const level = title.match(/\s(\d+(\.\d+)?)$/); 30 | const shortTitle = title 31 | .replace(/\s/g, ' ') // Replace non-breaking spaces 32 | .replace(/ \d+(\.\d+)?$/, '') // Drop level number for now 33 | .replace(/( -)? Level$/, '') // Drop "Level" 34 | .replace(/ Module$/, '') // Drop "Module" (now followed by level) 35 | .replace(/ Proposal$/, '') // Drop "Proposal" (TC39 proposals) 36 | .replace(/ Specification$/, '') // Drop "Specification" 37 | .replace(/ Standard$/, '') // Drop "Standard" and "Living Standard" 38 | .replace(/ Living$/, '') 39 | .replace(/ \([^\)]+ Edition\)/, '') // Drop edition indication 40 | .replace(/^.*\(([^\)]+)\).*$/, '$1'); // Use abbr between parentheses 41 | 42 | if (level) { 43 | return shortTitle + " " + level[1]; 44 | } 45 | else { 46 | return shortTitle; 47 | } 48 | }; 49 | -------------------------------------------------------------------------------- /src/monitor-specs.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const fs = require("fs"); 3 | 4 | const core = require('@actions/core'); 5 | 6 | const fetch = require("node-fetch"); 7 | 8 | const monitorList = require("./data/monitor.json"); 9 | 10 | const toGhUrl = repo => `https://${repo.split("/")[0].toLowerCase()}.github.io/${repo.split("/")[1]}/`; 11 | 12 | const today = new Date().toJSON().slice(0, 10); 13 | 14 | (async function() { 15 | // Check last-modified HTTP header for specs to highlight those that needs 16 | // re-review 17 | let review_needed = []; 18 | const candidates = Object.keys(monitorList.repos).map(r => {return {...monitorList.repos[r], url: toGhUrl(r)};}).concat( 19 | Object.keys(monitorList.specs).map(s => {return {...monitorList.specs[s], url: s};})); 20 | for (let candidate of candidates) { 21 | await fetch(candidate.url).then(({headers}) => { 22 | // The CSS drafts use a proprietary header to expose the real last modification date 23 | const lastRevised = headers.get('Last-Revised') ? new Date(headers.get('Last-Revised')) : new Date(headers.get('Last-Modified')); 24 | if (lastRevised > new Date(candidate.lastreviewed)) { 25 | review_needed.push({...candidate, lastupdated: lastRevised.toJSON()}); 26 | } 27 | }); 28 | } 29 | const review_list = review_needed.map(c => `- [ ] ${c.url} updated on ${c.lastupdated}; last comment: “${c.comment}” made on ${c.lastreviewed}`).join("\n"); 30 | core.exportVariable("review_list", review_list); 31 | console.log(review_list); 32 | 33 | // Update monitor.json setting lastreviewed date today on all entries 34 | // This will serve as input to the automated pull request 35 | Object.values(monitorList.repos).forEach(r => r.lastreviewed = today); 36 | Object.values(monitorList.specs).forEach(r => r.lastreviewed = today); 37 | fs.writeFileSync("./src/data/monitor.json", JSON.stringify(monitorList, null, 2)); 38 | })().catch(e => { 39 | console.error(e); 40 | process.exit(1); 41 | }); 42 | 43 | 44 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Update spec info 2 | 3 | on: 4 | schedule: 5 | - cron: '10 */6 * * *' 6 | push: 7 | branches: 8 | - main 9 | paths-ignore: 10 | - 'packages/*/package.json' 11 | - 'test/**' 12 | workflow_dispatch: 13 | 14 | jobs: 15 | fetch: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Setup node.js 19 | uses: actions/setup-node@v2 20 | with: 21 | node-version: 16.x 22 | 23 | - name: Checkout repo 24 | uses: actions/checkout@v2 25 | with: 26 | # Need to checkout all history as job also needs to access the 27 | # xxx-specs@latest branches 28 | fetch-depth: 0 29 | 30 | - name: Setup environment 31 | run: | 32 | echo "${{ secrets.CONFIG_JSON }}" | base64 --decode > config.json 33 | npm ci 34 | 35 | - name: Build new index file 36 | env: 37 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | run: npm run build 39 | 40 | - name: Test new index file 41 | env: 42 | CONFIG_JSON: ${{ secrets.CONFIG_JSON }} 43 | run: | 44 | echo "${{ secrets.CONFIG_JSON }}" | base64 --decode > config.json 45 | npm run test 46 | 47 | - name: Bump minor version of packages if needed 48 | run: node src/bump-packages-minor.js 49 | 50 | - name: Commit updates 51 | run: | 52 | git config user.name "fetch-info bot" 53 | git config user.email "<>" 54 | git commit -m "[data] Update spec info" -a || true 55 | 56 | - name: Push changes 57 | uses: ad-m/github-push-action@v0.6.0 58 | with: 59 | github_token: ${{ secrets.GITHUB_TOKEN }} 60 | branch: main 61 | 62 | - name: Prepare packages data 63 | run: node src/prepare-packages.js 64 | 65 | - name: Create/Update pre-release PR for web-specs 66 | run: node src/prepare-release.js web-specs 67 | env: 68 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 69 | 70 | - name: Create/Update pre-release PR for browser-specs 71 | run: node src/prepare-release.js browser-specs 72 | env: 73 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 74 | -------------------------------------------------------------------------------- /test/fetch-groups.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const fetchGroups = require("../src/fetch-groups.js"); 3 | 4 | describe("fetch-groups module (without API keys)", function () { 5 | // Tests may need to send network requests 6 | this.slow(5000); 7 | this.timeout(30000); 8 | 9 | async function fetchGroupsFor(url) { 10 | const spec = { url }; 11 | const result = await fetchGroups([spec]); 12 | return result[0]; 13 | }; 14 | 15 | it("handles WHATWG URLs", async () => { 16 | const res = await fetchGroupsFor("https://url.spec.whatwg.org/"); 17 | assert.equal(res.organization, "WHATWG"); 18 | assert.deepStrictEqual(res.groups, [{ 19 | name: "URL Workstream", 20 | url: "https://url.spec.whatwg.org/" 21 | }]); 22 | }); 23 | 24 | it("handles TC39 URLs", async () => { 25 | const res = await fetchGroupsFor("https://tc39.es/proposal-relative-indexing-method/"); 26 | assert.equal(res.organization, "Ecma International"); 27 | assert.deepStrictEqual(res.groups, [{ 28 | name: "TC39", 29 | url: "https://tc39.es/" 30 | }]); 31 | }); 32 | 33 | it("handles WebGL URLs", async () => { 34 | const res = await fetchGroupsFor("https://registry.khronos.org/webgl/extensions/EXT_clip_cull_distance/"); 35 | assert.equal(res.organization, "Khronos Group"); 36 | assert.deepStrictEqual(res.groups, [{ 37 | name: "WebGL Working Group", 38 | url: "https://www.khronos.org/webgl/" 39 | }]); 40 | }); 41 | 42 | it("preserves provided info", async () => { 43 | const spec = { 44 | url: "https://url.spec.whatwg.org/", 45 | organization: "Acme Corporation", 46 | groups: [{ 47 | name: "Road Runner Group", 48 | url: "https://en.wikipedia.org/wiki/Wile_E._Coyote_and_the_Road_Runner" 49 | }] 50 | }; 51 | const res = await fetchGroups([spec]); 52 | assert.equal(res[0].organization, spec.organization); 53 | assert.deepStrictEqual(res[0].groups, spec.groups); 54 | }); 55 | 56 | it("preserves provided info for Patent Policy", async () => { 57 | const spec = { 58 | "url": "https://www.w3.org/Consortium/Patent-Policy/", 59 | "shortname": "w3c-patent-policy", 60 | "groups": [ 61 | { 62 | "name": "Patents and Standards Interest Group", 63 | "url": "https://www.w3.org/2004/pp/psig/" 64 | } 65 | ] 66 | }; 67 | const res = await fetchGroups([spec]); 68 | assert.equal(res[0].organization, "W3C"); 69 | assert.deepStrictEqual(res[0].groups, spec.groups); 70 | }); 71 | }); 72 | -------------------------------------------------------------------------------- /src/parse-spec-url.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Helper method that parses a spec URL and returns some information on the 3 | * type of URL (github, /TR, WHATWG, etc.), the owning organization on GitHub 4 | * and the likely GitHub repository name. 5 | * 6 | * Note that the repository name may be incorrect for /TR specs (as spec 7 | * shortnames do not always match the name of the actual repo). 8 | */ 9 | 10 | module.exports = function (url) { 11 | if (!url) { 12 | throw "No URL passed as parameter"; 13 | } 14 | 15 | const githubcom = url.match(/^https:\/\/github\.com\/([^\/]*)\/([^\/]*)\/?/); 16 | if (githubcom) { 17 | return { type: "github", owner: githubcom[1], name: githubcom[2] }; 18 | } 19 | 20 | const githubio = url.match(/^https:\/\/([^\.]*)\.github\.io\/([^\/]*)\/?/); 21 | if (githubio) { 22 | return { type: "github", owner: githubio[1], name: githubio[2] }; 23 | } 24 | 25 | const whatwg = url.match(/^https:\/\/([^\.]*).spec.whatwg.org\//); 26 | if (whatwg) { 27 | return { type: "custom", owner: "whatwg", name: whatwg[1] }; 28 | } 29 | 30 | const tc39 = url.match(/^https:\/\/tc39.es\/([^\/]*)\//); 31 | if (tc39) { 32 | return { type: "custom", owner: "tc39", name: tc39[1] }; 33 | } 34 | 35 | const csswg = url.match(/^https?:\/\/drafts.csswg.org\/([^\/]*)\/?/); 36 | if (csswg) { 37 | return { type: "custom", owner: "w3c", name: "csswg-drafts" }; 38 | } 39 | 40 | const ghfxtf = url.match(/^https:\/\/drafts.fxtf.org\/([^\/]*)\/?/); 41 | if (ghfxtf) { 42 | return { type: "custom", owner: "w3c", name: "fxtf-drafts" }; 43 | } 44 | 45 | const houdini = url.match(/^https:\/\/drafts.css-houdini.org\/([^\/]*)\/?/); 46 | if (houdini) { 47 | return { type: "custom", owner: "w3c", name: "css-houdini-drafts" }; 48 | } 49 | 50 | const svgwg = url.match(/^https:\/\/svgwg.org\/specs\/([^\/]*)\/?/); 51 | if (svgwg) { 52 | return { type: "custom", owner: "w3c", name: "svgwg" }; 53 | } 54 | if (url === "https://svgwg.org/svg2-draft/") { 55 | return { type: "custom", owner: "w3c", name: "svgwg" }; 56 | } 57 | 58 | const webgl = url.match(/^https:\/\/registry\.khronos\.org\/webgl\//); 59 | if (webgl) { 60 | return { type: "custom", owner: "khronosgroup", name: "WebGL" }; 61 | } 62 | 63 | const httpwg = url.match(/^https:\/\/httpwg\.org\/specs\/rfc[0-9]+\.html$/); 64 | if (httpwg) { 65 | return { type: "custom", owner: "httpwg", name: "httpwg.github.io" }; 66 | } 67 | 68 | const w3cTr = url.match(/^https?:\/\/(?:www\.)?w3\.org\/TR\/([^\/]+)\/$/); 69 | if (w3cTr) { 70 | return { type: "tr", owner: "w3c", name: w3cTr[1] }; 71 | } 72 | 73 | return null; 74 | } 75 | -------------------------------------------------------------------------------- /test/data.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Make sure that the src/data/*.json files respect the right JSON schema 3 | */ 4 | 5 | const assert = require("assert"); 6 | const schema = require("../schema/data.json"); 7 | const dfnsSchema = require("../schema/definitions.json"); 8 | const Ajv = require("ajv"); 9 | const addFormats = require("ajv-formats") 10 | const ajv = (new Ajv()).addSchema(dfnsSchema); 11 | addFormats(ajv); 12 | 13 | describe("Ignore/Monitor lists", () => { 14 | describe("The JSON schema", () => { 15 | it("is valid", () => { 16 | const isSchemaValid = ajv.validateSchema(schema); 17 | assert.ok(isSchemaValid); 18 | }); 19 | }); 20 | 21 | describe("The ignore list", () => { 22 | it("respects the JSON schema", () => { 23 | const list = require("../src/data/ignore.json"); 24 | const validate = ajv.compile(schema); 25 | const isValid = validate(list, { format: "full" }); 26 | assert.strictEqual(validate.errors, null); 27 | }); 28 | }); 29 | 30 | describe("The monitor list", () => { 31 | it("respects the JSON schema", () => { 32 | const list = require("../src/data/monitor.json"); 33 | const validate = ajv.compile(schema); 34 | const isValid = validate(list, { format: "full" }); 35 | assert.strictEqual(validate.errors, null); 36 | }); 37 | 38 | it("has lastreviewed dates for all entries", () => { 39 | const list = require("../src/data/monitor.json"); 40 | const wrongRepos = Object.entries(list.repos) 41 | .filter(([key, value]) => !value.lastreviewed) 42 | .map(([key, value]) => key); 43 | assert.deepStrictEqual(wrongRepos, []); 44 | 45 | const wrongSpecs = Object.entries(list.specs) 46 | .filter(([key, value]) => !value.lastreviewed) 47 | .map(([key, value]) => key); 48 | assert.deepStrictEqual(wrongSpecs, []); 49 | }); 50 | }); 51 | 52 | describe("An entry in one of the lists", () => { 53 | it("appears only once in the repos list", () => { 54 | const ignore = Object.keys(require("../src/data/ignore.json").repos); 55 | const monitor = Object.keys(require("../src/data/monitor.json").repos); 56 | const dupl = ignore.filter(key => monitor.find(k => k === key)) 57 | assert.deepStrictEqual(dupl, []); 58 | }); 59 | 60 | it("appears only once in the specs list", () => { 61 | const ignore = Object.keys(require("../src/data/ignore.json").specs); 62 | const monitor = Object.keys(require("../src/data/monitor.json").specs); 63 | const dupl = ignore.filter(key => monitor.find(k => k === key)) 64 | assert.deepStrictEqual(dupl, []); 65 | }); 66 | }); 67 | }); 68 | -------------------------------------------------------------------------------- /packages/web-specs/README.md: -------------------------------------------------------------------------------- 1 | # Web browser specifications 2 | 3 | This repository contains a curated list of technical Web specifications. 4 | 5 | This list is meant to be an up-to-date input source for projects that run 6 | analyses on web technologies to create reports on test coverage, 7 | cross-references, WebIDL, quality, etc. 8 | 9 | 10 | ## Table of Contents 11 | 12 | - [Installation and usage](#installation-and-usage) 13 | 14 | - [Spec selection criteria](#spec-selection-criteria) 15 | 16 | 17 | ## Installation and usage 18 | 19 | The list is distributed as an NPM package. To incorporate it to your project, 20 | run: 21 | 22 | ```bash 23 | npm install web-specs 24 | ``` 25 | 26 | You can then retrieve the list from your Node.js program: 27 | 28 | ```js 29 | const specs = require("web-specs"); 30 | console.log(JSON.stringify(specs, null, 2)); 31 | ``` 32 | 33 | Alternatively, you can fetch [`index.json`](https://w3c.github.io/browser-specs/index.json) 34 | or retrieve the list from the [`web-specs@latest` branch](https://github.com/w3c/browser-specs/tree/web-specs%40latest). 35 | 36 | 37 | 38 | 39 | ## Spec selection criteria 40 | 41 | This repository contains a curated list of technical Web specifications that are 42 | deemed relevant for the Web platform. Roughly speaking, this list should match 43 | the list of web specs actively developed by W3C, the WHATWG and a few other 44 | organizations. 45 | 46 | To try to make things more concrete, the following criteria are used to assess 47 | whether a spec should a priori appear in the list: 48 | 49 | 1. The spec is stable or in development. Superseded and abandoned specs will not 50 | appear in the list. For instance, the list contains the HTML LS spec, but not 51 | HTML 4.01 or HTML 5). 52 | 2. The spec is being developed by a well-known standardization or 53 | pre-standardization group. Today, this means a W3C Working Group or Community 54 | Group, the WHATWG, the IETF, the TC39 group or the Khronos Group. 55 | 4. The spec sits at the application layer or is "close to it". For instance, 56 | most IETF specs are likely out of scope, but some that are exposed to Web developers are in scope. 57 | 5. The spec defines normative content (terms, CSS, IDL), or it contains 58 | informative content that other specs often need to refer to (e.g. guidelines 59 | from horizontal activities such as accessibility, internationalization, privacy 60 | and security). 61 | 62 | There are and there will be exceptions to the rule. Besides, some of these 63 | criteria remain fuzzy and/or arbitrary, and we expect them to evolve over time, 64 | typically driven by needs expressed by projects that may want to use the list. -------------------------------------------------------------------------------- /test/compute-categories.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const computeCategories = require("../src/compute-categories.js"); 3 | 4 | describe("compute-categories module", () => { 5 | it("sets `browser` category by default", function () { 6 | const spec = { groups: [] }; 7 | assert.deepStrictEqual(computeCategories(spec), ["browser"]); 8 | }); 9 | 10 | it("sets `browser` category when group targets browsers", function () { 11 | const spec = { 12 | groups: [ { name: "Web Applications Working Group" } ] 13 | }; 14 | assert.deepStrictEqual(computeCategories(spec), ["browser"]); 15 | }); 16 | 17 | it("does not set a `browser` category when group does not target browsers", function () { 18 | const spec = { 19 | groups: [ { name: "Accessible Platform Architectures Working Group" } ] 20 | }; 21 | assert.deepStrictEqual(computeCategories(spec), []); 22 | }); 23 | 24 | it("does not set a `browser` category when repo does not target browsers", function () { 25 | const spec = { 26 | groups: [ { name: "Web Applications Working Group" } ], 27 | nightly: { repository: "https://github.com/w3c/dpub-aam" } 28 | }; 29 | assert.deepStrictEqual(computeCategories(spec), []); 30 | }); 31 | 32 | it("resets categories when asked to", function () { 33 | const spec = { 34 | groups: [ { name: "Web Applications Working Group" } ], 35 | categories: "reset" 36 | }; 37 | assert.deepStrictEqual(computeCategories(spec), []); 38 | }); 39 | 40 | it("drops browser when asked to", function () { 41 | const spec = { 42 | groups: [ { name: "Web Applications Working Group" } ], 43 | categories: "-browser" 44 | }; 45 | assert.deepStrictEqual(computeCategories(spec), []); 46 | }); 47 | 48 | it("adds browser when asked to", function () { 49 | const spec = { 50 | groups: [ { name: "Accessible Platform Architectures Working Group" } ], 51 | categories: "+browser" 52 | }; 53 | assert.deepStrictEqual(computeCategories(spec), ["browser"]); 54 | }); 55 | 56 | it("accepts an array of categories", function () { 57 | const spec = { 58 | groups: [ { name: "Accessible Platform Architectures Working Group" } ], 59 | categories: ["reset", "+browser"] 60 | }; 61 | assert.deepStrictEqual(computeCategories(spec), ["browser"]); 62 | }); 63 | 64 | it("throws if spec object is empty", () => { 65 | assert.throws( 66 | () => computeCategories({}), 67 | /^Invalid spec object passed as parameter$/); 68 | }); 69 | 70 | it("throws if spec object does not have a groups property", () => { 71 | assert.throws( 72 | () => computeCategories({ url: "https://example.org/" }), 73 | /^Invalid spec object passed as parameter$/); 74 | }); 75 | }); -------------------------------------------------------------------------------- /src/compute-categories.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a spec object that already has its 3 | * `groups` property (see `fetch-groups.js`) and its `nightly.repository` 4 | * property (see `compute-repository.js`) set as input, and that returns 5 | * a list of categories for the spec. 6 | * 7 | * Note (2022-02-08): The function merely sets `browser` for now. Logic (and 8 | * initial spec properties used to compute the list) will likely be adjusted 9 | * over time. 10 | */ 11 | 12 | // Retrieve the list of groups and repositories that we know don't contain 13 | // specs targeted at browsers. That logic will also very likely evolve over 14 | // time, be it only to give the file a different name (the list of specs will 15 | // be expanded to contain specs in that "ignore" list) 16 | const { groups: nonbrowserGroups, repos: nonbrowserRepos } = require('./data/ignore.json'); 17 | 18 | /** 19 | * Exports main function that takes a spec object and returns a list of 20 | * categories for the spec. 21 | * 22 | * Function may return an empty array. If the spec object contains a 23 | * `categories` property, the list of categories is adjusted accordingly. For 24 | * instance, if the spec object contains `+browser`, `browser` is added to the 25 | * list. If it contains `-browser`, `browser` won't appear in the list. If it 26 | * contains `reset`, the function does not attempt to compute a list but rather 27 | * returns the list of categories in the spec object. 28 | */ 29 | module.exports = function (spec) { 30 | if (!spec || !spec.groups) { 31 | throw "Invalid spec object passed as parameter"; 32 | } 33 | 34 | let list = []; 35 | const requestedCategories = (typeof spec.categories === "string") ? 36 | [spec.categories] : 37 | (spec.categories || []); 38 | 39 | // All specs target browsers by default unless the spec object says otherwise 40 | if (!requestedCategories.includes("reset")) { 41 | // Note (2022-02-08): This assumes that a non browser group that co-owns a 42 | // spec disqualifies the spec as a browser spec. This is true today but 43 | // potentially wobbly. 44 | const browserGroup = !spec.groups.find(group => nonbrowserGroups[group.name]); 45 | const browserRepo = !spec.nightly?.repository || 46 | !nonbrowserRepos[spec.nightly.repository.replace(/^https:\/\/github\.com\//, "")]; 47 | if (browserGroup && browserRepo) { 48 | list.push("browser"); 49 | } 50 | } 51 | 52 | // Apply requested incremental updates 53 | requestedCategories.filter(incr => (incr !== "reset")).forEach(incr => { 54 | const category = incr.substring(1); 55 | if (incr.startsWith("+")) { 56 | list.push(category); 57 | } 58 | else { 59 | list = list.filter(cat => cat !== category); 60 | } 61 | }); 62 | 63 | return list; 64 | } -------------------------------------------------------------------------------- /test/compute-shorttitle.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const computeShortTitle = require("../src/compute-shorttitle.js"); 3 | 4 | describe("compute-shorttitle module", () => { 5 | function assertTitle(title, expected) { 6 | const shortTitle = computeShortTitle(title); 7 | assert.equal(shortTitle, expected); 8 | } 9 | 10 | it("finds abbreviation for main CSS spec", () => { 11 | assertTitle( 12 | "Cascading Style Sheets Level 2 Revision 1 (CSS 2.1) Specification", 13 | "CSS 2.1"); 14 | }); 15 | 16 | it("does not choke on non-breaking spaces", () => { 17 | assertTitle( 18 | "CSS Backgrounds and Borders Module Level\u00A04", 19 | "CSS Backgrounds and Borders 4"); 20 | }); 21 | 22 | it("does not choke on levels that are not levels", () => { 23 | assertTitle( 24 | "CORS and RFC1918", 25 | "CORS and RFC1918"); 26 | }); 27 | 28 | it("finds abbreviation for WAI-ARIA title", () => { 29 | assertTitle( 30 | "Accessible Rich Internet Applications (WAI-ARIA) 1.2", 31 | "WAI-ARIA 1.2"); 32 | }); 33 | 34 | it("drops 'Level' from title but keeps level number", () => { 35 | assertTitle( 36 | "CSS Foo Level 42", 37 | "CSS Foo 42"); 38 | }); 39 | 40 | it("drops 'Module' from title but keeps level number", () => { 41 | assertTitle( 42 | "CSS Foo Module Level 42", 43 | "CSS Foo 42"); 44 | }); 45 | 46 | it("drops '- Level' from title", () => { 47 | assertTitle( 48 | "Foo - Level 2", 49 | "Foo 2"); 50 | }); 51 | 52 | it("drops 'Module - Level' from title", () => { 53 | assertTitle( 54 | "Foo Module - Level 3", 55 | "Foo 3"); 56 | }); 57 | 58 | it("drops 'Specification' from end of title", () => { 59 | assertTitle( 60 | "Foo Specification", 61 | "Foo"); 62 | }); 63 | 64 | it("drops 'Standard' from end of title", () => { 65 | assertTitle( 66 | "Foo Standard", 67 | "Foo"); 68 | }); 69 | 70 | it("drops 'Living Standard' from end of title", () => { 71 | assertTitle( 72 | "Foo Living Standard", 73 | "Foo"); 74 | }); 75 | 76 | it("drops edition indications", () => { 77 | assertTitle( 78 | "Foo (Second Edition) Bar", 79 | "Foo Bar"); 80 | }); 81 | 82 | it("preserves title when needed", () => { 83 | assertTitle( 84 | "Edition Module Standard Foo", 85 | "Edition Module Standard Foo"); 86 | }); 87 | 88 | it("drops 'Proposal' from end of title", () => { 89 | assertTitle( 90 | "Hello world API Proposal", 91 | "Hello world API"); 92 | }); 93 | 94 | it("preserves scope in HTTP/1.1 spec titles", () => { 95 | assertTitle( 96 | "Hypertext Transfer Protocol (HTTP/1.1): Foo bar", 97 | "HTTP/1.1 Foo bar") 98 | }); 99 | }); 100 | -------------------------------------------------------------------------------- /test/compute-repository.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const computeRepo = require("../src/compute-repository.js"); 3 | 4 | describe("compute-repository module", async () => { 5 | async function computeSingleRepo(url) { 6 | const spec = { nightly: { url } }; 7 | const result = await computeRepo([spec]); 8 | return result[0].nightly.repository; 9 | }; 10 | 11 | it("handles github.com URLs", async () => { 12 | assert.equal( 13 | await computeSingleRepo("https://github.com/orgname/specname"), 14 | "https://github.com/orgname/specname"); 15 | }); 16 | 17 | it("handles xxx.github.io URLs", async () => { 18 | assert.equal( 19 | await computeSingleRepo("https://orgname.github.io/specname"), 20 | "https://github.com/orgname/specname"); 21 | }); 22 | 23 | it("handles xxx.github.io URLs with trailing slash", async () => { 24 | assert.equal( 25 | await computeSingleRepo("https://orgname.github.io/specname/"), 26 | "https://github.com/orgname/specname"); 27 | }); 28 | 29 | it("handles WHATWG URLs", async () => { 30 | assert.equal( 31 | await computeSingleRepo("https://specname.spec.whatwg.org/"), 32 | "https://github.com/whatwg/specname"); 33 | }); 34 | 35 | it("handles TC39 URLs", async () => { 36 | assert.equal( 37 | await computeSingleRepo("https://tc39.es/js-ftw/"), 38 | "https://github.com/tc39/js-ftw"); 39 | }); 40 | 41 | it("handles CSS WG URLs", async () => { 42 | assert.equal( 43 | await computeSingleRepo("https://drafts.csswg.org/css-everything-42/"), 44 | "https://github.com/w3c/csswg-drafts"); 45 | }); 46 | 47 | it("handles FX TF URLs", async () => { 48 | assert.equal( 49 | await computeSingleRepo("https://drafts.fxtf.org/wow/"), 50 | "https://github.com/w3c/fxtf-drafts"); 51 | }); 52 | 53 | it("handles CSS Houdini URLs", async () => { 54 | assert.equal( 55 | await computeSingleRepo("https://drafts.css-houdini.org/magic-11/"), 56 | "https://github.com/w3c/css-houdini-drafts"); 57 | }); 58 | 59 | it("handles SVG WG URLs", async () => { 60 | assert.equal( 61 | await computeSingleRepo("https://svgwg.org/specs/svg-ftw"), 62 | "https://github.com/w3c/svgwg"); 63 | }); 64 | 65 | it("handles the SVG2 URL", async () => { 66 | assert.equal( 67 | await computeSingleRepo("https://svgwg.org/svg2-draft/"), 68 | "https://github.com/w3c/svgwg"); 69 | }); 70 | 71 | it("handles WebGL URLs", async () => { 72 | assert.equal( 73 | await computeSingleRepo("https://registry.khronos.org/webgl/specs/latest/1.0/"), 74 | "https://github.com/khronosgroup/WebGL"); 75 | }); 76 | 77 | it("returns null when repository cannot be derived from URL", async () => { 78 | assert.equal( 79 | await computeSingleRepo("https://example.net/repoless"), 80 | null); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /src/compute-series-urls.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a spec object as input that already 3 | * has most of its info filled out ("series", but also "release" and "nightly" 4 | * properties filled out) and that returns an object with a "releaseUrl" and 5 | * "nightlyUrl" property when possible that target the unversioned versions, of 6 | * the spec, in other words the series itself. 7 | * 8 | * The function also takes the list of spec objects as second parameter. When 9 | * computing the release URL, it will iterate through the specs in the same 10 | * series to find one that has a release URL. 11 | */ 12 | 13 | function computeSeriesUrls(spec) { 14 | if (!spec?.shortname || !spec.series?.shortname) { 15 | throw "Invalid spec object passed as parameter"; 16 | } 17 | 18 | const res = {}; 19 | 20 | // If spec shortname and series shortname match, then series URLs match the 21 | // spec URLs. 22 | if (spec.shortname === spec.series.shortname) { 23 | if (spec.release?.url) { 24 | res.releaseUrl = spec.release.url; 25 | } 26 | if (spec.nightly?.url) { 27 | res.nightlyUrl = spec.nightly.url; 28 | } 29 | } 30 | 31 | // When shortnames do not match, replace the spec shortname by the series 32 | // shortname in the URL 33 | else { 34 | if (spec.release?.url) { 35 | res.releaseUrl = spec.release.url.replace( 36 | new RegExp(`/${spec.shortname}/`), 37 | `/${spec.series.shortname}/`); 38 | } 39 | if (spec.nightly?.url) { 40 | res.nightlyUrl = spec.nightly.url.replace( 41 | new RegExp(`/${spec.shortname}/`), 42 | `/${spec.series.shortname}/`); 43 | } 44 | } 45 | 46 | return res; 47 | } 48 | 49 | /** 50 | * Exports main function that takes a spec object and returns an object with 51 | * properties "releaseUrl" and "nightlyUrl". Function only sets the properties 52 | * when needed, so returned object may be empty. 53 | * 54 | * Function also takes the list of spec objects as input. It iterates through 55 | * the list to look for previous versions of a spec to find a suitable release 56 | * URL when the latest version does not have one. 57 | */ 58 | module.exports = function (spec, list) { 59 | list = list || []; 60 | 61 | // Compute series info for current version of the spec if it is in the list 62 | const currentSpec = list.find(s => s.shortname === spec.series?.currentSpecification); 63 | const res = computeSeriesUrls(currentSpec ?? spec); 64 | 65 | // Look for a release URL in previous versions of the spec if one exists 66 | if (!res.releaseUrl) { 67 | while (spec.seriesPrevious) { 68 | spec = list.find(s => s.shortname === spec.seriesPrevious); 69 | if (!spec) { 70 | break; 71 | } 72 | const prev = computeSeriesUrls(spec); 73 | if (prev.releaseUrl) { 74 | res.releaseUrl = prev.releaseUrl; 75 | break; 76 | } 77 | } 78 | } 79 | return res; 80 | } -------------------------------------------------------------------------------- /src/prepare-packages.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Prepare the contents of the NPM packages 3 | * 4 | * NPM packages include browser-specs. 5 | * 6 | * These packages contain a filtered view of the list of specs. 7 | * 8 | * The script copies relevant files to the "packages" folders. 9 | * 10 | * node src/prepare-packages.js 11 | */ 12 | 13 | const fs = require('fs').promises; 14 | const path = require('path'); 15 | const util = require('util'); 16 | 17 | async function preparePackages() { 18 | console.log('Load index file'); 19 | const index = require(path.join('..', 'index.json')); 20 | console.log(`- ${index.length} specs in index file`); 21 | 22 | const packages = [ 23 | { 24 | name: 'web-specs', 25 | filter: spec => true 26 | }, 27 | { 28 | name: 'browser-specs', 29 | filter: spec => spec.categories?.includes('browser') 30 | } 31 | ]; 32 | 33 | for (const { name, filter } of packages) { 34 | console.log(); 35 | console.log(`Prepare the ${name} package`); 36 | 37 | // Only keep relevant specs targeted at browsers 38 | const specs = index.filter(filter); 39 | console.log(`- ${specs.length}/${index.length} specs to include in the package`); 40 | 41 | // Write packages/${name}/index.json 42 | await fs.writeFile( 43 | path.resolve(__dirname, '..', 'packages', name, 'index.json'), 44 | JSON.stringify(specs, null, 2), 45 | 'utf8'); 46 | console.log(`- packages/${name}/index.json updated`); 47 | 48 | // Update README.md 49 | const commonReadme = await fs.readFile(path.resolve(__dirname, '..', 'README.md'), 'utf8'); 50 | const packageReadmeFile = path.resolve(__dirname, '..', 'packages', name, 'README.md'); 51 | let packageReadme = await fs.readFile(packageReadmeFile, 'utf8'); 52 | const commonBlocks = [ 53 | { start: '', end: '' }, 54 | { start: '', end: '' } 55 | ]; 56 | for (const { start, end } of commonBlocks) { 57 | const [commonStart, commonEnd] = [commonReadme.indexOf(start), commonReadme.indexOf(end)]; 58 | const [packageStart, packageEnd] = [packageReadme.indexOf(start), packageReadme.indexOf(end)]; 59 | const commonBlock = commonReadme.substring(commonStart, commonEnd); 60 | packageReadme = packageReadme.substring(0, packageStart) + 61 | commonBlock + 62 | packageReadme.substring(packageEnd); 63 | } 64 | await fs.writeFile(packageReadmeFile, packageReadme, 'utf8'); 65 | console.log(`- packages/${name}/README.md updated`); 66 | } 67 | } 68 | 69 | /******************************************************************************* 70 | Kick things off 71 | *******************************************************************************/ 72 | preparePackages() 73 | .then(() => { 74 | console.log(); 75 | console.log("== The end =="); 76 | }) 77 | .catch(err => { 78 | console.error(err); 79 | process.exit(1); 80 | }); -------------------------------------------------------------------------------- /src/request-pr-review.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Request a review on a pending pre-release PR 3 | */ 4 | 5 | const Octokit = require("./octokit"); 6 | 7 | // Repository to process and PR reviewers 8 | const owner = "w3c"; 9 | const repo = "browser-specs"; 10 | const reviewers = ["dontcallmedom", "tidoust"]; 11 | 12 | 13 | /** 14 | * Create or update pre-release pull request 15 | * 16 | * @function 17 | * @param {String} type Package name 18 | */ 19 | async function requestReview(type) { 20 | console.log(`Check pre-release PR for the ${type} package`); 21 | const searchResponse = await octokit.search.issuesAndPullRequests({ 22 | q: `repo:${owner}/${repo} type:pr state:open head:release-${type}-` 23 | }); 24 | const found = searchResponse?.data?.items?.[0]; 25 | 26 | const pendingPRResponse = found ? 27 | await octokit.pulls.get({ 28 | owner, repo, 29 | pull_number: found.number 30 | }) : 31 | null; 32 | const pendingPR = pendingPRResponse?.data; 33 | console.log(pendingPR ? 34 | `- Found pending pre-release PR: ${pendingPR.title} (#${pendingPR.number})` : 35 | "- No pending pre-release PR"); 36 | if (!pendingPR) { 37 | return; 38 | } 39 | 40 | console.log(`- Targeted list of reviewers: ${reviewers.join(", ")}`); 41 | console.log(`- Pending PR was created by: ${pendingPR.user.login}`); 42 | const currentReviewers = pendingPR.requested_reviewers.map(r => r.login); 43 | console.log(`- Current reviewers: ${currentReviewers.length > 0 ? currentReviewers.join(", ") : "none"}`); 44 | const reviewersToAdd = reviewers.filter(login => !currentReviewers.includes(login) && pendingPR.user.login !== login); 45 | console.log(`- Reviewers to add: ${reviewersToAdd.length > 0 ? reviewersToAdd.join(", ") : "none"}`); 46 | if (reviewersToAdd.length > 0) { 47 | await octokit.pulls.requestReviewers({ 48 | owner, 49 | repo, 50 | pull_number: pendingPR.number, 51 | reviewers: reviewersToAdd 52 | }); 53 | console.log("- Reviewers added"); 54 | } 55 | } 56 | 57 | 58 | /******************************************************************************* 59 | Retrieve GH_TOKEN from environment, prepare Octokit and kick things off 60 | *******************************************************************************/ 61 | const GH_TOKEN = (_ => { 62 | try { 63 | return require("../config.json").GH_TOKEN; 64 | } 65 | catch { 66 | return ""; 67 | } 68 | })() || process.env.GH_TOKEN; 69 | if (!GH_TOKEN) { 70 | console.error("GH_TOKEN must be set to some personal access token as an env variable or in a config.json file"); 71 | process.exit(1); 72 | } 73 | 74 | const octokit = new Octokit({ 75 | auth: GH_TOKEN, 76 | //log: console 77 | }); 78 | 79 | requestReview("web-specs") 80 | .then(() => console.log()) 81 | .then(() => requestReview("browser-specs")) 82 | .then(() => { 83 | console.log(); 84 | console.log("== The end =="); 85 | }) 86 | .catch(err => { 87 | console.error(err); 88 | process.exit(1); 89 | }); -------------------------------------------------------------------------------- /packages/browser-specs/README.md: -------------------------------------------------------------------------------- 1 | # Web browser specifications 2 | 3 | This repository contains a curated list of technical Web specifications that are 4 | directly implemented or that will be implemented by Web browsers. 5 | 6 | This list is meant to be an up-to-date input source for projects that run 7 | analyses on browser technologies to create reports on test coverage, 8 | cross-references, WebIDL, quality, etc. 9 | 10 | 11 | ## Table of Contents 12 | 13 | - [Installation and usage](#installation-and-usage) 14 | 15 | - [Spec selection criteria](#spec-selection-criteria) 16 | 17 | ## Installation and usage 18 | 19 | The list is distributed as an NPM package. To incorporate it to your project, 20 | run: 21 | 22 | ```bash 23 | npm install browser-specs 24 | ``` 25 | 26 | You can then retrieve the list from your Node.js program: 27 | 28 | ```js 29 | const specs = require("browser-specs"); 30 | console.log(JSON.stringify(specs, null, 2)); 31 | ``` 32 | 33 | Alternatively, you can fetch [`index.json`](https://w3c.github.io/browser-specs/index.json) 34 | or retrieve the list from the [`web-specs@latest` branch](https://github.com/w3c/browser-specs/tree/web-specs%40latest), 35 | and filter the resulting list to only preserve specs that have `"browser"` in 36 | their `categories` property. 37 | 38 | 39 | 40 | 41 | ## Spec selection criteria 42 | 43 | This repository contains a curated list of technical Web specifications that are 44 | deemed relevant for Web browsers. Roughly speaking, this list should match the 45 | list of specs that appear in projects such as [Web Platform 46 | Tests](https://github.com/web-platform-tests/wpt) or 47 | [MDN](https://developer.mozilla.org/). 48 | 49 | To try to make things more concrete, the following criteria are used to assess 50 | whether a spec should a priori appear in the list: 51 | 52 | 1. The spec is stable or in development. Superseded and abandoned specs will not 53 | appear in the list. For instance, the list contains the HTML LS spec, but not 54 | HTML 4.01 or HTML 5). 55 | 2. The spec is being developed by a well-known standardization or 56 | pre-standardization group. Today, this means a W3C Working Group or Community 57 | Group, the WHATWG, the IETF, the TC39 group or the Khronos Group. 58 | 3. Web browsers expressed some level of support for the spec, e.g. through a 59 | public intent to implement. 60 | 4. The spec sits at the application layer or is "close to it". For instance, 61 | most IETF specs are likely out of scope, but some that are exposed to Web developers are in scope. 62 | 5. The spec defines normative content (terms, CSS, IDL), or it contains 63 | informative content that other specs often need to refer to (e.g. guidelines 64 | from horizontal activities such as accessibility, internationalization, privacy 65 | and security). 66 | 67 | There are and there will be exceptions to the rule. Besides, some of these 68 | criteria remain fuzzy and/or arbitrary, and we expect them to evolve over time, 69 | typically driven by needs expressed by projects that may want to use the list. -------------------------------------------------------------------------------- /test/fetch-info.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Tests for the fetch-info module that do not require a W3C API key 3 | * 4 | * These tests are separated from the tests that require a W3C API key because 5 | * the key cannot be exposed on pull requests from forked repositories on 6 | * GitHub. 7 | */ 8 | 9 | const assert = require("assert"); 10 | const fetchInfo = require("../src/fetch-info.js"); 11 | 12 | describe("fetch-info module (without W3C API key)", function () { 13 | // Tests need to send network requests 14 | this.slow(5000); 15 | this.timeout(30000); 16 | 17 | function getW3CSpec(shortname) { 18 | return { shortname, url: `https://www.w3.org/TR/${shortname}/` }; 19 | } 20 | 21 | 22 | describe("fetch from Specref", () => { 23 | it("works on a TR spec in the absence of W3C API key", async () => { 24 | const spec = getW3CSpec("presentation-api"); 25 | const info = await fetchInfo([spec]); 26 | assert.ok(info[spec.shortname]); 27 | assert.equal(info[spec.shortname].source, "specref"); 28 | assert.equal(info[spec.shortname].nightly.url, "https://w3c.github.io/presentation-api/"); 29 | assert.equal(info[spec.shortname].title, "Presentation API"); 30 | }); 31 | 32 | it("works on a WHATWG spec", async () => { 33 | const spec = { 34 | url: "https://dom.spec.whatwg.org/", 35 | shortname: "dom" 36 | }; 37 | const info = await fetchInfo([spec]); 38 | assert.ok(info[spec.shortname]); 39 | assert.equal(info[spec.shortname].source, "specref"); 40 | assert.equal(info[spec.shortname].nightly.url, "https://dom.spec.whatwg.org/"); 41 | assert.equal(info[spec.shortname].title, "DOM Standard"); 42 | }); 43 | 44 | it("can operate on multiple specs at once", async () => { 45 | const spec = getW3CSpec("presentation-api"); 46 | const other = getW3CSpec("hr-time-2"); 47 | const info = await fetchInfo([spec, other]); 48 | assert.ok(info[spec.shortname]); 49 | assert.equal(info[spec.shortname].source, "specref"); 50 | assert.equal(info[spec.shortname].nightly.url, "https://w3c.github.io/presentation-api/"); 51 | assert.equal(info[spec.shortname].title, "Presentation API"); 52 | 53 | assert.ok(info[other.shortname]); 54 | assert.equal(info[other.shortname].source, "specref"); 55 | assert.equal(info[other.shortname].nightly.url, "https://w3c.github.io/hr-time/"); 56 | assert.equal(info[other.shortname].title, "High Resolution Time Level 2"); 57 | }); 58 | }); 59 | 60 | 61 | describe("fetch from spec", () => { 62 | it("extracts spec info from a Bikeshed spec when needed", async () => { 63 | const spec = { 64 | url: "https://tabatkins.github.io/bikeshed/", 65 | shortname: "bikeshed" 66 | }; 67 | const info = await fetchInfo([spec]); 68 | assert.ok(info[spec.shortname]); 69 | assert.equal(info[spec.shortname].source, "spec"); 70 | assert.equal(info[spec.shortname].nightly.url, spec.url); 71 | assert.equal(info[spec.shortname].title, "Bikeshed Documentation"); 72 | }); 73 | 74 | it("extracts spec info from a Respec spec when needed", async () => { 75 | const spec = { 76 | url: "https://w3c.github.io/respec/examples/tpac_2019.html", 77 | shortname: "respec" 78 | }; 79 | const info = await fetchInfo([spec]); 80 | assert.ok(info[spec.shortname]); 81 | assert.equal(info[spec.shortname].source, "spec"); 82 | assert.equal(info[spec.shortname].nightly.url, spec.url); 83 | assert.equal(info[spec.shortname].title, "TPAC 2019 - New Features"); 84 | }); 85 | 86 | it("extracts right title from an ECMAScript proposal spec", async () => { 87 | const spec = { 88 | url: "https://tc39.es/proposal-intl-segmenter/", 89 | shortname: "tc39-intl-segmenter" 90 | }; 91 | const info = await fetchInfo([spec]); 92 | assert.ok(info[spec.shortname]); 93 | assert.equal(info[spec.shortname].source, "spec"); 94 | assert.equal(info[spec.shortname].nightly.url, spec.url); 95 | assert.equal(info[spec.shortname].title, "Intl.Segmenter Proposal"); 96 | }); 97 | }); 98 | }); -------------------------------------------------------------------------------- /test/compute-currentlevel.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const computeCurrentLevel = require("../src/compute-currentlevel.js"); 3 | 4 | describe("compute-currentlevel module", () => { 5 | function getCurrentName(spec, list) { 6 | return computeCurrentLevel(spec, list).currentSpecification; 7 | } 8 | function getSpec(options) { 9 | options = options || {}; 10 | const res = { 11 | shortname: options.shortname ?? (options.seriesVersion ? `spec-${options.seriesVersion}` : "spec"), 12 | series: { shortname: "spec" }, 13 | }; 14 | for (const property of Object.keys(options)) { 15 | res[property] = options[property]; 16 | } 17 | return res; 18 | } 19 | function getOther(options) { 20 | options = options || {}; 21 | const res = { 22 | shortname: (options.seriesVersion ? `other-${options.seriesVersion}` : "other"), 23 | series: { shortname: "other" }, 24 | }; 25 | for (const property of Object.keys(options)) { 26 | res[property] = options[property]; 27 | } 28 | return res; 29 | } 30 | 31 | it("returns the spec name if list is empty", () => { 32 | const spec = getSpec(); 33 | assert.equal(getCurrentName(spec), spec.shortname); 34 | }); 35 | 36 | it("returns the name of the latest level", () => { 37 | const spec = getSpec({ seriesVersion: "1" }); 38 | const current = getSpec({ seriesVersion: "2" }); 39 | assert.equal( 40 | getCurrentName(spec, [spec, current]), 41 | current.shortname); 42 | }); 43 | 44 | it("returns the name of the latest level that is not a delta spec", () => { 45 | const spec = getSpec({ seriesVersion: "1" }); 46 | const delta = getSpec({ seriesVersion: "2", seriesComposition: "delta" }); 47 | assert.equal( 48 | getCurrentName(spec, [spec, delta]), 49 | spec.shortname); 50 | }); 51 | 52 | it("returns the name of the latest level that is not a fork spec", () => { 53 | const spec = getSpec({ seriesVersion: "1" }); 54 | const fork = getSpec({ seriesVersion: "2", seriesComposition: "fork" }); 55 | assert.equal( 56 | getCurrentName(spec, [spec, fork]), 57 | spec.shortname); 58 | }); 59 | 60 | it("gets back to the latest level when spec is a delta spec", () => { 61 | const spec = getSpec({ seriesVersion: "1" }); 62 | const delta = getSpec({ seriesVersion: "2", seriesComposition: "delta" }); 63 | assert.equal( 64 | getCurrentName(delta, [spec, delta]), 65 | spec.shortname); 66 | }); 67 | 68 | it("gets back to the latest level when spec is a fork spec", () => { 69 | const spec = getSpec({ seriesVersion: "1" }); 70 | const fork = getSpec({ seriesVersion: "2", seriesComposition: "fork" }); 71 | assert.equal( 72 | getCurrentName(fork, [spec, fork]), 73 | spec.shortname); 74 | }); 75 | 76 | it("returns the spec name if it is flagged as current", () => { 77 | const spec = getSpec({ seriesVersion: "1", forceCurrent: true }); 78 | const last = getSpec({ seriesVersion: "2" }); 79 | assert.equal( 80 | getCurrentName(spec, [spec, last]), 81 | spec.shortname); 82 | }); 83 | 84 | it("returns the name of the level flagged as current", () => { 85 | const spec = getSpec({ seriesVersion: "1" }); 86 | const current = getSpec({ seriesVersion: "2", forceCurrent: true }); 87 | const last = getSpec({ seriesVersion: "3" }); 88 | assert.equal( 89 | getCurrentName(spec, [spec, current, last]), 90 | current.shortname); 91 | }); 92 | 93 | it("does not take other shortnames into account", () => { 94 | const spec = getSpec({ seriesVersion: "1" }); 95 | const other = getOther({ seriesVersion: "2"}); 96 | assert.equal( 97 | getCurrentName(spec, [spec, other]), 98 | spec.shortname); 99 | }); 100 | 101 | it("does not take forks into account", () => { 102 | const spec = getSpec({ shortname: "spec-1-fork-1", seriesVersion: "1", seriesComposition: "fork" }); 103 | const base = getSpec({ seriesVersion: "1" }); 104 | assert.equal( 105 | getCurrentName(spec, [spec, base]), 106 | base.shortname); 107 | }); 108 | }); -------------------------------------------------------------------------------- /src/build-diff.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const { execSync } = require("child_process"); 3 | const { generateIndex } = require("./build-index"); 4 | 5 | function compareSpecs(a, b) { 6 | return a.url.localeCompare(b.url); 7 | } 8 | 9 | /** 10 | * Generate the new index file from the given initial list file. 11 | * 12 | * The function throws in case of errors. 13 | */ 14 | async function compareIndex(newRef, baseRef, { diffType = "diff", log = console.log }) { 15 | log(`Retrieve specs.json at "${newRef}"...`); 16 | let newSpecs; 17 | if (newRef.toLowerCase() === "working") { 18 | newSpecs = require(path.resolve(__dirname, "..", "specs.json")); 19 | } 20 | else { 21 | const newSpecsStr = execSync(`git show ${newRef}:specs.json`, { encoding: "utf8" }); 22 | newSpecs = JSON.parse(newSpecsStr); 23 | } 24 | log(`Retrieve specs.json at "${newRef}"... done`); 25 | 26 | log(`Retrieve specs.json at "${baseRef}"...`); 27 | const baseSpecsStr = execSync(`git show ${baseRef}:specs.json`, { encoding: "utf8" }); 28 | const baseSpecs = JSON.parse(baseSpecsStr); 29 | log(`Retrieve specs.json at "${baseRef}"... done`); 30 | 31 | log(`Retrieve index.json at "${baseRef}"...`); 32 | const baseIndexStr = execSync(`git show ${baseRef}:index.json`, { encoding: "utf8" }); 33 | const baseIndex = JSON.parse(baseIndexStr); 34 | log(`Retrieve index.json at "${baseRef}"... done`); 35 | 36 | log(`Compute specs.json diff...`); 37 | function hasSameUrl(s1, s2) { 38 | const url1 = (typeof s1 === "string") ? s1 : s1.url; 39 | const url2 = (typeof s2 === "string") ? s2 : s2.url; 40 | return url1 === url2; 41 | } 42 | const diff = { 43 | added: newSpecs.filter(spec => !baseSpecs.find(s => hasSameUrl(s, spec))), 44 | updated: newSpecs.filter(spec => 45 | !!baseSpecs.find(s => hasSameUrl(s, spec)) && 46 | !baseSpecs.find(s => JSON.stringify(s) === JSON.stringify(spec))), 47 | deleted: baseSpecs.filter(spec => !newSpecs.find(s => hasSameUrl(s, spec))) 48 | }; 49 | log(`Compute specs.json diff... done`); 50 | 51 | log(`Build specs that were added...`); 52 | diff.added = (diff.added.length === 0) ? [] : 53 | await generateIndex(diff.added, { 54 | previousIndex: baseIndex, 55 | log: function(...msg) { log(' ', ...msg); } }); 56 | log(`Build specs that were added... done`); 57 | 58 | log(`Build specs that were updated...`); 59 | diff.updated = (diff.updated.length === 0) ? [] : 60 | await generateIndex(diff.updated, { 61 | previousIndex: baseIndex, 62 | log: function(...msg) { log(' ', ...msg); } }); 63 | log(`Build specs that were updated... done`); 64 | 65 | log(`Retrieve specs that were dropped...`); 66 | diff.deleted = diff.deleted.map(spec => baseIndex.find(s => hasSameUrl(s, spec))); 67 | log(`Retrieve specs that were dropped... done`); 68 | 69 | if (diffType === "full") { 70 | log(`Create full new index...`); 71 | const newIndex = baseIndex 72 | .map(spec => { 73 | const updated = diff.updated.find(s => hasSameUrl(s, spec)); 74 | return updated ?? spec; 75 | }) 76 | .filter(spec => !diff.deleted.find(s => hasSameUrl(s, spec))); 77 | diff.added.forEach(spec => newIndex.push(spec)); 78 | newIndex.sort(compareSpecs); 79 | log(`Create full new index... done`); 80 | return newIndex; 81 | } 82 | else { 83 | return diff; 84 | } 85 | } 86 | 87 | 88 | /******************************************************************************* 89 | Main loop 90 | *******************************************************************************/ 91 | const newRef = process.argv[2] ?? "working"; 92 | const baseRef = process.argv[3] ?? "HEAD"; 93 | const diffType = process.argv[4] ?? "diff"; 94 | 95 | compareIndex(newRef, baseRef, { diffType, log: console.warn }) 96 | .then(diff => { 97 | // Note: using process.stdout.write to avoid creating a final newline in 98 | // "full" diff mode. This makes it easier to compare the result with the 99 | // index.json file in the repo (which does not have a final newline). 100 | process.stdout.write(JSON.stringify(diff, null, 2)); 101 | }) 102 | .catch(err => { 103 | console.error(err); 104 | process.exit(1); 105 | }); 106 | -------------------------------------------------------------------------------- /src/bump-packages-minor.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Bump the minor version of packages when the list of specs has changed. 3 | * 4 | * node src/bump-packages-minor.js 5 | * 6 | * This script is intended to be run at the end of a build before committing 7 | * the result back to the main branch to automatically bump the minor version 8 | * in the `package.json` files under the `packages` folders when the new index 9 | * files contains new/deleted specs to commit. 10 | * 11 | * The script does not bump a version that matches x.y.0 because such a version 12 | * means a minor bump is already pending release. 13 | */ 14 | 15 | const fs = require('fs').promises; 16 | const path = require('path'); 17 | const { execSync } = require('child_process'); 18 | 19 | function specsMatch(s1, s2) { 20 | return s1.url === s2.url && s1.shortname === s2.shortname; 21 | } 22 | 23 | function isMinorBumpNeeded(type) { 24 | // Retrieve the fullname of the remote ref "${type}@latest" 25 | const refs = execSync(`git show-ref ${type}@latest`, { encoding: 'utf8' }) 26 | .trim().split('\n').map(ref => ref.split(' ')[1]) 27 | .filter(ref => ref.startsWith('refs/remotes/')); 28 | if (refs.length > 1) { 29 | throw new Error(`More than one remote refs found for ${type}@latest`); 30 | } 31 | if (refs === 0) { 32 | throw new Error(`No remote ref found for ${type}@latest`); 33 | } 34 | 35 | // Retrieve contents of last released index file 36 | const res = execSync( 37 | `git show ${refs[0]}:index.json`, 38 | { encoding: 'utf8' }).trim(); 39 | let lastIndexFile = JSON.parse(res); 40 | 41 | // Load new file 42 | let newIndexFile = require('../index.json'); 43 | 44 | // Filter specs if needed 45 | if (type === "browser-specs") { 46 | lastIndexFile = lastIndexFile.filter(s => !s.categories || s.categories.includes('browser')); 47 | newIndexFile = newIndexFile.filter(s => s.categories.includes('browser')); 48 | } 49 | 50 | return !!( 51 | lastIndexFile.find(spec => !newIndexFile.find(s => specsMatch(spec, s))) || 52 | newIndexFile.find(spec => !lastIndexFile.find(s => specsMatch(spec, s))) 53 | ); 54 | } 55 | 56 | 57 | async function checkPackage(type) { 58 | console.log(`Check ${type} package`); 59 | const packageFile = path.join('..', 'packages', type, 'package.json'); 60 | const package = require(packageFile); 61 | const version = package.version; 62 | console.log(`- Current version: ${version}`); 63 | 64 | // Loosely adapted from semver: 65 | // https://github.com/npm/node-semver/blob/cb1ca1d5480a6c07c12ac31ba5f2071ed530c4ed/internal/re.js#L37 66 | // (not using semver directly to avoid having to install dependencies in job) 67 | const reVersion = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; 68 | const versionTokens = version.match(reVersion); 69 | const major = parseInt(versionTokens[1], 10); 70 | const minor = parseInt(versionTokens[2], 10); 71 | const patch = parseInt(versionTokens[3], 10); 72 | 73 | if (patch === 0) { 74 | console.log('- No bump needed, minor bump already pending'); 75 | return; 76 | } 77 | 78 | if (isMinorBumpNeeded(type)) { 79 | console.log('- new/deleted spec(s) found'); 80 | const newVersion = `${major}.${minor+1}.0`; 81 | package.version = newVersion; 82 | fs.writeFile(path.resolve(__dirname, packageFile), JSON.stringify(package, null, 2), 'utf8'); 83 | console.log(`- Version bumped to ${newVersion}`); 84 | } 85 | else { 86 | console.log('- No bump needed'); 87 | } 88 | } 89 | 90 | 91 | async function checkPackages() { 92 | const packagesFolder = path.resolve(__dirname, '..', 'packages'); 93 | const types = await fs.readdir(packagesFolder); 94 | for (const type of types) { 95 | const stat = await fs.lstat(path.join(packagesFolder, type)); 96 | if (stat.isDirectory()) { 97 | await checkPackage(type); 98 | } 99 | } 100 | } 101 | 102 | 103 | /******************************************************************************* 104 | Main loop 105 | *******************************************************************************/ 106 | checkPackages() 107 | .then(() => { 108 | console.log(); 109 | console.log("== The end =="); 110 | }) 111 | .catch(err => { 112 | console.error(err); 113 | process.exit(1); 114 | }); 115 | -------------------------------------------------------------------------------- /src/lint.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs").promises; 4 | const computeShortname = require("./compute-shortname.js"); 5 | const computePrevNext = require("./compute-prevnext.js"); 6 | 7 | 8 | function compareSpecs(a, b) { 9 | return a.url.localeCompare(b.url); 10 | } 11 | 12 | 13 | // Shorten definition of spec to more human-readable version 14 | function shortenDefinition(spec) { 15 | const short = {}; 16 | for (const property of Object.keys(spec)) { 17 | if (!((property === "seriesComposition" && spec[property] === "full") || 18 | (property === "seriesComposition" && spec[property] === "fork") || 19 | (property === "multipage" && !spec[property]) || 20 | (property === "forceCurrent" && !spec[property]))) { 21 | short[property] = spec[property]; 22 | } 23 | } 24 | if (Object.keys(short).length === 1) { 25 | return short.url; 26 | } 27 | else if (Object.keys(short).length === 2 && 28 | spec.seriesComposition === "delta") { 29 | return `${spec.url} delta`; 30 | } 31 | else if (Object.keys(short).length === 2 && 32 | spec.forceCurrent) { 33 | return `${spec.url} current`; 34 | } 35 | else if (Object.keys(short).length === 2 && 36 | spec.multipage) { 37 | return `${spec.url} multipage`; 38 | } 39 | else { 40 | return short; 41 | } 42 | } 43 | 44 | 45 | // Lint specs list defined as a JSON string 46 | function lintStr(specsStr) { 47 | const specs = JSON.parse(specsStr); 48 | 49 | // Normalize end of lines, different across platforms, for comparison 50 | specsStr = specsStr.replace(/\r\n/g, "\n"); 51 | 52 | // Convert entries to spec objects, drop duplicates, and sort per URL 53 | const sorted = specs 54 | .map(spec => (typeof spec === "string") ? 55 | { 56 | url: new URL(spec.split(" ")[0]).toString(), 57 | seriesComposition: (spec.split(' ')[1] === "delta") ? "delta" : "full", 58 | forceCurrent: (spec.split(' ')[1] === "current"), 59 | multipage: (spec.split(' ')[1] === "multipage"), 60 | } : 61 | Object.assign({}, spec, { url: new URL(spec.url).toString() })) 62 | .filter((spec, idx, list) => 63 | !list.find((s, i) => i < idx && compareSpecs(s, spec) === 0)) 64 | .sort(compareSpecs); 65 | 66 | // Generate names and links between levels 67 | const linkedList = sorted 68 | .map(s => Object.assign({}, s, computeShortname(s.shortname || s.url))) 69 | .map((s, _, list) => Object.assign({}, s, computePrevNext(s, list))); 70 | 71 | // Drop useless forceCurrent flag and shorten definition when possible 72 | const fixed = sorted 73 | .map(spec => { 74 | const linked = linkedList.find(p => p.url === spec.url); 75 | const next = linked.seriesNext ? 76 | linkedList.find(p => p.shortname === linked.seriesNext) : 77 | null; 78 | const isLast = !next || next.seriesComposition === "delta" || 79 | next.seriesComposition === "fork"; 80 | if (spec.forceCurrent && isLast) { 81 | spec.forceCurrent = false; 82 | } 83 | return spec; 84 | }) 85 | .map(shortenDefinition); 86 | 87 | const linted = JSON.stringify(fixed, null, 2) + "\n"; 88 | return (linted !== specsStr) ? linted : null; 89 | } 90 | 91 | 92 | // Lint by normalizing specs.json and comparing it to the original, 93 | // fixing it in place if |fix| is true. 94 | async function lint(fix = false) { 95 | const specs = await fs.readFile("./specs.json", "utf8"); 96 | const linted = lintStr(specs); 97 | if (linted) { 98 | if (fix) { 99 | console.log("specs.json has lint issues, updating in place"); 100 | await fs.writeFile("./specs.json", linted, "utf8"); 101 | } 102 | else { 103 | console.log("specs.json has lint issues, run with --fix"); 104 | } 105 | return false; 106 | } 107 | 108 | console.log("specs.json passed lint"); 109 | return true; 110 | } 111 | 112 | 113 | if (require.main === module) { 114 | // Code used as command-line interface (CLI), run linting process 115 | lint(process.argv.includes("--fix")).then( 116 | ok => { 117 | process.exit(ok ? 0 : 1); 118 | }, 119 | reason => { 120 | console.error(reason); 121 | process.exit(1); 122 | } 123 | ); 124 | } 125 | else { 126 | // Code imported to another JS module, export lint functions 127 | module.exports.lintStr = lintStr; 128 | module.exports.lint = lint; 129 | } 130 | -------------------------------------------------------------------------------- /test/fetch-groups-w3c.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Tests for the fetch-groups module that require a W3C API key 3 | * 4 | * These tests are separated from the tests that do not require a W3C API key 5 | * because the key cannot be exposed on pull requests from forked repositories 6 | * on GitHub. 7 | */ 8 | 9 | const assert = require("assert"); 10 | const fetchGroups = require("../src/fetch-groups.js"); 11 | 12 | const githubToken = (function () { 13 | try { 14 | return require("../config.json").githubToken; 15 | } 16 | catch (err) { 17 | return null; 18 | } 19 | })(); 20 | 21 | const w3cApiKey = (function () { 22 | try { 23 | return require("../config.json").w3cApiKey; 24 | } 25 | catch (err) { 26 | return null; 27 | } 28 | })(); 29 | 30 | 31 | describe("fetch-groups module (with API keys)", function () { 32 | // Tests need to send network requests 33 | this.slow(5000); 34 | this.timeout(30000); 35 | 36 | async function fetchGroupsFor(url, options) { 37 | const spec = { url }; 38 | const result = await fetchGroups([spec], options); 39 | return result[0]; 40 | }; 41 | 42 | describe("W3C API key", () => { 43 | it("is defined otherwise tests cannot pass", () => { 44 | assert.ok(w3cApiKey); 45 | }); 46 | }); 47 | 48 | 49 | describe("fetch from W3C API", () => { 50 | it("handles /TR URLs", async () => { 51 | const res = await fetchGroupsFor("https://www.w3.org/TR/gamepad/", { w3cApiKey }); 52 | assert.equal(res.organization, "W3C"); 53 | assert.deepStrictEqual(res.groups, [{ 54 | name: "Web Applications Working Group", 55 | url: "https://www.w3.org/groups/wg/webapps" 56 | }]); 57 | }); 58 | 59 | it("handles multiple /TR URLs", async () => { 60 | const specs = [ 61 | { url: "https://www.w3.org/TR/gamepad/" }, 62 | { url: "https://www.w3.org/TR/accname-1.2/" } 63 | ]; 64 | const res = await fetchGroups(specs, { w3cApiKey }); 65 | assert.equal(res[0].organization, "W3C"); 66 | assert.deepStrictEqual(res[0].groups, [{ 67 | name: "Web Applications Working Group", 68 | url: "https://www.w3.org/groups/wg/webapps" 69 | }]); 70 | assert.equal(res[1].organization, "W3C"); 71 | assert.deepStrictEqual(res[1].groups, [{ 72 | name: "Accessible Rich Internet Applications Working Group", 73 | url: "https://www.w3.org/WAI/ARIA/" 74 | }]); 75 | }); 76 | 77 | it("handles w3c.github.io URLs", async () => { 78 | const res = await fetchGroupsFor("https://w3c.github.io/web-nfc/", { githubToken, w3cApiKey }); 79 | assert.equal(res.organization, "W3C"); 80 | assert.deepStrictEqual(res.groups, [{ 81 | name: "Web NFC Community Group", 82 | url: "https://www.w3.org/community/web-nfc/" 83 | }]); 84 | }); 85 | 86 | it("handles SVG URLs", async () => { 87 | const res = await fetchGroupsFor("https://svgwg.org/specs/animations/", { w3cApiKey }); 88 | assert.equal(res.organization, "W3C"); 89 | assert.deepStrictEqual(res.groups, [{ 90 | name: "SVG Working Group", 91 | url: "https://www.w3.org/Graphics/SVG/WG/" 92 | }]); 93 | }); 94 | 95 | it("handles CSS WG URLs", async () => { 96 | const res = await fetchGroupsFor("https://drafts.csswg.org/css-animations-2/", { w3cApiKey }); 97 | assert.equal(res.organization, "W3C"); 98 | assert.deepStrictEqual(res.groups, [{ 99 | name: "Cascading Style Sheets (CSS) Working Group", 100 | url: "https://www.w3.org/Style/CSS/" 101 | }]); 102 | }); 103 | 104 | it("handles CSS Houdini TF URLs", async () => { 105 | const res = await fetchGroupsFor("https://drafts.css-houdini.org/css-typed-om-2/", { w3cApiKey }); 106 | assert.equal(res.organization, "W3C"); 107 | assert.deepStrictEqual(res.groups, [{ 108 | name: "Cascading Style Sheets (CSS) Working Group", 109 | url: "https://www.w3.org/Style/CSS/" 110 | }]); 111 | }); 112 | 113 | it("handles CSS FXTF URLs", async () => { 114 | const res = await fetchGroupsFor("https://drafts.fxtf.org/filter-effects-2/", { w3cApiKey }); 115 | assert.equal(res.organization, "W3C"); 116 | assert.deepStrictEqual(res.groups, [{ 117 | name: "Cascading Style Sheets (CSS) Working Group", 118 | url: "https://www.w3.org/Style/CSS/" 119 | }]); 120 | }); 121 | }); 122 | }); -------------------------------------------------------------------------------- /schema/definitions.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/schema#", 3 | "$id": "https://w3c.github.io/browser-specs/schema/definitions.json", 4 | 5 | "$defs": { 6 | "url": { 7 | "type": "string", 8 | "format": "uri" 9 | }, 10 | 11 | "filename": { 12 | "type": "string", 13 | "pattern": "^[\\w\\-\\.]+\\.html$" 14 | }, 15 | 16 | "relativePath": { 17 | "type": "string", 18 | "pattern": "^[\\w\\-\\.]+(\\/[\\w\\-\\.]+)*$" 19 | }, 20 | 21 | "shortname": { 22 | "type": "string", 23 | "pattern": "^[\\w\\-]+((?<=-v?\\d+)\\.\\d+)?$" 24 | }, 25 | 26 | "series": { 27 | "type": "object", 28 | "properties": { 29 | "shortname": { 30 | "type": "string", 31 | "pattern": "^[\\w\\-]+$" 32 | }, 33 | "title": { "$ref": "#/$defs/title" }, 34 | "shortTitle": { "$ref": "#/$defs/title" }, 35 | "currentSpecification": { "$ref": "#/$defs/shortname" }, 36 | "releaseUrl": { "$ref": "#/$defs/url" }, 37 | "nightlyUrl": { "$ref": "#/$defs/url" } 38 | }, 39 | "required": ["shortname"], 40 | "additionalProperties": false 41 | }, 42 | 43 | "seriesVersion": { 44 | "type": "string", 45 | "pattern": "^\\d+(\\.\\d+){0,2}$" 46 | }, 47 | 48 | "seriesComposition": { 49 | "type": "string", 50 | "enum": ["full", "delta", "fork"] 51 | }, 52 | 53 | "forceCurrent": { 54 | "type": "boolean" 55 | }, 56 | 57 | "title": { 58 | "type": "string" 59 | }, 60 | 61 | "source": { 62 | "type": "string", 63 | "enum": ["w3c", "specref", "spec"] 64 | }, 65 | 66 | "release": { 67 | "type": "object", 68 | "properties": { 69 | "url": { "$ref": "#/$defs/url" }, 70 | "filename": { "$ref": "#/$defs/filename" }, 71 | "pages": { 72 | "type": "array", 73 | "items": { "$ref": "#/$defs/url" } 74 | } 75 | }, 76 | "required": ["url"], 77 | "additionalProperties": false 78 | }, 79 | 80 | "nightly": { 81 | "type": "object", 82 | "properties": { 83 | "url": { "$ref": "#/$defs/url" }, 84 | "alternateUrls": { 85 | "type": "array", 86 | "items": { "$ref": "#/$defs/url" } 87 | }, 88 | "filename": { "$ref": "#/$defs/filename" }, 89 | "sourcePath": { "$ref": "#/$defs/relativePath" }, 90 | "pages": { 91 | "type": "array", 92 | "items": { "$ref": "#/$defs/url" } 93 | }, 94 | "repository": { "$ref": "#/$defs/url" } 95 | }, 96 | "additionalProperties": false 97 | }, 98 | 99 | "tests": { 100 | "type": "object", 101 | "properties": { 102 | "repository": { "$ref": "#/$defs/url" }, 103 | "testPaths": { 104 | "type": "array", 105 | "items": { "$ref": "#/$defs/relativePath" }, 106 | "minItems": 1 107 | }, 108 | "excludePaths": { 109 | "type": "array", 110 | "items": { "$ref": "#/$defs/relativePath" }, 111 | "minItems": 1 112 | } 113 | }, 114 | "required": ["repository"], 115 | "additionalProperties": false 116 | }, 117 | 118 | "groups": { 119 | "type": "array", 120 | "items": { 121 | "type": "object", 122 | "properties": { 123 | "name": { "type": "string" }, 124 | "url": { "$ref": "#/$defs/url" } 125 | }, 126 | "required": ["name", "url"], 127 | "additionalProperties": false 128 | } 129 | }, 130 | 131 | "organization": { 132 | "type": "string" 133 | }, 134 | 135 | "categories": { 136 | "type": "array", 137 | "items": { 138 | "type": "string", 139 | "enum": ["browser"] 140 | } 141 | }, 142 | 143 | "categories-specs": { 144 | "oneOf": [ 145 | { 146 | "type": "string", 147 | "enum": ["reset", "+browser", "-browser"] 148 | }, 149 | { 150 | "type": "array", 151 | "items": { 152 | "type": "string", 153 | "enum": ["reset", "+browser", "-browser"] 154 | }, 155 | "minItems": 1 156 | } 157 | ] 158 | }, 159 | 160 | "forks": { 161 | "type": "array", 162 | "items": { "$ref": "#/$defs/shortname" } 163 | } 164 | } 165 | } -------------------------------------------------------------------------------- /test/compute-prevnext.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const computePrevNext = require("../src/compute-prevnext.js"); 3 | 4 | describe("compute-prevnext module", () => { 5 | function getSpec(seriesVersion) { 6 | if (seriesVersion) { 7 | return { 8 | shortname: `spec-${seriesVersion}`, 9 | series: { shortname: "spec" }, 10 | seriesVersion 11 | }; 12 | } 13 | else { 14 | return { 15 | shortname: `spec-${seriesVersion}`, 16 | series: { shortname: "spec" } 17 | }; 18 | } 19 | } 20 | function getOther(seriesVersion) { 21 | if (seriesVersion) { 22 | return { 23 | shortname: `other-${seriesVersion}`, 24 | series: { shortname: "other" }, 25 | seriesVersion 26 | }; 27 | } 28 | else { 29 | return { 30 | shortname: `other-${seriesVersion}`, 31 | series: { shortname: "other" } 32 | }; 33 | } 34 | } 35 | 36 | it("sets previous link if it exists", () => { 37 | const prev = getSpec("1"); 38 | const spec = getSpec("2"); 39 | assert.deepStrictEqual( 40 | computePrevNext(spec, [prev]), 41 | { seriesPrevious: prev.shortname }); 42 | }); 43 | 44 | it("sets next link if it exists", () => { 45 | const spec = getSpec("1"); 46 | const next = getSpec("2"); 47 | assert.deepStrictEqual( 48 | computePrevNext(spec, [next]), 49 | { seriesNext: next.shortname }); 50 | }); 51 | 52 | it("sets previous/next links when both exist", () => { 53 | const prev = getSpec("1"); 54 | const spec = getSpec("2"); 55 | const next = getSpec("3"); 56 | assert.deepStrictEqual( 57 | computePrevNext(spec, [next, prev, spec]), 58 | { seriesPrevious: prev.shortname, seriesNext: next.shortname }); 59 | }); 60 | 61 | it("sets previous/next links when level are version numbers", () => { 62 | const prev = getSpec("1.1"); 63 | const spec = getSpec("1.2"); 64 | const next = getSpec("1.3"); 65 | assert.deepStrictEqual( 66 | computePrevNext(spec, [next, prev, spec]), 67 | { seriesPrevious: prev.shortname, seriesNext: next.shortname }); 68 | }); 69 | 70 | it("selects the right previous level when multiple exist", () => { 71 | const old = getSpec("1"); 72 | const prev = getSpec("2"); 73 | const spec = getSpec("4"); 74 | assert.deepStrictEqual( 75 | computePrevNext(spec, [spec, prev, old]), 76 | { seriesPrevious: prev.shortname }); 77 | }); 78 | 79 | it("selects the right next level when multiple exist", () => { 80 | const spec = getSpec("1"); 81 | const next = getSpec("2"); 82 | const last = getSpec("3"); 83 | assert.deepStrictEqual( 84 | computePrevNext(spec, [spec, last, next]), 85 | { seriesNext: next.shortname }); 86 | }); 87 | 88 | it("considers absence of level to be level 0", () => { 89 | const spec = getSpec(); 90 | const next = getSpec("1"); 91 | assert.deepStrictEqual( 92 | computePrevNext(spec, [next]), 93 | { seriesNext: next.shortname }); 94 | }); 95 | 96 | it("is not affected by presence of other specs", () => { 97 | const prev = getSpec("1"); 98 | const spec = getSpec("3"); 99 | const next = getSpec("5"); 100 | assert.deepStrictEqual( 101 | computePrevNext(spec, [next, getOther("2"), spec, getOther("4"), prev]), 102 | { seriesPrevious: prev.shortname, seriesNext: next.shortname }); 103 | }); 104 | 105 | it("returns an empty object if list is empty", () => { 106 | const spec = getSpec(); 107 | assert.deepStrictEqual(computePrevNext(spec), {}); 108 | }); 109 | 110 | it("returns an empty object if list is the spec to check", () => { 111 | const spec = getSpec(); 112 | assert.deepStrictEqual(computePrevNext(spec, [spec]), {}); 113 | }); 114 | 115 | it("returns an empty object in absence of other levels", () => { 116 | const spec = getSpec("2"); 117 | assert.deepStrictEqual( 118 | computePrevNext(spec, [getOther("1"), spec, getOther("3")]), {}); 119 | }); 120 | 121 | it("throws if spec object is not given", () => { 122 | assert.throws( 123 | () => computePrevNext(), 124 | /^Invalid spec object passed as parameter$/); 125 | }); 126 | 127 | it("throws if spec object is empty", () => { 128 | assert.throws( 129 | () => computePrevNext({}), 130 | /^Invalid spec object passed as parameter$/); 131 | }); 132 | 133 | it("throws if spec object does not have a name", () => { 134 | assert.throws( 135 | () => computePrevNext({ shortname: "spec" }), 136 | /^Invalid spec object passed as parameter$/); 137 | }); 138 | 139 | it("throws if spec object does not have a shortname", () => { 140 | assert.throws( 141 | () => computePrevNext({ name: "spec" }), 142 | /^Invalid spec object passed as parameter$/); 143 | }); 144 | }); -------------------------------------------------------------------------------- /test/fetch-info-w3c.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Tests for the fetch-info module that require a W3C API key 3 | * 4 | * These tests are separated from the tests that do not require a W3C API key 5 | * because the key cannot be exposed on pull requests from forked repositories 6 | * on GitHub. 7 | */ 8 | 9 | const assert = require("assert"); 10 | const fetchInfo = require("../src/fetch-info.js"); 11 | 12 | const w3cApiKey = (function () { 13 | try { 14 | return require("../config.json").w3cApiKey; 15 | } 16 | catch (err) { 17 | return null; 18 | } 19 | })(); 20 | 21 | 22 | describe("fetch-info module (with W3C API key)", function () { 23 | // Tests need to send network requests 24 | this.slow(5000); 25 | this.timeout(30000); 26 | 27 | function getW3CSpec(shortname, series) { 28 | const spec = { shortname, url: `https://www.w3.org/TR/${shortname}/` }; 29 | if (series) { 30 | spec.series = { shortname: series }; 31 | } 32 | return spec; 33 | } 34 | 35 | describe("W3C API key", () => { 36 | it("is defined otherwise tests cannot pass", () => { 37 | assert.ok(w3cApiKey); 38 | }); 39 | }); 40 | 41 | 42 | describe("fetch from W3C API", () => { 43 | it("works on a TR spec", async () => { 44 | const spec = getW3CSpec("hr-time-2", "hr-time"); 45 | const info = await fetchInfo([spec], { w3cApiKey }); 46 | assert.ok(info[spec.shortname]); 47 | assert.equal(info[spec.shortname].source, "w3c"); 48 | assert.equal(info[spec.shortname].release.url, spec.url); 49 | assert.equal(info[spec.shortname].nightly.url, "https://w3c.github.io/hr-time/"); 50 | assert.equal(info[spec.shortname].title, "High Resolution Time Level 2"); 51 | 52 | assert.ok(info.__series); 53 | assert.ok(info.__series["hr-time"]); 54 | assert.equal(info.__series["hr-time"].currentSpecification, "hr-time-3"); 55 | assert.equal(info.__series["hr-time"].title, "High Resolution Time"); 56 | }); 57 | 58 | it("can operate on multiple specs at once", async () => { 59 | const spec = getW3CSpec("hr-time-2", "hr-time"); 60 | const other = getW3CSpec("presentation-api", "presentation-api"); 61 | const info = await fetchInfo([spec, other], { w3cApiKey }); 62 | assert.ok(info[spec.shortname]); 63 | assert.equal(info[spec.shortname].source, "w3c"); 64 | assert.equal(info[spec.shortname].release.url, spec.url); 65 | assert.equal(info[spec.shortname].nightly.url, "https://w3c.github.io/hr-time/"); 66 | assert.equal(info[spec.shortname].title, "High Resolution Time Level 2"); 67 | 68 | assert.ok(info[other.shortname]); 69 | assert.equal(info[other.shortname].source, "w3c"); 70 | assert.equal(info[other.shortname].release.url, other.url); 71 | assert.equal(info[other.shortname].nightly.url, "https://w3c.github.io/presentation-api/"); 72 | assert.equal(info[other.shortname].title, "Presentation API"); 73 | 74 | assert.ok(info.__series); 75 | assert.ok(info.__series["hr-time"]); 76 | assert.ok(info.__series["presentation-api"]); 77 | assert.equal(info.__series["hr-time"].currentSpecification, "hr-time-3"); 78 | assert.equal(info.__series["presentation-api"].currentSpecification, "presentation-api"); 79 | }); 80 | 81 | it("throws when W3C API key is invalid", async () => { 82 | assert.rejects( 83 | fetchInfo([getW3CSpec("selectors-3")], { w3cApiKey: "invalid" }), 84 | /^W3C API returned an error, status code is 403$/); 85 | }); 86 | }); 87 | 88 | 89 | describe("fetch from Specref", () => { 90 | it("works on a WHATWG spec", async () => { 91 | const spec = { 92 | url: "https://dom.spec.whatwg.org/", 93 | shortname: "dom" 94 | }; 95 | const info = await fetchInfo([spec], { w3cApiKey }); 96 | assert.ok(info[spec.shortname]); 97 | assert.equal(info[spec.shortname].source, "specref"); 98 | assert.equal(info[spec.shortname].nightly.url, "https://dom.spec.whatwg.org/"); 99 | assert.equal(info[spec.shortname].title, "DOM Standard"); 100 | }); 101 | }); 102 | 103 | 104 | describe("fetch from all sources", () => { 105 | it("merges info from sources", async () => { 106 | const w3c = getW3CSpec("presentation-api"); 107 | const whatwg = { 108 | url: "https://html.spec.whatwg.org/multipage/", 109 | shortname: "html" 110 | }; 111 | const other = { 112 | url: "https://tabatkins.github.io/bikeshed/", 113 | shortname: "bikeshed" 114 | }; 115 | const info = await fetchInfo([w3c, whatwg, other], { w3cApiKey }); 116 | assert.ok(info[w3c.shortname]); 117 | assert.equal(info[w3c.shortname].source, "w3c"); 118 | assert.equal(info[w3c.shortname].release.url, w3c.url); 119 | assert.equal(info[w3c.shortname].nightly.url, "https://w3c.github.io/presentation-api/"); 120 | assert.equal(info[w3c.shortname].title, "Presentation API"); 121 | 122 | assert.ok(info[whatwg.shortname]); 123 | assert.equal(info[whatwg.shortname].source, "specref"); 124 | assert.equal(info[whatwg.shortname].nightly.url, whatwg.url); 125 | assert.equal(info[whatwg.shortname].title, "HTML Standard"); 126 | 127 | assert.ok(info[other.shortname]); 128 | assert.equal(info[other.shortname].source, "spec"); 129 | assert.equal(info[other.shortname].nightly.url, other.url); 130 | assert.equal(info[other.shortname].title, "Bikeshed Documentation"); 131 | }); 132 | }); 133 | }); -------------------------------------------------------------------------------- /src/release-package.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Publish an NPM package when pre-release PR is merged, using the commit on 3 | * which the pre-release PR is based as source of data. 4 | */ 5 | 6 | const Octokit = require("./octokit"); 7 | const fs = require("fs"); 8 | const path = require("path"); 9 | const os = require("os"); 10 | const { execSync } = require("child_process"); 11 | const rimraf = require("rimraf"); 12 | const npmPublish = require("@jsdevtools/npm-publish"); 13 | 14 | const owner = "w3c"; 15 | const repo = "browser-specs"; 16 | 17 | 18 | /** 19 | * Release package at the requested version. 20 | * 21 | * @function 22 | * @param {Number} prNumber Pre-release PR number 23 | */ 24 | async function releasePackage(prNumber) { 25 | console.log(`Retrieve pre-release PR`); 26 | const prResponse = await octokit.pulls.get({ 27 | owner, repo, 28 | pull_number: prNumber 29 | }); 30 | const preReleasePR = prResponse?.data; 31 | if (!preReleasePR) { 32 | console.log("- Given PR does not seem to exist, nothing to release"); 33 | return; 34 | } 35 | 36 | // Extract type from PR title 37 | console.log(`- Given PR title: ${preReleasePR.title}`); 38 | const match = preReleasePR.title.match(/^📦 Release (.*)@(.*)$/); 39 | if (!match) { 40 | console.log("- Given PR is not a pre-release PR, nothing to release"); 41 | return; 42 | } 43 | const type = match[1]; 44 | 45 | if (!["web-specs", "browser-specs"].includes(type)) { 46 | console.log(`- Unknown package type "${type}", nothing to release`); 47 | return; 48 | } 49 | 50 | // Extract commit to release from PR 51 | const preReleaseSha = preReleasePR.base.sha; 52 | console.log(`- Found commit to release: ${preReleaseSha}`); 53 | 54 | console.log(); 55 | console.log("Publish package to npm"); 56 | console.log("- Checkout repo at right commit in temporary folder"); 57 | const tmpFolder = fs.mkdtempSync(path.join(os.tmpdir(), `${repo}-`)); 58 | 59 | try { 60 | execSync(`git clone https://github.com/${owner}/${repo}`, { 61 | cwd: tmpFolder 62 | }); 63 | const installFolder = path.join(tmpFolder, repo); 64 | execSync(`git reset --hard ${preReleaseSha}`, { 65 | cwd: installFolder 66 | }); 67 | console.log(`- Installation folder: ${installFolder}`); 68 | 69 | console.log("- Prepare package files"); 70 | execSync("npm ci", { cwd: installFolder }); 71 | execSync("node src/prepare-packages.js", { cwd: installFolder }); 72 | 73 | console.log(`- Publish packages/${type} folder to npm`); 74 | const packageFolder = path.join(installFolder, "packages", type, "package.json"); 75 | const pubResult = await npmPublish({ 76 | package: packageFolder, 77 | token: NPM_TOKEN 78 | //, debug: console.debug 79 | }); 80 | console.log(`- Published version was ${pubResult.oldVersion}`); 81 | console.log(`- Version bump: ${pubResult.type}`); 82 | console.log(`- Published version is ${pubResult.version}`); 83 | 84 | console.log(); 85 | console.log("Add release tag to commit"); 86 | if (pubResult.version === pubResult.oldVersion) { 87 | console.log("- Skip, no actual package released"); 88 | } 89 | else { 90 | const rawTag = `${type}@${pubResult.version}`; 91 | await octokit.git.createRef({ 92 | owner, repo, 93 | ref: `refs/tags/${rawTag}`, 94 | sha: preReleaseSha 95 | }); 96 | console.log(`- Tagged released commit ${preReleaseSha} with tag "${rawTag}"`); 97 | 98 | await octokit.git.updateRef({ 99 | owner, repo, 100 | ref: `heads/${type}@latest`, 101 | sha: preReleaseSha 102 | }); 103 | console.log(`- Updated ${type}-latest to point to released commit ${preReleaseSha}`); 104 | } 105 | } 106 | finally { 107 | console.log("Clean temporary folder"); 108 | try { 109 | rimraf.sync(tmpFolder); 110 | console.log("- done"); 111 | } 112 | catch { 113 | } 114 | } 115 | } 116 | 117 | 118 | /******************************************************************************* 119 | Retrieve tokens from environment, prepare Octokit and kick things off 120 | *******************************************************************************/ 121 | const GH_TOKEN = (_ => { 122 | try { 123 | return require("../config.json").GH_TOKEN; 124 | } 125 | catch { 126 | return ""; 127 | } 128 | })() || process.env.GH_TOKEN; 129 | if (!GH_TOKEN) { 130 | console.error("GH_TOKEN must be set to some personal access token as an env variable or in a config.json file"); 131 | process.exit(1); 132 | } 133 | 134 | const NPM_TOKEN = (() => { 135 | try { 136 | return require("../config.json").NPM_TOKEN; 137 | } catch { 138 | return process.env.NPM_TOKEN; 139 | } 140 | })(); 141 | if (!NPM_TOKEN) { 142 | console.error("NPM_TOKEN must be set to an npm token as an env variable or in a config.json file"); 143 | process.exit(1); 144 | } 145 | 146 | // Note: npm-publish has a bug and needs an "INPUT_TOKEN" env variable: 147 | // https://github.com/JS-DevTools/npm-publish/issues/15 148 | // (we're passing the token to the function directly, no need to set it here) 149 | process.env.INPUT_TOKEN = ""; 150 | 151 | const octokit = new Octokit({ 152 | auth: GH_TOKEN 153 | //, log: console 154 | }); 155 | 156 | const prereleasePR = parseInt(process.argv[2], 10); 157 | 158 | releasePackage(prereleasePR) 159 | .then(() => { 160 | console.log(); 161 | console.log("== The end =="); 162 | }) 163 | .catch(err => { 164 | console.error(err); 165 | process.exit(1); 166 | }); 167 | -------------------------------------------------------------------------------- /test/compute-series-urls.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const computeSeriesUrls = require("../src/compute-series-urls.js"); 3 | 4 | describe("compute-series-urls module", () => { 5 | it("returns spec URLs when spec has no level", () => { 6 | const spec = { 7 | url: "https://www.w3.org/TR/preload/", 8 | shortname: "preload", 9 | series: { shortname: "preload" }, 10 | release: { url: "https://www.w3.org/TR/preload/" }, 11 | nightly: { url: "https://w3c.github.io/preload/" } 12 | }; 13 | assert.deepStrictEqual(computeSeriesUrls(spec), 14 | { releaseUrl: "https://www.w3.org/TR/preload/", 15 | nightlyUrl: "https://w3c.github.io/preload/" }); 16 | }); 17 | 18 | 19 | it("does not return a release URL if spec has none", () => { 20 | const spec = { 21 | url: "https://compat.spec.whatwg.org/", 22 | shortname: "compat", 23 | series: { shortname: "compat" }, 24 | nightly: { url: "https://compat.spec.whatwg.org/" } 25 | }; 26 | assert.deepStrictEqual(computeSeriesUrls(spec), 27 | { nightlyUrl: "https://compat.spec.whatwg.org/" }); 28 | }); 29 | 30 | 31 | it("does not return a nightly URL if spec has none", () => { 32 | const spec = { 33 | url: "https://compat.spec.whatwg.org/", 34 | shortname: "compat", 35 | series: { shortname: "compat" }, 36 | }; 37 | assert.deepStrictEqual(computeSeriesUrls(spec), {}); 38 | }); 39 | 40 | 41 | it("returns series URLs for Houdini specs", () => { 42 | const spec = { 43 | url: "https://www.w3.org/TR/css-paint-api-1/", 44 | shortname: "css-paint-api-1", 45 | series: { shortname: "css-paint-api" }, 46 | release: { url: "https://www.w3.org/TR/css-paint-api-1/" }, 47 | nightly: { url: "https://drafts.css-houdini.org/css-paint-api-1/" } 48 | }; 49 | assert.deepStrictEqual(computeSeriesUrls(spec), 50 | { releaseUrl: "https://www.w3.org/TR/css-paint-api/", 51 | nightlyUrl: "https://drafts.css-houdini.org/css-paint-api/" }); 52 | }); 53 | 54 | 55 | it("returns series URLs for CSS specs", () => { 56 | const spec = { 57 | url: "https://www.w3.org/TR/css-fonts-4/", 58 | shortname: "css-fonts-4", 59 | series: { shortname: "css-fonts" }, 60 | release: { url: "https://www.w3.org/TR/css-fonts-4/" }, 61 | nightly: { url: "https://drafts.csswg.org/css-fonts-4/" } 62 | }; 63 | assert.deepStrictEqual(computeSeriesUrls(spec), 64 | { releaseUrl: "https://www.w3.org/TR/css-fonts/", 65 | nightlyUrl: "https://drafts.csswg.org/css-fonts/" }); 66 | }); 67 | 68 | 69 | it("returns right nightly URL for series when spec's nightly has no level", () => { 70 | const spec = { 71 | url: "https://www.w3.org/TR/pointerlock-2/", 72 | shortname: "pointerlock-2", 73 | series: { shortname: "pointerlock" }, 74 | release: { url: "https://www.w3.org/TR/pointerlock-2/" }, 75 | nightly: { url: "https://w3c.github.io/pointerlock/" } 76 | }; 77 | assert.deepStrictEqual(computeSeriesUrls(spec), 78 | { releaseUrl: "https://www.w3.org/TR/pointerlock/", 79 | nightlyUrl: "https://w3c.github.io/pointerlock/" }); 80 | }); 81 | 82 | 83 | it("does not invent an unversioned nightly URL for SVG 2", () => { 84 | const spec = { 85 | url: "https://www.w3.org/TR/SVG2/", 86 | shortname: "SVG2", 87 | series: { shortname: "SVG" }, 88 | release: { url: "https://www.w3.org/TR/SVG2/" }, 89 | nightly: { url: "https://svgwg.org/svg2-draft/" } 90 | }; 91 | assert.deepStrictEqual(computeSeriesUrls(spec), 92 | { releaseUrl: "https://www.w3.org/TR/SVG/", 93 | nightlyUrl: "https://svgwg.org/svg2-draft/" }); 94 | }); 95 | 96 | 97 | it("looks for a release URL in previous versions", () => { 98 | const spec = { 99 | url: "https://drafts.csswg.org/css-fonts-5/", 100 | shortname: "css-fonts-5", 101 | series: { shortname: "css-fonts" }, 102 | seriesPrevious: "css-fonts-4", 103 | nightly: { url: "https://drafts.csswg.org/css-fonts-5/" } 104 | }; 105 | 106 | const list = [ 107 | spec, 108 | { 109 | url: "https://drafts.csswg.org/css-fonts-4/", 110 | shortname: "css-fonts-4", 111 | series: { shortname: "css-fonts" }, 112 | seriesPrevious: "css-fonts-3", 113 | nightly: { url: "https://drafts.csswg.org/css-fonts-4/" } 114 | }, 115 | { 116 | url: "https://drafts.csswg.org/css-fonts-3/", 117 | shortname: "css-fonts-3", 118 | series: { shortname: "css-fonts" }, 119 | release: { url: "https://www.w3.org/TR/css-fonts-3/" }, 120 | nightly: { url: "https://drafts.csswg.org/css-fonts-3/" } 121 | } 122 | ]; 123 | 124 | assert.deepStrictEqual(computeSeriesUrls(spec, list), 125 | { releaseUrl: "https://www.w3.org/TR/css-fonts/", 126 | nightlyUrl: "https://drafts.csswg.org/css-fonts/" }); 127 | }); 128 | 129 | 130 | it("computes info based on current specification", () => { 131 | const spec = { 132 | url: "https://www.w3.org/TR/SVG11/", 133 | seriesComposition: "full", 134 | shortname: "SVG11", 135 | series: { shortname: "SVG", currentSpecification: "SVG2" }, 136 | release: { url: "https://www.w3.org/TR/SVG11/" }, 137 | nightly: { url: "https://www.w3.org/TR/SVG11/" } 138 | }; 139 | 140 | const list = [ 141 | spec, 142 | { 143 | url: "https://www.w3.org/TR/SVG2/", 144 | seriesComposition: "full", 145 | shortname: "SVG2", 146 | series: { shortname: "SVG", currentSpecification: "SVG2" }, 147 | release: { url: "https://www.w3.org/TR/SVG2/" }, 148 | nightly: { url: "https://svgwg.org/svg2-draft/" } 149 | } 150 | ]; 151 | 152 | assert.deepStrictEqual(computeSeriesUrls(spec, list), 153 | { releaseUrl: "https://www.w3.org/TR/SVG/", 154 | nightlyUrl: "https://svgwg.org/svg2-draft/" }); 155 | }); 156 | }); -------------------------------------------------------------------------------- /test/lint.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const { lintStr } = require("../src/lint.js"); 3 | 4 | describe("Linter", () => { 5 | describe("lintStr()", () => { 6 | function toStr(specs) { 7 | return JSON.stringify(specs, null, 2) + "\n"; 8 | } 9 | 10 | it("passes if specs contains a valid URL", () => { 11 | const specs = ["https://www.w3.org/TR/spec/"]; 12 | assert.equal(lintStr(toStr(specs)), null); 13 | }); 14 | 15 | it("passes if specs contains multiple sorted URLs", () => { 16 | const specs = [ 17 | "https://www.w3.org/TR/spec1/", 18 | "https://www.w3.org/TR/spec2/" 19 | ]; 20 | assert.equal(lintStr(toStr(specs)), null); 21 | }); 22 | 23 | it("passes if specs contains a URL with a delta spec", () => { 24 | const specs = [ 25 | "https://www.w3.org/TR/spec-1/", 26 | "https://www.w3.org/TR/spec-2/ delta" 27 | ]; 28 | assert.equal(lintStr(toStr(specs)), null); 29 | }); 30 | 31 | it("passes if specs contains a URL with a spec flagged as current", () => { 32 | const specs = [ 33 | "https://www.w3.org/TR/spec-1/ current", 34 | "https://www.w3.org/TR/spec-2/" 35 | ]; 36 | assert.equal(lintStr(toStr(specs)), null); 37 | }); 38 | 39 | it("passes if specs contains a URL with a spec flagged as multipage", () => { 40 | const specs = [ 41 | "https://www.w3.org/TR/spec-1/ multipage" 42 | ]; 43 | assert.equal(lintStr(toStr(specs)), null); 44 | }); 45 | 46 | it("sorts URLs", () => { 47 | const specs = [ 48 | "https://www.w3.org/TR/spec2/", 49 | "https://www.w3.org/TR/spec1/" 50 | ]; 51 | assert.equal( 52 | lintStr(toStr(specs)), 53 | toStr([ 54 | "https://www.w3.org/TR/spec1/", 55 | "https://www.w3.org/TR/spec2/" 56 | ])); 57 | }); 58 | 59 | it("lints a URL", () => { 60 | const specs = [ 61 | { "url": "https://example.org", "shortname": "test" } 62 | ]; 63 | assert.equal(lintStr(toStr(specs)), toStr([ 64 | { "url": "https://example.org/", "shortname": "test" } 65 | ])); 66 | }); 67 | 68 | it("lints an object with only a URL to a URL", () => { 69 | const specs = [ 70 | { "url": "https://www.w3.org/TR/spec/" } 71 | ]; 72 | assert.equal(lintStr(toStr(specs)), toStr([ 73 | "https://www.w3.org/TR/spec/" 74 | ])); 75 | }); 76 | 77 | it("lints an object with only a URL and a delta flag to a string", () => { 78 | const specs = [ 79 | "https://www.w3.org/TR/spec-1/", 80 | { "url": "https://www.w3.org/TR/spec-2/", seriesComposition: "delta" } 81 | ]; 82 | assert.equal(lintStr(toStr(specs)), toStr([ 83 | "https://www.w3.org/TR/spec-1/", 84 | "https://www.w3.org/TR/spec-2/ delta" 85 | ])); 86 | }); 87 | 88 | it("lints an object with only a URL and a current flag to a string", () => { 89 | const specs = [ 90 | { "url": "https://www.w3.org/TR/spec-1/", "forceCurrent": true }, 91 | "https://www.w3.org/TR/spec-2/" 92 | ]; 93 | assert.equal(lintStr(toStr(specs)), toStr([ 94 | "https://www.w3.org/TR/spec-1/ current", 95 | "https://www.w3.org/TR/spec-2/" 96 | ])); 97 | }); 98 | 99 | it("lints an object with only a URL and a multipage flag to a string", () => { 100 | const specs = [ 101 | { "url": "https://www.w3.org/TR/spec-1/", "multipage": true } 102 | ]; 103 | assert.equal(lintStr(toStr(specs)), toStr([ 104 | "https://www.w3.org/TR/spec-1/ multipage" 105 | ])); 106 | }); 107 | 108 | it("lints an object with a useless current flag", () => { 109 | const specs = [ 110 | "https://www.w3.org/TR/spec/ current" 111 | ]; 112 | assert.equal(lintStr(toStr(specs)), toStr([ 113 | "https://www.w3.org/TR/spec/" 114 | ])); 115 | }); 116 | 117 | it("lints an object with a useless current flag (delta version)", () => { 118 | const specs = [ 119 | "https://www.w3.org/TR/spec-1/ current", 120 | "https://www.w3.org/TR/spec-2/ delta" 121 | ]; 122 | assert.equal(lintStr(toStr(specs)), toStr([ 123 | "https://www.w3.org/TR/spec-1/", 124 | "https://www.w3.org/TR/spec-2/ delta", 125 | ])); 126 | }); 127 | 128 | it("lints an object with a 'full' flag", () => { 129 | const specs = [ 130 | { "url": "https://www.w3.org/TR/spec/", "seriesComposition": "full" } 131 | ]; 132 | assert.equal(lintStr(toStr(specs)), toStr([ 133 | "https://www.w3.org/TR/spec/" 134 | ])); 135 | }); 136 | 137 | it("lints an object with a current flag set to false", () => { 138 | const specs = [ 139 | { "url": "https://www.w3.org/TR/spec/", "forceCurrent": false } 140 | ]; 141 | assert.equal(lintStr(toStr(specs)), toStr([ 142 | "https://www.w3.org/TR/spec/" 143 | ])); 144 | }); 145 | 146 | it("lints an object with a multipage flag set to false", () => { 147 | const specs = [ 148 | { "url": "https://www.w3.org/TR/spec/", "multipage": false } 149 | ]; 150 | assert.equal(lintStr(toStr(specs)), toStr([ 151 | "https://www.w3.org/TR/spec/" 152 | ])); 153 | }); 154 | 155 | it("drops duplicate URLs", () => { 156 | const specs = [ 157 | "https://www.w3.org/TR/duplicate/", 158 | "https://www.w3.org/TR/duplicate/" 159 | ]; 160 | assert.equal( 161 | lintStr(toStr(specs)), 162 | toStr(["https://www.w3.org/TR/duplicate/"])); 163 | }); 164 | 165 | it("drops duplicate URLs defined as string and object", () => { 166 | const specs = [ 167 | { "url": "https://www.w3.org/TR/duplicate/" }, 168 | "https://www.w3.org/TR/duplicate/" 169 | ]; 170 | assert.equal( 171 | lintStr(toStr(specs)), 172 | toStr(["https://www.w3.org/TR/duplicate/"])); 173 | }); 174 | 175 | it("lints an object with a forkOf and a seriesComposition property", () => { 176 | const specs = [ 177 | "https://www.w3.org/TR/spec-1/", 178 | { "url": "https://www.w3.org/TR/spec-2/", seriesComposition: "fork", forkOf: "spec-1" } 179 | ]; 180 | assert.equal(lintStr(toStr(specs)), toStr([ 181 | "https://www.w3.org/TR/spec-1/", 182 | { "url": "https://www.w3.org/TR/spec-2/", forkOf: "spec-1" } 183 | ])); 184 | }); 185 | }); 186 | }); 187 | -------------------------------------------------------------------------------- /src/determine-testpath.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that takes a list of spec objects as input and returns, for each spec, 3 | * the URL of the repository that contains the test suite of the spec along with 4 | * the path under which the tests are to be found in that repository. 5 | * 6 | * The function will run git commands on the command-line and populate the local 7 | * ".cache" folder. 8 | */ 9 | 10 | const fs = require("fs"); 11 | const path = require("path"); 12 | const execSync = require("child_process").execSync; 13 | 14 | // Cache folder under which the WPT repository will be cloned 15 | const cacheFolder = path.resolve(__dirname, "..", ".cache"); 16 | const wptFolder = path.resolve(cacheFolder, "wpt"); 17 | 18 | /** 19 | * Helper function to setup the cache folder 20 | */ 21 | function setupCacheFolder() { 22 | try { 23 | fs.mkdirSync(cacheFolder); 24 | } 25 | catch (err) { 26 | if (err.code !== 'EEXIST') { 27 | throw err; 28 | } 29 | } 30 | } 31 | 32 | /** 33 | * Helper function that returns true when the WPT folder already exists 34 | * (which is taken to mean that the repository has already been cloned) 35 | */ 36 | function wptFolderExists() { 37 | try { 38 | fs.accessSync(wptFolder); 39 | return true; 40 | } 41 | catch (err) { 42 | if (err.code !== "ENOENT") { 43 | throw err; 44 | } 45 | return false; 46 | } 47 | } 48 | 49 | /** 50 | * Helper function that fetches the latest version of the WPT repository, 51 | * restricting the checkout to META.yml files 52 | */ 53 | function fetchWPT() { 54 | setupCacheFolder(); 55 | if (wptFolderExists()) { 56 | // Pull latest commit from master branch 57 | execSync("git pull origin master", { cwd: wptFolder }); 58 | } 59 | else { 60 | // Clone repo using sparse mode: the repo is huge and we're only interested 61 | // in META.yml files 62 | execSync("git clone https://github.com/web-platform-tests/wpt.git --depth 1 --sparse", { cwd: cacheFolder }); 63 | execSync("git sparse-checkout set --no-cone", { cwd: wptFolder }); 64 | execSync("git sparse-checkout add **/META.yml", { cwd: wptFolder }); 65 | } 66 | } 67 | 68 | /** 69 | * Helper function that reads "spec" entries in all META.yml files of the WPT 70 | * repository. 71 | * 72 | * Note the function parses META.yml files as regular text files. That works 73 | * well but a proper YAML parser would be needed if we need to handle things 74 | * such as comments. 75 | */ 76 | async function readWptMetaFiles() { 77 | async function readFolder(folder) { 78 | let res = []; 79 | const contents = await fs.promises.readdir(folder); 80 | for (const name of contents) { 81 | const filename = path.resolve(folder, name); 82 | const stat = await fs.promises.stat(filename); 83 | if (stat.isDirectory()) { 84 | const nestedFiles = await readFolder(filename); 85 | res = res.concat(nestedFiles); 86 | } 87 | else if (name === "META.yml") { 88 | const file = await fs.promises.readFile(filename, "utf8"); 89 | const match = file.match(/(?:^|\n)spec: (.*)$/m); 90 | if (match) { 91 | res.push({ 92 | folder: folder.substring(wptFolder.length + 1).replace(/\\/g, "/"), 93 | spec: match[1] 94 | }); 95 | } 96 | } 97 | } 98 | return res; 99 | } 100 | 101 | fetchWPT(); 102 | return await readFolder(wptFolder); 103 | } 104 | 105 | 106 | /** 107 | * Returns the first item in the list found in the array, or null if none of 108 | * the items exists in the array. 109 | */ 110 | function getFirstFoundInArray(paths, ...items) { 111 | for (const item of items) { 112 | const path = paths.find(p => p === item); 113 | if (path) { 114 | return path; 115 | } 116 | } 117 | return null; 118 | } 119 | 120 | 121 | /** 122 | * Exports main function that takes a list of specs as input, completes entries 123 | * with a tests property when possible and returns the list. 124 | * 125 | * The options parameter is used to specify the GitHub API authentication token. 126 | */ 127 | module.exports = async function (specs, options) { 128 | if (!specs || specs.find(spec => !spec.shortname || !spec.series || !spec.series.shortname)) { 129 | throw "Invalid list of specifications passed as parameter"; 130 | } 131 | options = options || {}; 132 | 133 | const wptFolders = await readWptMetaFiles(); 134 | 135 | function determineTestInfo(spec) { 136 | const info = { 137 | repository: "https://github.com/web-platform-tests/wpt" 138 | }; 139 | 140 | if (spec.tests) { 141 | return Object.assign(info, spec.tests); 142 | } 143 | 144 | if (spec.url.startsWith("https://registry.khronos.org/")) { 145 | info.repository = "https://github.com/KhronosGroup/WebGL"; 146 | info.testPaths = ["conformance-suites"]; 147 | // TODO: Be more specific, tests for extensions should one of the files in: 148 | // https://github.com/KhronosGroup/WebGL/tree/master/conformance-suites/2.0.0/conformance2/extensions 149 | // https://github.com/KhronosGroup/WebGL/tree/master/conformance-suites/2.0.0/conformance/extensions 150 | // https://github.com/KhronosGroup/WebGL/tree/master/conformance-suites/1.0.3/conformance/extensions 151 | return info; 152 | } 153 | 154 | if (spec.url.startsWith("https://tc39.es/proposal-")) { 155 | // TODO: proposals may or may not have tests under tc39/test262, it would 156 | // be good to have that info here. However, that seems hard to assess 157 | // automatically and tedious to handle as exceptions in specs.json. 158 | return null; 159 | } 160 | 161 | // Note the use of startsWith below, needed to cover cases where a META.yml 162 | // file targets a specific page in a multipage spec (for HTML, typically), 163 | // or a fragment within a spec. 164 | const folders = wptFolders 165 | .filter(item => 166 | item.spec.startsWith(spec.nightly.url) || 167 | item.spec.startsWith(spec.nightly.url.replace(/-\d+\/$/, "/"))) 168 | .map(item => item.folder); 169 | if (folders.length > 0) { 170 | // Don't list subfolders when parent folder is already in the list 171 | info.testPaths = folders.filter(p1 => !folders.find(p2 => (p1 !== p2) && p1.startsWith(p2))); 172 | 173 | // Exclude subfolders of listed folders when they map to another spec 174 | const excludePaths = folders 175 | .map(path => wptFolders.filter(item => 176 | (item.folder !== path) && 177 | item.folder.startsWith(path + "/") && 178 | !item.spec.startsWith(spec.nightly.url) && 179 | !item.spec.startsWith(spec.nightly.url.replace(/-\d+\/$/, "/")))) 180 | .flat() 181 | .map(item => item.folder); 182 | if (excludePaths.length > 0) { 183 | info.excludePaths = excludePaths; 184 | } 185 | 186 | return info; 187 | } 188 | return null; 189 | } 190 | 191 | const testInfos = specs.map(determineTestInfo); 192 | for (const spec of specs) { 193 | const testInfo = testInfos.shift(); 194 | if (testInfo) { 195 | spec.tests = testInfo; 196 | } 197 | } 198 | 199 | return specs; 200 | }; 201 | -------------------------------------------------------------------------------- /src/compute-repository.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a list of specifications as input 3 | * and computes, for each of them, the URL of the repository that contains the 4 | * source code for this, as well as the source file of the specification at the 5 | * HEAD of default branch in the repository. 6 | * 7 | * The function needs an authentication token for the GitHub API. 8 | */ 9 | 10 | const Octokit = require("./octokit"); 11 | const parseSpecUrl = require("./parse-spec-url.js"); 12 | 13 | 14 | /** 15 | * Returns the first item in the list found in the Git tree, or null if none of 16 | * the items exists in the array. 17 | */ 18 | function getFirstFoundInTree(paths, ...items) { 19 | for (const item of items) { 20 | const path = paths.find(p => p.path === item); 21 | if (path) { 22 | return path; 23 | } 24 | } 25 | return null; 26 | } 27 | 28 | 29 | /** 30 | * Exports main function that takes a list of specs (with a nighly.url property) 31 | * as input, completes entries with a nightly.repository property when possible 32 | * and returns the list. 33 | * 34 | * The options parameter is used to specify the GitHub API authentication token. 35 | * In the absence of it, the function does not go through the GitHub API and 36 | * thus cannot set most of the information. This is useful to run tests without 37 | * an authentication token (but obviously means that the owner name returned 38 | * by the function will remain the lowercased version, and that the returned 39 | * info won't include the source file). 40 | */ 41 | module.exports = async function (specs, options) { 42 | if (!specs || specs.find(spec => !spec.nightly || !spec.nightly.url)) { 43 | throw "Invalid list of specifications passed as parameter"; 44 | } 45 | options = options || {}; 46 | 47 | const octokit = new Octokit({ auth: options.githubToken }); 48 | const repoCache = new Map(); 49 | const repoPathCache = new Map(); 50 | const userCache = new Map(); 51 | 52 | /** 53 | * Take a GitHub repo owner name (lowercase version) and retrieve the real 54 | * owner name (with possible uppercase characters) from the GitHub API. 55 | */ 56 | async function fetchRealGitHubOwnerName(username) { 57 | if (!userCache.has(username)) { 58 | const { data } = await octokit.users.getByUsername({ username }); 59 | if (data.message) { 60 | // Alert when user does not exist 61 | throw res.message; 62 | } 63 | userCache.set(username, data.login); 64 | } 65 | return userCache.get(username); 66 | } 67 | 68 | /** 69 | * Determine the name of the file that contains the source of the spec in the 70 | * default branch of the GitHub repository associated with the specification. 71 | */ 72 | async function determineSourcePath(spec, repo) { 73 | // Retrieve all paths of the GitHub repository 74 | const cacheKey = `${repo.owner}/${repo.name}`; 75 | if (!repoPathCache.has(cacheKey)) { 76 | const { data } = await octokit.git.getTree({ 77 | owner: repo.owner, 78 | repo: repo.name, 79 | tree_sha: "HEAD", 80 | recursive: true 81 | }); 82 | const paths = data.tree; 83 | repoPathCache.set(cacheKey, paths); 84 | } 85 | const paths = repoPathCache.get(cacheKey); 86 | 87 | // Extract filename from nightly URL when there is one 88 | const match = spec.nightly.url.match(/\/(\w+)\.html$/); 89 | const nightlyFilename = match ? match[1] : ""; 90 | 91 | const sourcePath = getFirstFoundInTree(paths, 92 | // Common paths for CSS specs 93 | `${spec.shortname}.bs`, 94 | `${spec.shortname}/Overview.bs`, 95 | `${spec.shortname}/Overview.src.html`, 96 | `${spec.series.shortname}/Overview.bs`, 97 | `${spec.series.shortname}/Overview.src.html`, 98 | 99 | // Named after the nightly filename 100 | `${nightlyFilename}.bs`, 101 | `${nightlyFilename}.html`, 102 | `${nightlyFilename}.src.html`, 103 | 104 | // WebGL extensions 105 | `extensions/${spec.shortname}/extension.xml`, 106 | 107 | // WebAssembly specs 108 | `document/${spec.series.shortname.replace(/^wasm-/, '')}/index.bs`, 109 | 110 | // SVG specs 111 | `specs/${spec.shortname.replace(/^svg-/, '')}/master/Overview.html`, 112 | `master/Overview.html`, 113 | 114 | // HTTPWG specs 115 | `specs/${spec.shortname}.xml`, 116 | 117 | // Following patterns are used in a small number of cases, but could 118 | // perhaps appear again in the future, so worth handling here. 119 | "spec/index.bs", 120 | "spec/index.html", // Only one TC39 spec 121 | "spec/Overview.html", // Only WebCrypto 122 | "docs/index.bs", // Only ServiceWorker 123 | "spec.html", // Most TC39 specs 124 | 125 | // Most common patterns, checking on "index.html" last as some repos 126 | // include such a file to store the generated spec from the source. 127 | "index.src.html", 128 | "index.bs", 129 | "spec.bs", 130 | "index.html" 131 | ); 132 | 133 | if (!sourcePath) { 134 | return null; 135 | } 136 | 137 | // Fetch target file for symlinks 138 | if (sourcePath.mode === "120000") { 139 | const { data } = await octokit.git.getBlob({ 140 | owner: repo.owner, 141 | repo: repo.name, 142 | file_sha: sourcePath.sha 143 | }); 144 | return Buffer.from(data.content, "base64").toString("utf8"); 145 | } 146 | return sourcePath.path; 147 | } 148 | 149 | async function isRealRepo(repo) { 150 | if (!options.githubToken) { 151 | // Assume the repo exists if we can't check 152 | return true; 153 | } 154 | const cacheKey = `${repo.owner}/${repo.name}`; 155 | if (!repoCache.has(cacheKey)) { 156 | try { 157 | await octokit.repos.get({ 158 | owner: repo.owner, 159 | repo: repo.name 160 | }); 161 | repoCache.set(cacheKey, true); 162 | } 163 | catch (err) { 164 | if (err.status === 404) { 165 | repoCache.set(cacheKey, false); 166 | } 167 | else { 168 | throw err; 169 | } 170 | } 171 | } 172 | return repoCache.get(cacheKey); 173 | } 174 | 175 | // Compute GitHub repositories with lowercase owner names 176 | const repos = specs.map(spec => parseSpecUrl(spec.nightly.repository ?? spec.nightly.url)); 177 | 178 | if (options.githubToken) { 179 | // Fetch the real name of repository owners (preserving case) 180 | for (const repo of repos) { 181 | if (repo) { 182 | repo.owner = await fetchRealGitHubOwnerName(repo.owner); 183 | } 184 | } 185 | } 186 | 187 | // Compute final repo URL and add source file if possible 188 | for (const spec of specs) { 189 | const repo = repos.shift(); 190 | if (repo && await isRealRepo(repo)) { 191 | spec.nightly.repository = `https://github.com/${repo.owner}/${repo.name}`; 192 | 193 | if (options.githubToken && !spec.nightly.sourcePath) { 194 | const sourcePath = await determineSourcePath(spec, repo); 195 | if (sourcePath) { 196 | spec.nightly.sourcePath = sourcePath; 197 | } 198 | } 199 | } 200 | } 201 | 202 | return specs; 203 | }; 204 | -------------------------------------------------------------------------------- /src/fetch-groups.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a list of specifications as input 3 | * and computes, for each of them, the name of the organization and groups 4 | * within that organization that develop the specification. 5 | * 6 | * The function needs an authentication token for the GitHub API as well as for 7 | * the W3C API. 8 | */ 9 | 10 | const fetch = require("node-fetch"); 11 | const Octokit = require("./octokit"); 12 | const parseSpecUrl = require("./parse-spec-url.js"); 13 | 14 | 15 | /** 16 | * Exports main function that takes a list of specs (with a url property) 17 | * as input, completes entries with an "organization" property that contains the 18 | * name of the organization such as W3C, WHATWG, IETF, Khronos Group, 19 | * Ecma International, and a "groups" property that contains an array of objects 20 | * that describe the groups responsible for the spec. 21 | * 22 | * The function preserves the properties if they have already been provided in 23 | * the input array. 24 | * 25 | * The options parameter is used to specify the GitHub API and W3C API 26 | * authentication tokens. 27 | */ 28 | module.exports = async function (specs, options) { 29 | // Maintain a cache of fetched resources in memory to avoid sending the 30 | // same fetch request again and again 31 | const cache = {}; 32 | 33 | // Helper function to retrieve a JSON resource or return null if resource 34 | // cannot be retrieved 35 | async function fetchJSON(url, options) { 36 | const body = cache[url] ?? await fetch(url, options).then(res => { 37 | if (res.status !== 200) { 38 | throw new Error(`W3C API returned an error, status code is ${res.status}`); 39 | } 40 | return res.json(); 41 | }); 42 | cache[url] = body; 43 | return body; 44 | } 45 | 46 | const w3cOptions = options?.w3cApiKey ? { headers: { 47 | Authorization: `W3C-API apikey="${options.w3cApiKey}"` 48 | }} : {}; 49 | 50 | for (const spec of specs) { 51 | const info = parseSpecUrl(spec.url); 52 | if (!info) { 53 | // No general rule to identify repos and groups for IETF specs 54 | // instead, fetching info on groups from https://www.rfc-editor.org/in-notes/rfcXXX.json 55 | if (spec.url.match(/rfc-editor\.org/)) { 56 | spec.organization = spec.organization ?? "IETF"; 57 | if (spec.groups) continue; 58 | const rfcNumber = spec.url.slice(spec.url.lastIndexOf('/') + 1); 59 | const rfcJSON = await fetchJSON(`https://www.rfc-editor.org/in-notes/${rfcNumber}.json`); 60 | const wgName = rfcJSON.source; 61 | // draft-ietf-websec-origin-06 → websec 62 | // per https://www.ietf.org/standards/ids/guidelines/#7 63 | // not clear there is anything forbidding hyphens in WG acronyms though 64 | // https://datatracker.ietf.org/doc/html/rfc2418#section-2.2 65 | const wgId = rfcJSON.draft.split('-')[2]; 66 | if (wgName && wgId) { 67 | spec.groups = [{ 68 | name: `${wgName} Working Group`, 69 | url: `https://datatracker.ietf.org/wg/${wgId}/` 70 | }]; 71 | continue; 72 | } else { 73 | throw new Error(`Could not identify IETF group producing ${spec.url}`); 74 | } 75 | } 76 | if (!spec.groups) { 77 | throw new Error(`Cannot extract any useful info from ${spec.url}`); 78 | } 79 | } 80 | 81 | if (info && info.owner === "whatwg") { 82 | const workstreams = await fetchJSON("https://raw.githubusercontent.com/whatwg/sg/main/db.json"); 83 | const workstream = workstreams.workstreams.find(ws => ws.standards.find(s => s.href === spec.url)); 84 | if (!workstream) { 85 | throw new Error(`No WHATWG workstream found for ${spec.url}`); 86 | } 87 | spec.organization = spec.organization ?? "WHATWG"; 88 | spec.groups = spec.groups ?? [{ 89 | name: `${workstream.name} Workstream`, 90 | url: spec.url 91 | }]; 92 | continue; 93 | } 94 | 95 | if (info && info.owner === "tc39") { 96 | spec.organization = spec.organization ?? "Ecma International"; 97 | spec.groups = spec.groups ?? [{ 98 | name: "TC39", 99 | url: "https://tc39.es/" 100 | }]; 101 | continue; 102 | } 103 | 104 | if (info && info.owner === "khronosgroup") { 105 | spec.organization = spec.organization ?? "Khronos Group"; 106 | spec.groups = spec.groups ?? [{ 107 | name: "WebGL Working Group", 108 | url: "https://www.khronos.org/webgl/" 109 | }]; 110 | continue; 111 | } 112 | 113 | // All specs that remain should be developed by some W3C group. 114 | spec.organization = spec.organization ?? "W3C"; 115 | 116 | if (!spec.groups) { 117 | let groups = null; 118 | if (info.name === "svgwg") { 119 | groups = [19480]; 120 | } 121 | else if (info.type === "tr") { 122 | // Use the W3C API to find info about /TR specs 123 | const url = `https://api.w3.org/specifications/${info.name}/versions/latest`; 124 | let resp = await fetchJSON(url, w3cOptions); 125 | if (!resp?._links?.deliverers) { 126 | throw new Error(`W3C API did not return deliverers for the spec`); 127 | } 128 | resp = await fetchJSON(resp._links.deliverers.href, w3cOptions); 129 | 130 | if (!resp?._links?.deliverers) { 131 | throw new Error(`W3C API did not return deliverers for the spec`); 132 | } 133 | groups = []; 134 | for (const deliverer of resp._links.deliverers) { 135 | groups.push(deliverer.href); 136 | } 137 | } 138 | else { 139 | // Use info in w3c.json file, which we'll either retrieve from the 140 | // repository when one is defined or directly from the spec origin 141 | // (we may need to go through the repository in all cases in the future, 142 | // but that approach works for now) 143 | let url = null; 144 | if (info.type === "github") { 145 | const octokit = new Octokit({ auth: options?.githubToken }); 146 | const cacheId = info.owner + "/" + info.name; 147 | const repo = cache[cacheId] ?? 148 | await octokit.repos.get({ owner: info.owner, repo: info.name }); 149 | cache[cacheId] = repo; 150 | const branch = repo?.data?.default_branch; 151 | if (!branch) { 152 | throw new Error(`Expected GitHub repository does not exist (${spec.url})`); 153 | } 154 | url = new URL(`https://raw.githubusercontent.com/${info.owner}/${info.name}/${branch}/w3c.json`); 155 | } 156 | else { 157 | url = new URL(spec.url); 158 | url.pathname = "/w3c.json"; 159 | } 160 | const body = await fetchJSON(url.toString()); 161 | 162 | // Note the "group" property is either an ID or an array of IDs 163 | groups = [body?.group].flat().filter(g => !!g); 164 | } 165 | 166 | // Retrieve info about W3C groups from W3C API 167 | // (Note the "groups" array may contain numbers, strings or API URLs) 168 | spec.groups = []; 169 | for (const id of groups) { 170 | const url = ('' + id).startsWith("https://") ? id : `https://api.w3.org/groups/${id}`; 171 | const info = await fetchJSON(url, w3cOptions); 172 | spec.groups.push({ 173 | name: info.name, 174 | url: info._links.homepage.href 175 | }); 176 | } 177 | } 178 | } 179 | 180 | return specs; 181 | }; 182 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | ## Applicable licenses 2 | 3 | This software and associated documentation files (the "Software") are licensed under the terms of the [MIT License](#mit-license). 4 | 5 | Additionally, the list of technical Web specifications (the [index.json](index.json) file) is published under the terms of the [CC0 license](#cc0-license). 6 | 7 | 8 | ## MIT License 9 | 10 | Copyright (c) 2020 World Wide Web Consortium 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | 19 | ## CC0 License 20 | 21 | ### Statement of Purpose 22 | 23 | The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work"). 24 | 25 | Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others. 26 | 27 | For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights. 28 | 29 | ### Copyright and Related Rights 30 | 31 | A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: 32 | 33 | i. the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work; 34 | ii. moral rights retained by the original author(s) and/or performer(s); 35 | iii. publicity and privacy rights pertaining to a person's image or likeness depicted in a Work; 36 | iv. rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below; 37 | v. rights protecting the extraction, dissemination, use and reuse of data in a Work; 38 | vi. database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and 39 | vii. other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof. 40 | 41 | ### Waiver 42 | 43 | To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. 44 | 45 | ### Public License Fallback 46 | 47 | Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. 48 | 49 | ### Limitations and Disclaimers 50 | 51 | a. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document. 52 | b. Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law. 53 | c. Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work. 54 | d. Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work. 55 | -------------------------------------------------------------------------------- /src/compute-shortname.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes a URL as input and computes a 3 | * meaningful shortname (i.e. the name with version), series' shortname and 4 | * version for it (when appropriate). 5 | * 6 | * The function returns an object with a "shortname" property. The name matches 7 | * the /TR/ name for specs published there. It includes the spec level. For 8 | * instance: "css-color-4" for "https://www.w3.org/TR/css-color-4/". 9 | * 10 | * For non-TR specs, the name returned is the "most logical" name that can be 11 | * extracted from the URL. The function typically handles a few typical cases 12 | * (such as "https://xxx.github.io/" URLs). It throws an exception when no 13 | * meaningful name can be extracted. 14 | * 15 | * Returned object will also alway have a "series" object that contains 16 | * an unleveled name for the specification. That shortname is shared across 17 | * levels of the specification. In most cases, it is the name without its level 18 | * suffix. For instance: "css-page" for "https://www.w3.org/TR/css-page-4/". 19 | * In rare cases, note the shortname may be different. For instance: 20 | * "css-conditional" for "https://www.w3.org/TR/css3-conditional/". 21 | * 22 | * If the URL contains a level indication, the returned object will have a 23 | * "seriesVersion" property with that level/version, represented as a string 24 | * which is either "x", "x.y" or "x.y.z", with x, y, z integers. If the spec 25 | * has no level, the "level" property is not set. 26 | * 27 | * Note that the function is NOT intended for use as a generic function that 28 | * returns a shortname, series' shortname and level for ANY URL. It is only 29 | * intended for use within the "browser-specs" project to automatically create 30 | * shortnames for common-looking URLs. In particular, individual exceptions to 31 | * the rule should NOT be hardcoded here but should rather be directly specified 32 | * in the "specs.json" file. For instance, it does not make sense to extend the 33 | * function to hardcode the fact that the "css3-mediaqueries" name should 34 | * create a "mediaqueries" series' shortname. 35 | */ 36 | 37 | 38 | /** 39 | * Internal function that takes a URL as input and returns a name for it 40 | * if the URL matches well-known patterns, or if the given parameter is actually 41 | * already a name (meaning that it does not contains any "/"). 42 | * 43 | * The function throws if it cannot compute a meaningful name from the URL. 44 | */ 45 | function computeShortname(url) { 46 | function parseUrl(url) { 47 | // Handle /TR/ URLs 48 | const w3cTr = url.match(/^https?:\/\/(?:www\.)?w3\.org\/TR\/([^\/]+)\/$/); 49 | if (w3cTr) { 50 | return w3cTr[1]; 51 | } 52 | 53 | // Handle WHATWG specs 54 | const whatwg = url.match(/\/\/(.+)\.spec\.whatwg\.org\//); 55 | if (whatwg) { 56 | return whatwg[1]; 57 | } 58 | 59 | // Handle TC39 Proposals 60 | const tc39 = url.match(/\/\/tc39\.es\/proposal-([^\/]+)\/$/); 61 | if (tc39) { 62 | return "tc39-" + tc39[1]; 63 | } 64 | 65 | 66 | // Handle Khronos extensions 67 | const khronos = url.match(/https:\/\/registry\.khronos\.org\/webgl\/extensions\/([^\/]+)\/$/); 68 | if (khronos) { 69 | return khronos[1]; 70 | } 71 | 72 | // Handle extension specs defined in the same repo as the main spec 73 | // (e.g. generate a "gamepad-extensions" name for 74 | // https://w3c.github.io/gamepad/extensions.html") 75 | const ext = url.match(/\/.*\.github\.io\/([^\/]+)\/(extensions?)\.html$/); 76 | if (ext) { 77 | return ext[1] + '-' + ext[2]; 78 | } 79 | 80 | // Handle draft specs on GitHub, excluding the "webappsec-" prefix for 81 | // specifications developed by the Web Application Security Working Group 82 | const github = url.match(/\/.*\.github\.io\/(?:webappsec-)?([^\/]+)\//); 83 | if (github) { 84 | return github[1]; 85 | } 86 | 87 | // Handle CSS WG specs 88 | const css = url.match(/\/drafts\.(?:csswg|fxtf|css-houdini)\.org\/([^\/]+)\//); 89 | if (css) { 90 | return css[1]; 91 | } 92 | 93 | // Handle SVG drafts 94 | const svg = url.match(/\/svgwg\.org\/specs\/(?:svg-)?([^\/]+)\//); 95 | if (svg) { 96 | return "svg-" + svg[1]; 97 | } 98 | 99 | // Handle IETF RFCs 100 | const rfcs = url.match(/\/www.rfc-editor\.org\/rfc\/(rfc[0-9]+)/); 101 | if (rfcs) { 102 | return rfcs[1]; 103 | } 104 | 105 | // Return name when one was given 106 | if (!url.match(/\//)) { 107 | return url; 108 | } 109 | 110 | throw `Cannot extract meaningful name from ${url}`; 111 | } 112 | 113 | // Parse the URL to extract the name 114 | const name = parseUrl(url); 115 | 116 | // Make sure name looks legit, in other words that it is composed of basic 117 | // Latin characters (a-z letters, digits, underscore and "-"), and that it 118 | // only contains a dot for fractional levels at the end of the name 119 | // (e.g. "blah-1.2" is good but "blah.blah" and "blah-3.1-blah" are not) 120 | if (!name.match(/^[\w\-]+((?<=\-v?\d+)\.\d+)?$/)) { 121 | throw `Specification name contains unexpected characters: ${name} (extracted from ${url})`; 122 | } 123 | 124 | return name; 125 | } 126 | 127 | 128 | /** 129 | * Compute the shortname and level from the spec name, if possible. 130 | */ 131 | function completeWithSeriesAndLevel(shortname, url, forkOf) { 132 | // Use latest convention for CSS specs 133 | function modernizeShortname(name) { 134 | if (name.startsWith("css3-")) { 135 | return "css-" + name.substring("css3-".length); 136 | } 137 | else if (name.startsWith("css4-")) { 138 | return "css-" + name.substring("css4-".length); 139 | } 140 | else { 141 | return name; 142 | } 143 | } 144 | 145 | const seriesBasename = forkOf ?? shortname; 146 | const specShortname = forkOf ? `${forkOf}-fork-${shortname}` : shortname; 147 | 148 | // Shortnames of WebGL extensions sometimes end up with digits which are *not* 149 | // to be interpreted as level numbers. Similarly, shortnames of ECMA specs 150 | // typically have the form "ecma-ddd", and "ddd" is *not* a level number. 151 | if (seriesBasename.match(/^ecma-/) || url.match(/^https:\/\/registry\.khronos\.org\/webgl\/extensions\//)) { 152 | return { 153 | shortname: specShortname, 154 | series: { shortname: seriesBasename } 155 | }; 156 | } 157 | 158 | // Extract X and X.Y levels, with form "name-X" or "name-X.Y". 159 | // (e.g. 5 for "mediaqueries-5", 1.2 for "wai-aria-1.2") 160 | let match = seriesBasename.match(/^(.*?)-v?(\d+)(.\d+)?$/); 161 | if (match) { 162 | return { 163 | shortname: specShortname, 164 | series: { shortname: modernizeShortname(match[1]) }, 165 | seriesVersion: match[3] ? match[2] + match[3] : match[2] 166 | }; 167 | } 168 | 169 | // Extract X and X.Y levels with form "nameX" or "nameXY" (but not "nameXXY") 170 | // (e.g. 2.1 for "CSS21", 1.1 for "SVG11", 4 for "selectors4") 171 | match = seriesBasename.match(/^(.*?)(? { 5 | 6 | describe("shortname property", () => { 7 | function assertName(url, name) { 8 | assert.equal(computeInfo(url).shortname, name); 9 | } 10 | 11 | it("handles TR URLs", () => { 12 | assertName("https://www.w3.org/TR/the-spec/", "the-spec"); 13 | }); 14 | 15 | it("handles WHATWG URLs", () => { 16 | assertName("https://myspec.spec.whatwg.org/whatever/", "myspec"); 17 | }); 18 | 19 | it("handles ECMAScript proposal URLs", () => { 20 | assertName("https://tc39.es/proposal-smartidea/", "tc39-smartidea"); 21 | }); 22 | 23 | it("handles Khronos Group WebGL extensions", () => { 24 | assertName("https://registry.khronos.org/webgl/extensions/EXT_wow32/", "EXT_wow32"); 25 | }); 26 | 27 | it("handles URLs of drafts on GitHub", () => { 28 | assertName("https://wicg.github.io/whataspec/", "whataspec"); 29 | }); 30 | 31 | it("handles URLs of WebAppSec drafts on GitHub", () => { 32 | assertName("https://w3c.github.io/webappsec-ultrasecret/", "ultrasecret"); 33 | }); 34 | 35 | it("handles extension specs defined in the same repo as the main spec (singular)", () => { 36 | assertName("https://w3c.github.io/specwithext/extension.html", "specwithext-extension"); 37 | }); 38 | 39 | it("handles extension specs defined in the same repo as the main spec (plural)", () => { 40 | assertName("https://w3c.github.io/specwithext/extensions.html", "specwithext-extensions"); 41 | }); 42 | 43 | it("handles CSS WG draft URLs", () => { 44 | assertName("https://drafts.csswg.org/css-is-aweso/", "css-is-aweso"); 45 | }); 46 | 47 | it("handles CSS FXTF draft URLs", () => { 48 | assertName("https://drafts.fxtf.org/megafx/", "megafx"); 49 | }); 50 | 51 | it("handles CSS Houdini TF draft URLs", () => { 52 | assertName("https://drafts.css-houdini.org/magic/", "magic"); 53 | }); 54 | 55 | it("handles SVG draft URLs", () => { 56 | assertName("https://svgwg.org/specs/module/", "svg-module"); 57 | }); 58 | 59 | it("handles SVG draft URLs that have an svg prefix", () => { 60 | assertName("https://svgwg.org/specs/svg-module/", "svg-module"); 61 | }); 62 | 63 | it("returns the name when given one", () => { 64 | assertName("myname", "myname"); 65 | }); 66 | 67 | it("preserves case", () => { 68 | assertName("https://www.w3.org/TR/IndexedDB/", "IndexedDB"); 69 | }); 70 | 71 | it("includes the version number in the name (int)", () => { 72 | assertName("https://www.w3.org/TR/level-42/", "level-42"); 73 | }); 74 | 75 | it("includes the version number in the name (float)", () => { 76 | assertName("https://www.w3.org/TR/level-4.2/", "level-4.2"); 77 | }); 78 | 79 | it("throws when URL is a dated TR one", () => { 80 | assert.throws( 81 | () => computeInfo("https://www.w3.org/TR/2017/CR-presentation-api-20170601/"), 82 | /^Cannot extract meaningful name from /); 83 | }); 84 | 85 | it("throws when URL that does not follow a known pattern", () => { 86 | assert.throws( 87 | () => computeInfo("https://www.w3.org/2001/tag/doc/promises-guide/"), 88 | /^Cannot extract meaningful name from /); 89 | }); 90 | 91 | it("throws when name contains non basic Latin characters", () => { 92 | assert.throws( 93 | () => computeInfo("https://www.w3.org/TR/thé-ou-café/"), 94 | /^Specification name contains unexpected characters/); 95 | }); 96 | 97 | it("throws when name contains a dot outside of a level definition", () => { 98 | assert.throws( 99 | () => computeInfo("https://w3c.github.io/spec.name/"), 100 | /^Specification name contains unexpected characters/); 101 | }); 102 | 103 | it("throws when name contains a non separated fractional level", () => { 104 | assert.throws( 105 | () => computeInfo("https://w3c.github.io/spec4.2/"), 106 | /^Specification name contains unexpected characters/); 107 | }); 108 | 109 | it("handles forks", () => { 110 | const url = "https://www.w3.org/TR/extension/"; 111 | assert.equal(computeInfo(url, "source-2").shortname, "source-2-fork-extension"); 112 | }); 113 | }); 114 | 115 | 116 | describe("series' shortname property", () => { 117 | function assertSeries(url, shortname) { 118 | assert.equal(computeInfo(url).series.shortname, shortname); 119 | } 120 | 121 | it("parses form 'shortname-X'", () => { 122 | assertSeries("spec-4", "spec"); 123 | }); 124 | 125 | it("parses form 'shortname-XXX'", () => { 126 | assertSeries("horizon-2050", "horizon"); 127 | }); 128 | 129 | it("parses form 'shortname-X.Y'", () => { 130 | assertSeries("pi-3.1", "pi"); 131 | }); 132 | 133 | it("parses form 'shortnameX'", () => { 134 | assertSeries("loveu2", "loveu"); 135 | }); 136 | 137 | it("parses form 'shortnameXY'", () => { 138 | assertSeries("answer42", "answer"); 139 | }); 140 | 141 | it("includes final digits when they do not seem to be a level", () => { 142 | assertSeries("cors-rfc1918", "cors-rfc1918"); 143 | }); 144 | 145 | it("does not get lost with inner digits", () => { 146 | assertSeries("my-2-cents", "my-2-cents"); 147 | }); 148 | 149 | it("automatically updates CSS specs with an old 'css3-' name", () => { 150 | assertSeries("css3-conditional", "css-conditional"); 151 | }); 152 | 153 | it("preserves ECMA spec numbers", () => { 154 | assertSeries("ecma-402", "ecma-402"); 155 | }); 156 | 157 | it("preserves digits at the end of WebGL extension names", () => { 158 | assertSeries("https://registry.khronos.org/webgl/extensions/EXT_wow32/", "EXT_wow32"); 159 | }); 160 | 161 | it("handles forks", () => { 162 | const url = "https://www.w3.org/TR/the-ext/"; 163 | assert.equal(computeInfo(url, "source-2").series.shortname, "source"); 164 | }); 165 | }); 166 | 167 | 168 | describe("seriesVersion property", () => { 169 | function assertSeriesVersion(url, level) { 170 | assert.equal(computeInfo(url).seriesVersion, level); 171 | } 172 | function assertNoSeriesVersion(url) { 173 | assert.equal(computeInfo(url).hasOwnProperty("seriesVersion"), false, 174 | "did not expect to see a seriesVersion property"); 175 | } 176 | 177 | it("finds the right series version for form 'shortname-X'", () => { 178 | assertSeriesVersion("spec-4", "4"); 179 | }); 180 | 181 | it("finds the right series version for form 'shortname-XXX'", () => { 182 | assertSeriesVersion("horizon-2050", "2050"); 183 | }); 184 | 185 | it("finds the right series version for form 'shortname-X.Y'", () => { 186 | assertSeriesVersion("pi-3.1", "3.1"); 187 | }); 188 | 189 | it("finds the right series version for form 'shortnameX'", () => { 190 | assertSeriesVersion("loveu2", "2"); 191 | }); 192 | 193 | it("finds the right series version for form 'shortnameXY'", () => { 194 | assertSeriesVersion("answer42", "4.2"); 195 | }); 196 | 197 | it("does not report any series version when there are none", () => { 198 | assertNoSeriesVersion("nolevel"); 199 | }); 200 | 201 | it("does not report a series version when final digits do not seem to be one", () => { 202 | assertNoSeriesVersion("cors-rfc1918"); 203 | }); 204 | 205 | it("does not get lost with inner digits", () => { 206 | assertNoSeriesVersion("my-2-cents"); 207 | }); 208 | 209 | it("does not confuse an ECMA spec number with a series version", () => { 210 | assertNoSeriesVersion("ecma-402"); 211 | }); 212 | 213 | it("does not confuse digits at the end of a WebGL extension spec with a series version", () => { 214 | assertNoSeriesVersion("https://registry.khronos.org/webgl/extensions/EXT_wow32/"); 215 | }); 216 | 217 | it("handles forks", () => { 218 | const url = "https://www.w3.org/TR/the-ext/"; 219 | assert.equal(computeInfo(url, "source-2").seriesVersion, "2"); 220 | }); 221 | }); 222 | }); 223 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Make sure that the list of specs exposed in index.json looks consistent and 3 | * includes the right info. 4 | */ 5 | 6 | const assert = require("assert"); 7 | const specs = require("../index.json"); 8 | const schema = require("../schema/index.json"); 9 | const dfnsSchema = require("../schema/definitions.json"); 10 | const computeShortname = require("../src/compute-shortname"); 11 | const Ajv = require("ajv"); 12 | const addFormats = require("ajv-formats") 13 | const ajv = new Ajv(); 14 | addFormats(ajv); 15 | 16 | 17 | describe("List of specs", () => { 18 | it("has a valid JSON schema", () => { 19 | const isSchemaValid = ajv.validateSchema(schema); 20 | assert.ok(isSchemaValid); 21 | }); 22 | 23 | it("respects the JSON schema", () => { 24 | const validate = ajv.addSchema(dfnsSchema).compile(schema); 25 | const isValid = validate(specs, { format: "full" }); 26 | assert.strictEqual(validate.errors, null); 27 | assert.ok(isValid); 28 | }); 29 | 30 | it("is an array of objects with url, shortname, and series properties", () => { 31 | const wrong = specs.filter(s => !(s.url && s.shortname && s.series)); 32 | assert.deepStrictEqual(wrong, []); 33 | }); 34 | 35 | it("has unique shortnames", () => { 36 | const wrong = specs.filter((spec, idx) => 37 | specs.findIndex(s => s.shortname === spec.shortname) !== idx); 38 | assert.deepStrictEqual(wrong, []); 39 | }); 40 | 41 | it("only contains HTTPS URLs", () => { 42 | const wrong = specs.filter(s => 43 | !s.url.startsWith('https:') || 44 | (s.release && !s.release.url.startsWith('https:')) || 45 | (s.nightly && !s.nightly.url.startsWith('https:'))); 46 | assert.deepStrictEqual(wrong, []); 47 | }); 48 | 49 | it("has level info for specs that have a previous link", () => { 50 | const wrong = specs.filter(s => s.seriesPrevious && !s.seriesVersion); 51 | assert.deepStrictEqual(wrong, []); 52 | }); 53 | 54 | it("has previous links for all delta specs", () => { 55 | const wrong = specs.filter(s => 56 | s.seriesComposition === "delta" && !s.seriesPrevious); 57 | assert.deepStrictEqual(wrong, []); 58 | }); 59 | 60 | it("has previous links that can be resolved to a spec", () => { 61 | const wrong = specs.filter(s => 62 | s.seriesPrevious && !specs.find(p => p.shortname === s.seriesPrevious)); 63 | assert.deepStrictEqual(wrong, []); 64 | }); 65 | 66 | it("has next links that can be resolved to a spec", () => { 67 | const wrong = specs.filter(s => 68 | s.seriesNext && !specs.find(n => n.shortname === s.seriesNext)); 69 | assert.deepStrictEqual(wrong, []); 70 | }); 71 | 72 | it("has correct next links for specs targeted by a previous link", () => { 73 | const wrong = specs.filter(s => { 74 | if (!s.seriesPrevious) { 75 | return false; 76 | } 77 | const previous = specs.find(p => p.shortname === s.seriesPrevious); 78 | return !previous || previous.seriesNext !== s.shortname; 79 | }); 80 | assert.deepStrictEqual(wrong, []); 81 | }); 82 | 83 | it("has correct previous links for specs targeted by a next link", () => { 84 | const wrong = specs.filter(s => { 85 | if (!s.seriesNext) { 86 | return false; 87 | } 88 | const next = specs.find(n => n.shortname === s.seriesNext); 89 | return !next || next.seriesPrevious !== s.shortname; 90 | }); 91 | assert.deepStrictEqual(wrong, []); 92 | }); 93 | 94 | it("does not have previous links for fork specs", () => { 95 | const wrong = specs.filter(s => 96 | s.seriesComposition === "fork" && s.seriesPrevious); 97 | assert.deepStrictEqual(wrong, []); 98 | }); 99 | 100 | it("does not have next links for fork specs", () => { 101 | const wrong = specs.filter(s => 102 | s.seriesComposition === "fork" && s.seriesNext); 103 | assert.deepStrictEqual(wrong, []); 104 | }); 105 | 106 | it("has consistent series info", () => { 107 | const wrong = specs.filter(s => { 108 | if (!s.seriesPrevious) { 109 | return false; 110 | } 111 | const previous = specs.find(p => p.shortname === s.seriesPrevious); 112 | assert.deepStrictEqual(s.series, previous.series); 113 | }); 114 | }); 115 | 116 | it("has series titles for all specs", () => { 117 | const wrong = specs.filter(s => !s.series?.title); 118 | assert.deepStrictEqual(wrong, []); 119 | }); 120 | 121 | it("has series titles that look consistent with spec titles", () => { 122 | // Note the WebRTC and JSON-LD specs follow a slightly different pattern 123 | // TEMP (2022-01-05): temp exception to the rule: published version of CSS 124 | // Images Level 4 has an obscure title à la "CSS Image Values..." 125 | // (should get fixed next time the spec gets published to /TR) 126 | const wrong = specs.filter(s => !s.title.includes(s.series.title)) 127 | .filter(s => !["webrtc", "json-ld11-api", "json-ld11-framing", "css-images-4"].includes(s.shortname)); 128 | assert.deepStrictEqual(wrong, []); 129 | }); 130 | 131 | it("has series short titles for all specs", () => { 132 | const wrong = specs.filter(s => !s.series?.shortTitle); 133 | assert.deepStrictEqual(wrong, []); 134 | }); 135 | 136 | it("contains nightly URLs for all specs", () => { 137 | const wrong = specs.filter(s => !s.nightly.url); 138 | assert.deepStrictEqual(wrong, []); 139 | }); 140 | 141 | it("contains repository URLs for all non IETF specs", () => { 142 | // Some more exceptions to the rule 143 | const wrong = specs.filter(s => !s.nightly.repository && 144 | !s.nightly.url.match(/rfc-editor\.org/) && 145 | !s.nightly.url.match(/\/Consortium\/Patent-Policy\/$/) && 146 | !s.nightly.url.match(/\/sourcemaps\.info\//) && 147 | !s.nightly.url.match(/fidoalliance\.org\//) && 148 | s.shortname !== 'SVG11'); 149 | assert.deepStrictEqual(wrong, []); 150 | }); 151 | 152 | it("contains relative paths to source of nightly spec for all non IETF specs", () => { 153 | // Some more exceptions to the rule 154 | const wrong = specs.filter(s => !s.nightly.sourcePath && 155 | !s.nightly.url.match(/rfc-editor\.org/) && 156 | !s.nightly.url.match(/\/Consortium\/Patent-Policy\/$/) && 157 | !s.nightly.url.match(/tc39\.es\/proposal\-decorators\/$/) && 158 | !s.nightly.url.match(/\/sourcemaps\.info\//) && 159 | !s.nightly.url.match(/fidoalliance\.org\//) && 160 | s.shortname !== 'SVG11'); 161 | assert.deepStrictEqual(wrong, []); 162 | }); 163 | 164 | it("contains filenames for all nightly URLs", () => { 165 | const wrong = specs.filter(s => !s.nightly.filename); 166 | assert.deepStrictEqual(wrong, []); 167 | }); 168 | 169 | it("contains filenames for all release URLs", () => { 170 | const wrong = specs.filter(s => s.release && !s.release.filename); 171 | assert.deepStrictEqual(wrong, []); 172 | }); 173 | 174 | it("has a forkOf property for all fork specs", () => { 175 | const wrong = specs.filter(s => s.seriesComposition === "fork" && !s.forkOf); 176 | assert.deepStrictEqual(wrong, []); 177 | }); 178 | 179 | it("has a fork composition level for all fork specs", () => { 180 | const wrong = specs.filter(s => s.forkOf && s.seriesComposition !== "fork"); 181 | assert.deepStrictEqual(wrong, []); 182 | }); 183 | 184 | it("only has forks of existing specs", () => { 185 | const wrong = specs.filter(s => s.forkOf && !specs.find(spec => spec.shortname === s.forkOf)); 186 | assert.deepStrictEqual(wrong, []); 187 | }); 188 | 189 | it("has consistent forks properties", () => { 190 | const wrong = specs.filter(s => !!s.forks && 191 | s.forks.find(shortname => !specs.find(spec => 192 | spec.shortname === shortname && 193 | spec.seriesComposition === "fork" && 194 | spec.forkOf === s.shortname))); 195 | assert.deepStrictEqual(wrong, []); 196 | }); 197 | 198 | it("has a w3c.github.io alternate URL for CSS drafts", () => { 199 | const wrong = specs 200 | .filter(s => s.nightly.url.match(/\/drafts\.csswg\.org/)) 201 | .filter(s => { 202 | const draft = computeShortname(s.nightly.url); 203 | return !s.nightly.alternateUrls.includes( 204 | `https://w3c.github.io/csswg-drafts/${draft.shortname}/`); 205 | }); 206 | assert.deepStrictEqual(wrong, []); 207 | }); 208 | 209 | it("has distinct source paths for all specs", () => { 210 | // ... provided entries don't share the same nightly draft 211 | // (typically the case for CSS 2.1 and CSS 2.2) 212 | const wrong = specs.filter(s => 213 | s.nightly.repository && s.nightly.sourcePath && 214 | specs.find(spec => spec !== s && 215 | spec.nightly.url !== s.nightly.url && 216 | spec.nightly.repository === s.nightly.repository && 217 | spec.nightly.sourcePath === s.nightly.sourcePath)); 218 | assert.deepStrictEqual(wrong, [], JSON.stringify(wrong, null, 2)); 219 | }); 220 | }); 221 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We welcome contributions! If you believe that a spec should be added, modified, 4 | or removed from the list, consider submitting a pull request, taking the 5 | considerations below into account. Alternatively, feel free to [raise an 6 | issue](https://github.com/w3c/browser-specs/issues/new). 7 | 8 | Please note the open source data and code [license](LICENSE.md) for this 9 | project. 10 | 11 | 12 | ## Table of Contents 13 | 14 | - [How to add/update/delete a spec](#how-to-addupdatedelete-a-spec) 15 | - [Pre-requisites](#pre-requisites) 16 | - [The actual list is in `specs.json`](#the-actual-list-is-in-specsjson) 17 | - [Compact form preferred](#compact-form-preferred) 18 | - [No `index.json` in the pull request](#no-indexjson-in-the-pull-request) 19 | - [Lint before push](#lint-before-push) 20 | - [Check before push](#check-before-push) 21 | 22 | 23 | ## How to add/update/delete a spec 24 | 25 | If you believe that a spec should be added, modified, or removed from the list, 26 | consider submitting a pull request, taking the considerations below into 27 | account. Alternatively, feel free to [raise an 28 | issue](https://github.com/w3c/browser-specs/issues/new). 29 | 30 | ### Pre-requisites 31 | 32 | To prepare a pull request, please: 33 | - check the [Spec selection criteria](README.md#spec-selection-criteria), 34 | - install [Node.js](https://nodejs.org/en/) if not already done, 35 | - fork this Git repository, 36 | - install dependencies through a call to `npm install` 37 | 38 | 39 | ### The actual list is in `specs.json` 40 | 41 | In practice, the `index.json` file is automatically generated by processing the 42 | `specs.json` file, which thus contains the actual list. In other words, all 43 | proposed changes must be made against the `specs.json` file, **do not edit the 44 | `index.json` file directly**. 45 | 46 | The `specs.json` is essentially a sorted (A-Z order) list of URLs. In most 47 | cases, to propose a new spec, all you have to do is insert its versioned URL 48 | (see [`url`](README.md#url) in README) at the right position in the list. 49 | 50 | In some cases, you may need to go beyond a simple URL because the spec does not 51 | follow usual rules and the code cannot compute the right information as a 52 | result. A spec entry in the list may also be an object with the following 53 | properties: 54 | 55 | - `url`: same as the [`url`](README.md#url) property in `index.json`. 56 | - `shortname`: same as the [`shortname`](README.md#shortname) property in 57 | `index.json`. When the `forkOf` property is also set, note that the actual 58 | shortname in the final list will be prefixed by the shortname of the base 59 | spec (as in: `${forkOf}-fork-${shortname}`). 60 | - `forkOf`: same as the [`forkOf`](README.md#forkof) property in `index.json`. 61 | No need to set `seriesComposition` to `"fork"` when this property is set, the 62 | build logic will take care of that automatically. 63 | - `series`: same as the [`series`](README.md#series) property in `index.json`, 64 | but note the `currentSpecification` property will be ignored. 65 | - `seriesVersion`: same as the [`seriesVersion`](README.md#seriesversion) 66 | property in `index.json`. 67 | - `seriesComposition`: same as the [`seriesComposition`](README.md#seriesComposition) 68 | property in `index.json`. The property must only be set for delta spec, since 69 | full is the default and fork specs are identified through the `forkOf` property 70 | in `specs.json`. 71 | - `organization`: same as the [`organization`](README.md#organization) property 72 | in `index.json` to specify the name of the organization that owns the spec. 73 | - `groups`: same as the [`groups`](README.md#groups) property in `index.json` 74 | to specify the list of groups that develop or developed the spec. 75 | - `nightly`: same as the [`nightly`](README.md#nightly) property in 76 | `index.json`. The property must only be set when: 77 | - The URL of the nightly spec returned by external sources would be wrong 78 | **and** when it cannot be fixed at the source. 79 | - The code cannot compute the right [`sourcePath`](README.md#nightlysourcepath) 80 | because the source file of the nightly spec does not follow a common pattern. 81 | - One or more alternate URLs, used in external sources, need to be recorded in 82 | an [`alternateUrls`](README.md#nightlyalternateurls) property (note the 83 | `w3c.github.io` URL of CSS drafts is automatically added as an alternate 84 | URL, no need to specify it in `specs.json`) 85 | - `tests`: same as the [`tests`](README.md#tests) property in `index.json`. The 86 | property must only be set when: 87 | - The test suite of the specification is not in a well-known repository. 88 | - The code cannot determine the correct list of [`testPaths`](README.md#teststestpaths) 89 | and/or [`excludePaths`](README.md#testsexcludepaths). 90 | - `shortTitle`: same as the [`shortTitle`](README.md#shorttitle) property in 91 | `index.json`. The property must only be set when the short title computed 92 | automatically is not the expected one. 93 | - `forceCurrent`: a boolean flag to tell the code that the spec should be seen 94 | as the current spec in the series. The property must only be set when value is 95 | `true`. 96 | - `multipage`: a boolean flag to identify the spec as a multipage spec. This 97 | instructs the code to extract the list of pages from the index page and fill 98 | out the `release.pages` and `nightly.pages` properties in the list. 99 | - `categories`: an array that is treated as incremental update to adjust the 100 | list of [`categories`](README.md#categories) that the spec belongs to. Values 101 | may be one of `"reset"` to start from an empty list, `"+browser"` to add 102 | `"browser"` to the list, and `"-browser"` to remove `"browser"` from the list. 103 | 104 | You should **only** set these properties when they are required to generate the 105 | right info. For instance, some of these properties are needed for Media Queries 106 | Level 3, because the spec uses an old shortname format, leading to the following 107 | definition in `specs.json`, to specify the version of the spec and link it to 108 | other specs in the same series: 109 | 110 | ```json 111 | { 112 | "url": "https://www.w3.org/TR/css3-mediaqueries/", 113 | "seriesVersion": "3", 114 | "series": { 115 | "shortname": "mediaqueries" 116 | } 117 | } 118 | ``` 119 | 120 | The [linter](#lint-before-push) will enforce typical constraints on the 121 | properties, such as making sure that there is only one spec flagged as current 122 | in a series. It will also complain when a property is set whereas it does not 123 | seem needed. 124 | 125 | 126 | ### Compact form preferred 127 | 128 | Some of the above properties can be specified with a keyword next to the URL of 129 | the spec, allowing to keep using a string instead of an object in most cases: 130 | - A delta spec can be defined by appending a `delta` keyword to the URL, instead 131 | of through the `seriesComposition`. 132 | - The `forceCurrent` flag can be set by appending a `current` keyword to the URL 133 | - The `multipage` flag can be set by appending a `multipage` keyword to the URL 134 | 135 | For instance, to flag the CSS Fragmentation Module Level 3 as the current spec 136 | in the series, the CSS Grid Layout Module Level 2 as a delta spec, and the SVG2 137 | spec as a multipage spec, use the following compact definitions: 138 | 139 | ```json 140 | [ 141 | "https://www.w3.org/TR/css-break-3/ current", 142 | "https://www.w3.org/TR/css-grid-2/ delta", 143 | "https://www.w3.org/TR/SVG2/ multipage" 144 | ] 145 | ``` 146 | 147 | This compact form is preferred to keep the list (somewhat) human-readable. The 148 | [linter](#lint-before-push) automatically convert objects to the more compact 149 | string format whenever possible. 150 | 151 | 152 | ### No `index.json` in the pull request 153 | 154 | The `index.json` file will be automatically generated once your pull request has 155 | been merged. Please do not include it in your pull request. You may still wish 156 | to re-generate the file (see the [Check before push](#check-before-push) section 157 | below) to check that the generated info will be correct, but please don't commit 158 | these changes. 159 | 160 | 161 | ### Lint before push 162 | 163 | Before you push your changes and submit a pull request, please run the linter 164 | to identify potential linting issues: 165 | 166 | ```bash 167 | npm run lint 168 | ``` 169 | 170 | If the linter reports errors that can be fixed (e.g. wrong spec order, or more 171 | compact form needed), run the following command to overwrite your local 172 | `specs.json` file with the linted version. 173 | 174 | ```bash 175 | npm run lint-fix 176 | ``` 177 | 178 | **Note:** The linter cannot fix broken JSON and/or incorrect properties. Please 179 | fix these errors manually and run the linter again. 180 | 181 | 182 | ### Check before push 183 | 184 | Before you push your changes and submit a pull request, you may also want to 185 | check that the changes will produce the right info. You may 186 | [re-generate the file](README.md#how-to-generate-indexjson-manually) but 187 | generation typically takes several minutes. To only generate the entries that 188 | match the specs that you changed in `specs.json`, you may use the 189 | [diff tool](README.md#build-a-diff-of-indexjson). -------------------------------------------------------------------------------- /test/specs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Make sure that the specs.json respects the JSON schema and all constraints 3 | * that cannot be automatically linted. 4 | * 5 | * Note: The specs.json file may still need to be linted, and that's all fine! 6 | */ 7 | 8 | const assert = require("assert"); 9 | const specs = require("../specs.json"); 10 | const schema = require("../schema/specs.json"); 11 | const dfnsSchema = require("../schema/definitions.json"); 12 | const computeInfo = require("../src/compute-shortname.js"); 13 | const computePrevNext = require("../src/compute-prevnext.js"); 14 | const Ajv = require("ajv"); 15 | const addFormats = require("ajv-formats") 16 | const ajv = (new Ajv()).addSchema(dfnsSchema); 17 | addFormats(ajv); 18 | 19 | // When an entry is invalid, the schema validator returns one error for each 20 | // "oneOf" option and one error on overall "oneOf" problem. This is confusing 21 | // for humans. The following function improves the error being returned. 22 | function clarifyErrors(errors) { 23 | if (!errors) { 24 | return errors; 25 | } 26 | 27 | // Update instancePath to drop misleading "[object Object]" 28 | errors.forEach(err => 29 | err.instancePath = err.instancePath.replace(/^\[object Object\]/, '')); 30 | 31 | if (errors.length < 2) { 32 | return errors; 33 | } 34 | 35 | // If first two errors are type errors for oneOf choices, item is neither 36 | // a string nor an object 37 | if ((errors[0].schemaPath === "#/items/oneOf/0/type") && 38 | (errors[1].schemaPath === "#/items/oneOf/1/type")) { 39 | return [ 40 | Object.assign(errors[0], { "message": "must be a string or an object" }) 41 | ]; 42 | } 43 | 44 | // Otherwise, if second error is a type error for second oneOf choice, 45 | // it means the item is actually a string that represents an invalid URL, 46 | // which the first error should capture. 47 | if (errors[1].schemaPath === "#/items/oneOf/1/type") { 48 | return [errors[0]]; 49 | } 50 | 51 | // Otherwise, item is an object that does not follow the schema, drop the 52 | // error that says that item is not a string and the error that says that it 53 | // does not meet one of the "oneOf" options. What remains should be the error 54 | // that explains why the item does not meet the schema for the object. 55 | const clearerErrors = errors.filter(error => 56 | (error.schemaPath !== "#/items/oneOf/0/type") && 57 | (error.schemaPath !== "#/items/oneOf")); 58 | 59 | // Improve an additional property message to point out the property that 60 | // should not be there (default message does not say it) 61 | clearerErrors.forEach(error => { 62 | if ((error.keyword === "additionalProperties") && 63 | error.params && error.params.additionalProperty) { 64 | error.message = "must not have additional property '" + 65 | error.params.additionalProperty + "'"; 66 | } 67 | }); 68 | 69 | // If there are no more errors left to return, roll back to the initial set 70 | // to make sure an error gets reported. That should never happen, but better 71 | // be ready for it. 72 | return (clearerErrors.length > 0) ? clearerErrors : errors; 73 | } 74 | 75 | function compareSpecs(a, b) { 76 | return a.url.localeCompare(b.url); 77 | } 78 | 79 | function specs2objects(specs) { 80 | return specs 81 | .map(spec => (typeof spec === "string") ? 82 | { 83 | url: new URL(spec.split(" ")[0]).toString(), 84 | seriesComposition: (spec.split(' ')[1] === "delta") ? "delta" : "full", 85 | forceCurrent: (spec.split(' ')[1] === "current"), 86 | multipage: (spec.split(' ')[1] === "multipage"), 87 | } : 88 | Object.assign({}, spec, { url: new URL(spec.url).toString() })) 89 | .filter((spec, idx, list) => 90 | !list.find((s, i) => i < idx && compareSpecs(s, spec) === 0)); 91 | } 92 | 93 | function specs2LinkedList(specs) { 94 | return specs2objects(specs) 95 | .map(s => Object.assign({}, s, computeInfo(s.shortname || s.url, s.forkOf))) 96 | .map((s, _, list) => Object.assign({}, s, computePrevNext(s, list))); 97 | } 98 | 99 | function check(specs) { 100 | const validate = ajv.compile(schema); 101 | const isValid = validate(specs, { format: "full" }); 102 | const msg = ajv.errorsText(clarifyErrors(validate.errors), { 103 | dataVar: "specs", separator: "\n" 104 | }); 105 | return msg; 106 | } 107 | 108 | 109 | describe("Input list", () => { 110 | describe("JSON schema", () => { 111 | it("is valid", () => { 112 | const isSchemaValid = ajv.validateSchema(schema); 113 | assert.ok(isSchemaValid); 114 | }); 115 | 116 | it("rejects list if it is not an array", () => { 117 | const specs = 0; 118 | assert.strictEqual(check(specs), "specs must be array"); 119 | }); 120 | 121 | it("rejects an empty list", () => { 122 | const specs = []; 123 | assert.strictEqual(check(specs), "specs must NOT have fewer than 1 items"); 124 | }); 125 | 126 | it("rejects items that have a wrong type", () => { 127 | const specs = [0]; 128 | assert.strictEqual(check(specs), "specs/0 must be a string or an object"); 129 | }); 130 | 131 | it("rejects spec objects without URL", () => { 132 | const specs = [{}]; 133 | assert.strictEqual(check(specs), "specs/0 must have required property 'url'"); 134 | }); 135 | 136 | it("rejects spec objects with an invalid URL", () => { 137 | const specs = [{ url: "invalid" }]; 138 | assert.strictEqual(check(specs), "specs/0/url must match format \"uri\""); 139 | }); 140 | 141 | it("rejects spec objects with additional properties", () => { 142 | const specs = [{ url: "https://example.org/", invalid: "test" }]; 143 | assert.strictEqual(check(specs), "specs/0 must not have additional property 'invalid'"); 144 | }); 145 | }); 146 | 147 | 148 | describe("specs.json", () => { 149 | it("respects the JSON schema", () => { 150 | assert.strictEqual(check(specs), 'No errors'); 151 | }); 152 | 153 | it("only points at valid URLs", () => { 154 | specs.forEach(spec => (typeof spec === "string") ? 155 | new URL(spec.split(" ")[0]).toString() : null); 156 | assert.ok(true); 157 | }) 158 | 159 | it("only contains specs for which a shortname can be generated", () => { 160 | // Convert entries to spec objects and compute shortname 161 | const specsWithoutShortname = specs2objects(specs) 162 | .map(spec => Object.assign({}, spec, computeInfo(spec.shortname || spec.url, spec.forkOf))) 163 | .filter(spec => !spec.shortname); 164 | 165 | // No exception thrown? That means we're good! 166 | // We'll just check that there aren't any spec with an empty name and report 167 | // the first one (That should never happen since computeInfo would throw but 168 | // better be safe) 169 | assert.strictEqual(specsWithoutShortname[0], undefined); 170 | }); 171 | 172 | it("does not have a delta spec without a previous full spec", () => { 173 | const fullPrevious = (spec, list) => { 174 | const previous = list.find(s => s.shortname === spec.seriesPrevious); 175 | if (previous && previous.seriesComposition && previous.seriesComposition !== "full") { 176 | return fullPrevious(previous, list); 177 | } 178 | return previous; 179 | }; 180 | const deltaWithoutFull = specs2LinkedList(specs) 181 | .filter((s, _, list) => s.seriesComposition === "delta" && !fullPrevious(s, list)); 182 | assert.strictEqual(deltaWithoutFull[0], undefined); 183 | }); 184 | 185 | it("does not have a delta spec flagged as 'current'", () => { 186 | const deltaCurrent = specs2LinkedList(specs) 187 | .filter(s => s.forceCurrent && s.seriesComposition === "delta"); 188 | assert.strictEqual(deltaCurrent[0], undefined); 189 | }); 190 | 191 | it("does not have a fork spec flagged as 'current'", () => { 192 | const forkCurrent = specs2LinkedList(specs) 193 | .filter(s => s.forceCurrent && s.forkOf); 194 | assert.strictEqual(forkCurrent[0], undefined); 195 | }); 196 | 197 | it("has only one spec flagged as 'current' per series shortname", () => { 198 | const linkedList = specs2LinkedList(specs); 199 | const problematicCurrent = linkedList 200 | .filter(s => s.forceCurrent) 201 | .filter(s => s !== linkedList.find(p => 202 | p.series.shortname === s.series.shortname && p.forceCurrent)); 203 | assert.strictEqual(problematicCurrent[0], undefined); 204 | }); 205 | 206 | it("does not have a spec with a 'fork' seriesComposition property", () => { 207 | const wrong = specs.find(s => s.seriesComposition === "fork"); 208 | assert.strictEqual(wrong, undefined); 209 | }); 210 | 211 | it("does not have a 'delta fork' spec", () => { 212 | const wrong = specs.find(s => s.forkOf && s.seriesComposition === "delta"); 213 | assert.strictEqual(wrong, undefined); 214 | }); 215 | 216 | it("only has fork specs that reference existing specs", () => { 217 | const linkedList = specs2LinkedList(specs); 218 | const forkWithoutFull = linkedList.filter((s, _, list) => s.forkOf && 219 | !linkedList.find(spec => spec.shortname === s.forkOf)); 220 | assert.strictEqual(forkWithoutFull[0], undefined); 221 | }); 222 | }); 223 | }); 224 | -------------------------------------------------------------------------------- /src/find-specs.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const fs = require("fs"); 3 | 4 | const core = require('@actions/core'); 5 | 6 | const fetch = require("node-fetch"); 7 | 8 | const {JSDOM} = require("jsdom"); 9 | 10 | const computeShortname = require("./compute-shortname"); 11 | 12 | const specs = require("../index.json"); 13 | const ignorable = require("./data/ignore.json"); 14 | const monitorList = require("./data/monitor.json"); 15 | 16 | const {repos: temporarilyIgnorableRepos, specs: temporarilyIgnorableSpecs} = monitorList; 17 | 18 | const nonBrowserSpecWgs = Object.keys(ignorable.groups); 19 | const watchedBrowserCgs = [ 20 | "Web Platform Incubator Community Group", 21 | "Web Assembly Community Group", 22 | "Immersive Web Community Group", 23 | "Audio Community Group", 24 | "Privacy Community Group", 25 | "GPU for the Web Community Group" 26 | ]; 27 | const cssMetaDir = ["shared", "indexes", "bin", ".github", "css-module", "css-module-bikeshed"]; 28 | const svgMetaDir = ["template"]; 29 | const fxtfMetaDir = [".github", "shared"]; 30 | const houdiniMetaDir = [".github", "images"]; 31 | 32 | function canonicalizeGhUrl(r) { 33 | const url = new URL(r.homepageUrl); 34 | url.protocol = 'https:'; 35 | if (url.pathname.lastIndexOf('/') === 0 && url.pathname.length > 1) { 36 | url.pathname += '/'; 37 | } 38 | return {repo: r.owner.login + '/' + r.name, spec: url.toString()}; 39 | } 40 | 41 | function canonicalizeTRUrl(url) { 42 | url = new URL(url); 43 | url.protocol = 'https:'; 44 | return url.toString(); 45 | } 46 | 47 | const trimSlash = url => url.endsWith('/') ? url.slice(0, -1) : url; 48 | const toGhUrl = repo => { return {repo: `${repo.owner.login}/${repo.name}`, spec: `https://${repo.owner.login.toLowerCase()}.github.io/${repo.name}/`}; }; 49 | const matchRepoName = fullName => r => fullName === r.owner.login + '/' + r.name; 50 | const isRelevantRepo = fullName => !Object.keys(ignorable.repos).includes(fullName) && !Object.keys(temporarilyIgnorableRepos).includes(fullName); 51 | const isInScope = ({spec: url, repo: fullName}) => 52 | !Object.keys(ignorable.specs).includes(url) && 53 | !Object.keys(temporarilyIgnorableSpecs).includes(url) && 54 | isRelevantRepo(fullName); 55 | // Set loose parameter when checking loosely if another version exists 56 | const hasMoreRecentLevel = (s, url, loose) => { 57 | try { 58 | const shortnameData = computeShortname(url); 59 | return s.series.shortname === shortnameData.series.shortname 60 | && (s.seriesVersion > (shortnameData.seriesVersion ?? '') 61 | || loose && (s.seriesVersion === shortnameData.seriesVersion 62 | // case of CSS drafts whose known editors drafts are version-less, but the directories in the repo use versions 63 | || !s.seriesVersion 64 | // Case of houdini drafts whose known editors drafts are versioned, but the directories in the repo use version-less 65 | || (!shortnameData.seriesVersion && s.seriesVersion == 1) 66 | )); 67 | } catch (e) { 68 | return false; 69 | } 70 | }; 71 | const hasUntrackedURL = ({spec: url}) => !specs.find(s => s.nightly.url.startsWith(trimSlash(url)) 72 | || (s.release && trimSlash(s.release.url) === trimSlash(url))) 73 | && !specs.find(s => hasMoreRecentLevel(s, url, url.match(/\/drafts\./) && !url.match(/\/w3\.org/) // Because CSS specs have editors draft with and without levels, we look loosely for more recent levels when checking with editors draft 74 | )); 75 | const hasUnknownTrSpec = ({spec: url}) => !specs.find(s => s.release && trimSlash(s.release.url) === trimSlash(url)) && !specs.find(s => hasMoreRecentLevel(s,url)); 76 | 77 | const hasRepoType = type => r => r.w3c && r.w3c["repo-type"] 78 | && (r.w3c["repo-type"] === type || r.w3c["repo-type"].includes(type)); 79 | const hasPublishedContent = (candidate) => fetch(candidate.spec).then(({ok, url}) => { 80 | if (ok) return {...candidate, spec: url}; 81 | }); 82 | 83 | (async function() { 84 | let candidates = []; 85 | 86 | const {groups, repos} = await fetch("https://w3c.github.io/validate-repos/report.json").then(r => r.json()); 87 | const specRepos = await fetch("https://w3c.github.io/spec-dashboard/repo-map.json").then(r => r.json()); 88 | const whatwgSpecs = await fetch("https://raw.githubusercontent.com/whatwg/sg/master/db.json").then(r => r.json()) 89 | .then(d => d.workstreams.map(w => w.standards.map(s => { return {...s, id: s.href.replace(/.*\/([a-z]+)\.spec\.whatwg\.org\//, '$1')}; }) ).flat()); 90 | const cssSpecs = await fetch("https://api.github.com/repos/w3c/csswg-drafts/contents/").then(r => r.json()).then(data => data.filter(p => p.type === "dir" && !cssMetaDir.includes(p.path)).map(p => p.path)); 91 | const svgSpecs = await fetch("https://api.github.com/repos/w3c/svgwg/contents/specs").then(r => r.json()).then(data => data.filter(p => p.type === "dir" && !svgMetaDir.includes(p.name)).map(p => p.path)); 92 | const fxtfSpecs = await fetch("https://api.github.com/repos/w3c/fxtf-drafts/contents/").then(r => r.json()).then(data => data.filter(p => p.type === "dir" && !fxtfMetaDir.includes(p.path)).map(p => p.path)); 93 | const houdiniSpecs = await fetch("https://api.github.com/repos/w3c/css-houdini-drafts/contents/").then(r => r.json()).then(data => data.filter(p => p.type === "dir" && !houdiniMetaDir.includes(p.path)).map(p => p.path)); 94 | 95 | const ecmaProposals = await JSDOM.fromURL("https://github.com/tc39/proposals/blob/master/README.md") 96 | // we only watch stage 3 proposals, which are in the first table on the page above 97 | .then(dom => [...dom.window.document.querySelector("table").querySelectorAll("tr td:first-child a")].map(a => a.href)); 98 | 99 | const ecmaIntlProposals = await JSDOM.fromURL("https://github.com/tc39/proposals/blob/master/ecma402/README.md") 100 | // we only watch stage 3 proposals, which are in the first table on the page above 101 | .then(dom => [...dom.window.document.querySelector("table").querySelectorAll("tr td:first-child a")].map(a => a.href)); 102 | 103 | const chromeFeatures = await fetch("https://www.chromestatus.com/features.json").then(r => r.json()); 104 | 105 | const wgs = Object.values(groups).filter(g => g.type === "working group" && !nonBrowserSpecWgs.includes(g.name)); 106 | const cgs = Object.values(groups).filter(g => g.type === "community group" && watchedBrowserCgs.includes(g.name)); 107 | 108 | // WGs 109 | // * check repos with w3c.json/repo-type including rec-track 110 | const wgRepos = wgs.map(g => g.repos.map(r => r.fullName)).flat() 111 | .map(fullName => repos.find(matchRepoName(fullName))); 112 | const recTrackRepos = wgRepos.filter(hasRepoType('rec-track')); 113 | 114 | // * look if those with homepage URLs have a match in the list of specs 115 | candidates = recTrackRepos.filter(r => r.homepageUrl) 116 | .map(canonicalizeGhUrl) 117 | .filter(hasUntrackedURL) 118 | .filter(isInScope); 119 | 120 | // * look if those without a homepage URL have a match with their generated URL 121 | candidates = candidates.concat((await Promise.all(recTrackRepos.filter(r => !r.homepageUrl) 122 | .map(toGhUrl) 123 | .filter(hasUntrackedURL) 124 | .filter(isInScope) 125 | .map(hasPublishedContent))).filter(x => x)); 126 | 127 | // Look which of the specRepos on recTrack from a browser-producing WG have no match 128 | candidates = candidates.concat( 129 | Object.keys(specRepos).map( 130 | r => specRepos[r].filter(s => s.recTrack && wgs.find(g => g.id === s.group)).map(s => { return {repo: r, spec: canonicalizeTRUrl(s.url)};})) 131 | .flat() 132 | .filter(hasUnknownTrSpec) 133 | .filter(isInScope) 134 | ); 135 | 136 | // CGs 137 | //check repos with w3c.json/repo-type includes cg-report or with no w3c.json 138 | const cgRepos = cgs.map(g => g.repos.map(r => r.fullName)).flat() 139 | .map(fullName => repos.find(matchRepoName(fullName))); 140 | 141 | const cgSpecRepos = cgRepos.filter(r => !r.w3c 142 | || hasRepoType('cg-report')(r)); 143 | // * look if those with homepage URLs have a match in the list of specs 144 | candidates = candidates.concat(cgSpecRepos.filter(r => r.homepageUrl) 145 | .map(canonicalizeGhUrl) 146 | .filter(hasUntrackedURL) 147 | .filter(isInScope) 148 | ); 149 | 150 | // for those without homepageUrl, check which have published content 151 | const publishedCandidates = (await Promise.all(cgSpecRepos.filter(r => !r.homepageUrl) 152 | .map(toGhUrl) 153 | .filter(hasUntrackedURL) 154 | .filter(isInScope) 155 | .map(hasPublishedContent) 156 | )).filter(x => x); 157 | 158 | candidates = candidates.concat(publishedCandidates); 159 | 160 | // * look if those without homepage URLs but marked as a cg-report 161 | // have a match in the list of specs 162 | const monitorAdditions = cgSpecRepos 163 | .filter(r => !r.homepageUrl && hasRepoType('cg-report')(r) && 164 | !publishedCandidates.find(p => p.repo === `${r.owner.login}/${r.name}`)) 165 | .map(toGhUrl) 166 | .filter(hasUntrackedURL) 167 | .filter(isInScope) 168 | // we remove the spec field since we haven't found a usable url 169 | .map(c => Object.assign({}, {repo: c.repo})); 170 | 171 | // Check for new WHATWG streams 172 | candidates = candidates.concat(whatwgSpecs.map(s => { return {repo: `whatwg/${s.id}`, spec: s.href};}) 173 | .filter(hasUntrackedURL) 174 | .filter(isInScope)); 175 | 176 | 177 | // Check for new CSS specs 178 | candidates = candidates.concat(cssSpecs.map(s => { return {repo: "w3c/csswg-drafts", spec: `https://drafts.csswg.org/${s}/`};}) 179 | .filter(hasUntrackedURL) 180 | .filter(isInScope)); 181 | 182 | // Check for new SVG specs 183 | candidates = candidates.concat(svgSpecs.map(s => { return {repo: "w3c/svgwg", spec: `https://svgwg.org/${s}/`};}) 184 | .filter(hasUntrackedURL) 185 | .filter(isInScope)); 186 | 187 | // Check for new FXTF specs 188 | candidates = candidates.concat(fxtfSpecs.map(s => { return {repo: "w3c/fxtf-drafts", spec: `https://drafts.fxtf.org/${s}/`};}) 189 | .filter(hasUntrackedURL) 190 | .filter(isInScope)); 191 | 192 | // Check for new Houdini specs 193 | candidates = candidates.concat(houdiniSpecs.map(s => { return {repo: "w3c/css-houdini-drafts", spec: `https://drafts.css-houdini.org/${s}/`};}) 194 | .filter(hasUntrackedURL) 195 | .filter(isInScope)); 196 | 197 | // Check for new TC39 Stage 3 proposals 198 | candidates = candidates.concat(ecmaProposals.concat(ecmaIntlProposals).map(s => { return {repo: s.replace('https://github.com/', ''), spec: s.replace('https://github.com/tc39/', 'https://tc39.es/').replace('https://github.com/tc39-transfer/', 'https://tc39.es/') + '/'};}) 199 | .filter(hasUntrackedURL) 200 | .filter(isInScope)); 201 | 202 | 203 | // Add information from Chrome Feature status 204 | candidates = candidates.map(c => { return {...c, impl: { chrome: (chromeFeatures.find(f => f.standards.spec && f.standards.spec.startsWith(c.spec)) || {}).id}};}); 205 | 206 | const candidate_list = candidates.sort((c1, c2) => c1.spec.localeCompare(c2.spec)) 207 | .map(c => `- [ ] ${c.spec} from [${c.repo}](https://github.com/${c.repo})` + (c.impl.chrome ? ` [chrome status](https://www.chromestatus.com/features/${c.impl.chrome})` : '')).join("\n"); 208 | core.exportVariable("candidate_list", candidate_list); 209 | console.log(candidate_list); 210 | if (monitorAdditions.length) { 211 | const today = new Date().toJSON().slice(0, 10); 212 | const monitored = monitorAdditions.map(({repo}) => `- [ ] [${repo}](https://github.com/${repo})`).join("\n"); 213 | core.exportVariable("monitor_list", monitored); 214 | monitorAdditions.forEach(({repo}) => { 215 | monitorList.repos[repo] = { 216 | lastreviewed: today, 217 | comment: "no published content yet" 218 | }; 219 | }); 220 | fs.writeFileSync("./src/data/monitor.json", JSON.stringify(monitorList, null, 2)); 221 | console.log(monitored); 222 | } 223 | })().catch(e => { 224 | console.error(e); 225 | process.exit(1); 226 | }); 227 | -------------------------------------------------------------------------------- /src/fetch-info.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module that exports a function that takes an array of specifications objects 3 | * that each have at least a "url" and a "short" property. The function returns 4 | * an object indexed by specification "shortname" with additional information 5 | * about the specification fetched from the W3C API, Specref, or from the spec 6 | * itself. Object returned for each specification contains the following 7 | * properties: 8 | * 9 | * - "nightly": an object that describes the nightly version. The object will 10 | * feature the URL of the Editor's Draft for W3C specs, of the living document 11 | * for WHATWG specifications, or of the published Khronos Group specification. 12 | * The object may also feature the URL of the repository that hosts the nightly 13 | * version of the spec. 14 | * - "release": an object that describes the published version. The object will 15 | * feature the URL of the TR document for W3C specs when it exists, and is not 16 | * present for specs that don't have release versions (WHATWG specs, CG drafts). 17 | * - "title": the title of the specification. Always set. 18 | * - "source": one of "w3c", "specref", "spec", depending on how the information 19 | * was determined. 20 | * 21 | * The function throws when something goes wrong, e.g. if the given spec object 22 | * describes a /TR/ specification but the specification has actually not been 23 | * published to /TR/, if the specification cannot be fetched, or if no W3C API 24 | * key was specified for a /TR/ URL. 25 | * 26 | * The function will start by querying the W3C API, using the given "shortname" 27 | * properties. For specifications where this fails, the function will query 28 | * SpecRef, using the given "shortname" as well. If that too fails, the function 29 | * assumes that the given "url" is the URL of the Editor's Draft, and will fetch 30 | * it to determine the title. 31 | * 32 | * The function needs an API key to fetch the W3C API, which can be passed 33 | * within an "options" object with a "w3cApiKey" property. 34 | * 35 | * If the function needs to retrieve the spec itself, note that it will parse 36 | * the HTTP response body as a string, applying regular expressions to extract 37 | * the title. It will not parse it as HTML in particular. This means that the 38 | * function will fail if the title cannot easily be extracted for some reason. 39 | * 40 | * Note: the function operates on a list of specs and not only on one spec to 41 | * bundle requests to Specref. 42 | */ 43 | 44 | const throttle = require("./throttle"); 45 | const baseFetch = require("node-fetch"); 46 | const fetch = throttle(baseFetch, 2); 47 | const computeShortname = require("./compute-shortname"); 48 | const {JSDOM} = require("jsdom"); 49 | const JSDOMFromURL = throttle(JSDOM.fromURL, 2); 50 | 51 | 52 | 53 | async function fetchInfoFromW3CApi(specs, options) { 54 | // Cannot query the W3C API if API key was not given 55 | if (!options || !options.w3cApiKey) { 56 | return []; 57 | } 58 | options.headers = options.headers || {}; 59 | options.headers.Authorization = `W3C-API apikey="${options.w3cApiKey}"`; 60 | 61 | const info = await Promise.all(specs.map(async spec => { 62 | // Skip specs when the known URL is not a /TR/ URL, because there may still 63 | // be a spec with the same name published to /TR/ but that is probably an 64 | // outdated version (e.g. WHATWG specs such as DOM or Fullscreen, or CSS 65 | // drafts published a long long time ago) 66 | if (!spec.url.match(/^https?:\/\/(?:www\.)?w3\.org\/TR\/([^\/]+)\/$/)) { 67 | return; 68 | } 69 | 70 | const url = `https://api.w3.org/specifications/${spec.shortname}`; 71 | const res = await fetch(url, options); 72 | if (res.status === 404) { 73 | return; 74 | } 75 | if (res.status === 301) { 76 | const rawLocation = res.headers.get('location'); 77 | const location = rawLocation.startsWith('/specifications/') ? 78 | rawLocation.substring('/specifications/'.length) : 79 | rawLocation.location; 80 | throw new Error(`W3C API redirected to "${location}" ` + 81 | `for "${spec.shortname}" (${spec.url}), update the shortname!`); 82 | } 83 | if (res.status !== 200) { 84 | throw new Error(`W3C API returned an error, status code is ${res.status}`); 85 | } 86 | try { 87 | const body = await res.json(); 88 | return body; 89 | } 90 | catch (err) { 91 | throw new Error("W3C API returned invalid JSON"); 92 | } 93 | })); 94 | 95 | const seriesShortnames = new Set(); 96 | const results = {}; 97 | specs.forEach((spec, idx) => { 98 | if (info[idx]) { 99 | if (info[idx].shortlink && 100 | info[idx].shortlink.startsWith('http:')) { 101 | console.warn(`[warning] force HTTPS for release of ` + 102 | `"${spec.shortname}", W3C API returned "${info[idx].shortlink}"`); 103 | } 104 | if (info[idx]["editor-draft"] && 105 | info[idx]["editor-draft"].startsWith('http:')) { 106 | console.warn(`[warning] force HTTPS for nightly of ` + 107 | `"${spec.shortname}", W3C API returned "${info[idx]["editor-draft"]}"`); 108 | } 109 | const release = info[idx].shortlink ? 110 | info[idx].shortlink.replace(/^http:/, 'https:') : 111 | null; 112 | const nightly = info[idx]["editor-draft"] ? 113 | info[idx]["editor-draft"].replace(/^http:/, 'https:') : 114 | null; 115 | 116 | results[spec.shortname] = { 117 | release: { url: release }, 118 | nightly: { url: nightly }, 119 | title: info[idx].title 120 | }; 121 | 122 | if (spec.series?.shortname) { 123 | seriesShortnames.add(spec.series.shortname); 124 | } 125 | } 126 | }); 127 | 128 | // Fetch info on the series 129 | const seriesInfo = await Promise.all([...seriesShortnames].map(async shortname => { 130 | const url = `https://api.w3.org/specification-series/${shortname}`; 131 | const res = await fetch(url, options); 132 | if (res.status === 404) { 133 | return; 134 | } 135 | if (res.status !== 200) { 136 | throw new Error(`W3C API returned an error, status code is ${res.status}`); 137 | } 138 | try { 139 | const body = await res.json(); 140 | return body; 141 | } 142 | catch (err) { 143 | throw new Error("W3C API returned invalid JSON"); 144 | } 145 | })); 146 | 147 | results.__series = {}; 148 | seriesInfo.forEach(info => { 149 | const currSpecUrl = info._links["current-specification"].href; 150 | const currSpec = currSpecUrl.substring(currSpecUrl.lastIndexOf('/') + 1); 151 | results.__series[info.shortname] = { 152 | title: info.name, 153 | currentSpecification: currSpec 154 | }; 155 | }); 156 | 157 | return results; 158 | } 159 | 160 | 161 | async function fetchInfoFromSpecref(specs, options) { 162 | function chunkArray(arr, len) { 163 | let chunks = []; 164 | let i = 0; 165 | let n = arr.length; 166 | while (i < n) { 167 | chunks.push(arr.slice(i, i += len)); 168 | } 169 | return chunks; 170 | } 171 | 172 | const chunks = chunkArray(specs, 50); 173 | const chunksRes = await Promise.all(chunks.map(async chunk => { 174 | let specrefUrl = "https://api.specref.org/bibrefs?refs=" + 175 | chunk.map(spec => spec.shortname).join(','); 176 | 177 | const res = await fetch(specrefUrl, options); 178 | if (res.status !== 200) { 179 | throw new Error(`Could not query Specref, status code is ${res.status}`); 180 | } 181 | try { 182 | const body = await res.json(); 183 | return body; 184 | } 185 | catch (err) { 186 | throw new Error("Specref returned invalid JSON"); 187 | } 188 | })); 189 | 190 | const results = {}; 191 | chunksRes.forEach(chunkRes => { 192 | 193 | // Specref manages aliases, let's follow the chain to the final spec 194 | function resolveAlias(name, counter) { 195 | counter = counter || 0; 196 | if (counter > 100) { 197 | throw "Too many aliases returned by Respec"; 198 | } 199 | if (chunkRes[name].aliasOf) { 200 | return resolveAlias(chunkRes[name].aliasOf, counter + 1); 201 | } 202 | else { 203 | return name; 204 | } 205 | } 206 | Object.keys(chunkRes).forEach(name => { 207 | if (specs.find(spec => spec.shortname === name)) { 208 | const info = chunkRes[resolveAlias(name)]; 209 | if (info.edDraft && info.edDraft.startsWith('http:')) { 210 | console.warn(`[warning] force HTTPS for nightly of ` + 211 | `"${spec.shortname}", Specref returned "${info.edDraft}"`); 212 | } 213 | if (info.href && info.href.startsWith('http:')) { 214 | console.warn(`[warning] force HTTPS for nightly of ` + 215 | `"${spec.shortname}", Specref returned "${info.href}"`); 216 | } 217 | const nightly = info.edDraft ? 218 | info.edDraft.replace(/^http:/, 'https:') : 219 | info.href ? info.href.replace(/^http:/, 'https:') : 220 | null; 221 | results[name] = { 222 | nightly: { url: nightly }, 223 | title: info.title 224 | }; 225 | } 226 | }); 227 | }); 228 | 229 | return results; 230 | } 231 | 232 | 233 | async function fetchInfoFromSpecs(specs, options) { 234 | const info = await Promise.all(specs.map(async spec => { 235 | const url = spec.nightly?.url || spec.url; 236 | // Force use of more stable w3c.github.io address for CSS drafts 237 | let fetchUrl = url; 238 | if (url.match(/\/drafts\.csswg\.org/)) { 239 | const draft = computeShortname(url); 240 | fetchUrl = `https://w3c.github.io/csswg-drafts/${draft.shortname}/`; 241 | } 242 | let dom = null; 243 | try { 244 | dom = await JSDOMFromURL(fetchUrl); 245 | } 246 | catch (err) { 247 | throw new Error(`Could not retrieve ${fetchUrl} with JSDOM: ${err.message}`); 248 | } 249 | 250 | if (spec.url.startsWith("https://tc39.es/")) { 251 | // Title is either flagged with specific class or the second h1 252 | const h1ecma = 253 | dom.window.document.querySelector('#spec-container h1.title') ?? 254 | dom.window.document.querySelectorAll("h1")[1]; 255 | if (h1ecma) { 256 | return { 257 | nightly: { url: url }, 258 | title: h1ecma.textContent.replace(/\n/g, '').trim() 259 | }; 260 | } 261 | } 262 | 263 | // Extract first heading when set 264 | let title = dom.window.document.querySelector("h1"); 265 | if (!title) { 266 | // Use the document's title if first heading could not be found 267 | // (that typically happens in Respec specs) 268 | title = dom.window.document.querySelector("title"); 269 | } 270 | 271 | if (title) { 272 | title = title.textContent.replace(/\n/g, '').trim(); 273 | 274 | // The draft CSS specs server sometimes goes berserk and returns 275 | // the contents of the directory instead of the actual spec. Let's 276 | // throw an error when that happens so as not to create fake titles. 277 | if (title.startsWith('Index of ')) { 278 | throw new Error(`CSS server issue detected in ${url} for ${spec.shortname}`); 279 | } 280 | 281 | return { 282 | nightly: { url }, 283 | title 284 | }; 285 | } 286 | 287 | throw new Error(`Could not find title in ${url} for ${spec.shortname}`); 288 | })); 289 | 290 | const results = {}; 291 | specs.forEach((spec, idx) => results[spec.shortname] = info[idx]); 292 | return results; 293 | } 294 | 295 | 296 | /** 297 | * Main function that takes a list of specifications and returns an object 298 | * indexed by specification "shortname" that provides, for each specification, 299 | * the URL of the Editor's Draft, of the /TR/ version, and the title. 300 | */ 301 | async function fetchInfo(specs, options) { 302 | if (!specs || specs.find(spec => !spec.shortname || !spec.url)) { 303 | throw "Invalid list of specifications passed as parameter"; 304 | } 305 | 306 | options = Object.assign({}, options); 307 | options.timeout = options.timeout || 30000; 308 | 309 | // Compute information from W3C API 310 | let remainingSpecs = specs; 311 | const w3cInfo = await fetchInfoFromW3CApi(remainingSpecs, options); 312 | 313 | // Compute information from Specref for remaining specs 314 | remainingSpecs = remainingSpecs.filter(spec => !w3cInfo[spec.shortname]); 315 | const specrefInfo = await fetchInfoFromSpecref(remainingSpecs, options); 316 | 317 | // Extract information directly from the spec for remaining specs 318 | remainingSpecs = remainingSpecs.filter(spec => !specrefInfo[spec.shortname]); 319 | const specInfo = await fetchInfoFromSpecs(remainingSpecs, options); 320 | 321 | // Merge results 322 | const results = {}; 323 | specs.map(spec => spec.shortname).forEach(name => results[name] = 324 | (w3cInfo[name] ? Object.assign(w3cInfo[name], { source: "w3c" }) : null) || 325 | (specrefInfo[name] ? Object.assign(specrefInfo[name], { source: "specref" }) : null) || 326 | (specInfo[name] ? Object.assign(specInfo[name], { source: "spec" }) : null)); 327 | 328 | // Add series info from W3C API 329 | results.__series = w3cInfo.__series; 330 | 331 | return results; 332 | } 333 | 334 | 335 | module.exports = fetchInfo; 336 | --------------------------------------------------------------------------------