├── .github
├── dependabot.yml
└── workflows
│ ├── lint.yml
│ └── test.yml
├── .gitignore
├── .husky
├── .gitignore
└── pre-commit
├── .prettierrc
├── LICENSE
├── README.md
├── eslint.config.cjs
├── generators
└── respec.js
├── index.html
├── package.json
├── server.js
├── test
├── mocha.opts
└── test.js
└── w3c.json
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 | updates:
3 | - package-ecosystem: npm
4 | directory: "/"
5 | schedule:
6 | interval: weekly
7 | time: '00:00'
8 | open-pull-requests-limit: 20
9 | ignore:
10 | - dependency-name: "*"
11 | update-types: ["version-update:semver-patch"]
12 | # ignore minor updates for eslint and any packages starting "eslint"
13 | - dependency-name: "eslint*"
14 | update-types: ['version-update:semver-minor']
15 | - dependency-name: "nodemon"
16 | update-types: ['version-update:semver-minor']
17 | - dependency-name: "husky"
18 | update-types: ['version-update:semver-minor']
19 | - package-ecosystem: github-actions
20 | directory: '/'
21 | schedule:
22 | interval: weekly
23 | time: '00:00'
24 | open-pull-requests-limit: 10
25 |
--------------------------------------------------------------------------------
/.github/workflows/lint.yml:
--------------------------------------------------------------------------------
1 | name: Check Lint
2 |
3 | on: [push, pull_request]
4 |
5 | jobs:
6 | lint:
7 | runs-on: ubuntu-latest
8 | strategy:
9 | matrix:
10 | node-version: [18.x, 20.x]
11 | steps:
12 | - uses: actions/checkout@v4
13 |
14 | - name: Use Node.js ${{ matrix.node-version }}
15 | uses: actions/setup-node@v4.4.0
16 | with:
17 | node-version: ${{ matrix.node-version }}
18 |
19 | - run: npm install
20 |
21 | - run: npm run lint
22 |
--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
1 | name: spec-generator tests
2 |
3 | on: [push, pull_request, workflow_dispatch]
4 |
5 | jobs:
6 | test:
7 | runs-on: ubuntu-22.04
8 | strategy:
9 | matrix:
10 | node-version: [20.x, 22.x]
11 | steps:
12 | - uses: actions/checkout@v4
13 | - name: Use Node.js ${{ matrix.node-version }}
14 | uses: actions/setup-node@v4.4.0
15 | with:
16 | node-version: ${{ matrix.node-version }}
17 |
18 | - run: npm install
19 | - run: npx respec2html -e --timeout 30 --src "https://w3c.github.io/vc-di-ecdsa/"
20 | - run: npm test
21 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 |
5 | # Runtime data
6 | pids
7 | *.pid
8 | *.seed
9 |
10 | # Directory for instrumented libs generated by jscoverage/JSCover
11 | lib-cov
12 |
13 | # Coverage directory used by tools like istanbul
14 | coverage
15 |
16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
17 | .grunt
18 |
19 | # Compiled binary addons (http://nodejs.org/api/addons.html)
20 | build/Release
21 |
22 | # temporary uploaded files
23 | uploads
24 |
25 | # Dependency directory
26 | # Commenting this out is preferred by some people, see
27 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git-
28 | node_modules
29 |
30 | # Users Environment Variables
31 | .lock-wscript
32 | package-lock.json
33 |
--------------------------------------------------------------------------------
/.husky/.gitignore:
--------------------------------------------------------------------------------
1 | _
2 |
--------------------------------------------------------------------------------
/.husky/pre-commit:
--------------------------------------------------------------------------------
1 | npm run lint
2 |
--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "singleQuote": false,
3 | "trailingComma": "all",
4 | "tabWidth": 4,
5 | "arrowParens": "avoid"
6 | }
7 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2014 World Wide Web Consortium
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](https://travis-ci.com/w3c/spec-generator)
2 |
3 | # Spec Generator
4 |
5 | This exposes a service to automatically generate specs from various source formats.
6 |
7 | ## API
8 |
9 | ### Via HTTP
10 |
11 | Start the server listening on port 80 by default:
12 |
13 | ```bash
14 | node server
15 | ```
16 |
17 | You can specify a port like so:
18 |
19 | ```bash
20 | PORT=3000 node server
21 | ```
22 |
23 | When developing, you can use auto-reload:
24 |
25 | ```bash
26 | nodemon server
27 | ```
28 |
29 | Spec Generator has a single endpoint, which is a `GET /`. This endpoint accepts parameters on its
30 | query string. If the call is successful the generated content of the specification is returned.
31 |
32 | * `type` (required). The type of generator for this content. Currently the only supported value is
33 | `respec`.
34 | * `url` (required). The URL of the draft to fetch and generate from.
35 | * `publishDate`. The date at which the publication of this draft is supposed to occur.
36 |
37 | ### As a Node.js module
38 |
39 | ```js
40 | const SPEC_GEN = require('w3c-spec-generator');
41 | const SERVER = SPEC_GEN.start(); // Optional port number (80 by default)
42 | // Now Spec Generator is listening on port 80
43 | SERVER.close(); // To stop the server
44 | ```
45 |
46 | ### Errors
47 |
48 | If a required parameter is missing or has a value that is not understood, the generator returns a
49 | `500` error with a JSON payload the `error` field of which is the human-readable error message.
50 |
51 | If the specific generator encounters a problem a similar error (mostly likely `500`) with the same
52 | sort of JSON message is returned. Specific generator types can extend this behaviour. The `respec`
53 | generator only returns `500` errors.
54 |
55 | The HTTP response status code is `200` even when there are processing errors and warnings. Processing errors and warnings are signaled with the help of `x-errors-count` and `x-warnings-count` response headers respectively instead.
56 |
57 | ## Writing generators
58 |
59 | Generators are simple to write and new ones can easily be added. Simply add a new one under
60 | `generators` and load it into the `genMap` near the top of `server.js`.
61 |
62 | Generators must export a `generate()` method which takes a URL, a set of parameters (from the list
63 | of optional ones that the API supports), and a callback to invoke upon completion.
64 |
65 | If there is an error, the callback's first argument must be an object with a `status` field being
66 | an HTTP error code and a `message` field containing the error message. If the generator is
67 | successful the first argument is `null` and the second is the generated content.
68 |
--------------------------------------------------------------------------------
/eslint.config.cjs:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "extends": ["airbnb-base", "prettier"],
4 | "plugins": ["prettier"],
5 | "rules": {
6 | "prettier/prettier": "error",
7 | "func-names": "off",
8 | "vars-on-top": "off",
9 | "consistent-return": "off"
10 | },
11 | "overrides": [
12 | {
13 | "files": ["index.html"],
14 | "plugins": [
15 | "html"
16 | ]
17 | },
18 | {
19 | "files": ["server.js", "generators/*.js", "tools/**/*.js"],
20 | "plugins": ["node"],
21 | "extends": ["plugin:node/recommended"]
22 | },
23 | {
24 | "files": ["test/*.js"],
25 | "env": {
26 | "mocha": true
27 | },
28 | "rules": {
29 | "node/no-unpublished-require": "off"
30 | },
31 | "plugins": ["node"],
32 | "extends": "plugin:node/recommended"
33 | }
34 | ]
35 | }
36 | ]
37 |
--------------------------------------------------------------------------------
/generators/respec.js:
--------------------------------------------------------------------------------
1 | import { toHTML } from "respec";
2 |
3 | class SpecGeneratorError extends Error {
4 | constructor({ status, message }) {
5 | super(message);
6 | this.status = status;
7 | }
8 | }
9 |
10 | export async function generate(url) {
11 | try {
12 |
13 | console.log("Generating", url);
14 | const { html, errors, warnings } = await toHTML(url, {
15 | timeout: 30000,
16 | disableSandbox: true,
17 | disableGPU: true,
18 | });
19 | return { html, errors: errors.length, warnings: warnings.length };
20 | } catch (err) {
21 | throw new SpecGeneratorError({ status: 500, message: err.message });
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | Spec Generator
7 |
83 |
84 |
85 |
106 |
109 |
110 |
111 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "w3c-spec-generator",
3 | "version": "1.2.8",
4 | "license": "MIT",
5 | "type": "module",
6 | "dependencies": {
7 | "express": "5.1.0",
8 | "express-fileupload": "1.5.0",
9 | "file-type": "21.0.0",
10 | "jsdom": "26.1.0",
11 | "mkdirp": "3.0",
12 | "node-fetch": "3.3.0",
13 | "request": "2.88.2",
14 | "respec": "35.4.0",
15 | "tar-stream": "3.1.1"
16 | },
17 | "devDependencies": {
18 | "eslint": "9.0.0",
19 | "eslint-config-prettier": "10.0.1",
20 | "eslint-plugin-html": "8.0.0",
21 | "eslint-plugin-node": "11.1.0",
22 | "eslint-plugin-prettier": "5.0.0-alpha.2",
23 | "husky": "^9.0.6",
24 | "mocha": "11.7.0",
25 | "nodemon": "3.0.1",
26 | "prettier": "3.6.2"
27 | },
28 | "engines": {
29 | "node": "20 || 22",
30 | "npm": ">=7"
31 | },
32 | "scripts": {
33 | "lint": "eslint .",
34 | "test": "mocha --timeout 30000",
35 | "prepare": "husky install"
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/server.js:
--------------------------------------------------------------------------------
1 | import { extname, dirname } from "path";
2 | import { URL, URLSearchParams } from "url";
3 | import { readFile, unlink, rm, mkdtemp, writeFile } from "fs/promises";
4 | import { readFileSync, mkdirSync, createWriteStream } from "fs";
5 |
6 | import express from "express";
7 | import fileUpload from "express-fileupload";
8 | import { fileTypeFromBuffer } from "file-type";
9 | import tar from "tar-stream";
10 | import { JSDOM } from "jsdom";
11 | import request from "request";
12 | import { mkdirp } from "mkdirp";
13 | import fetch from "node-fetch";
14 |
15 | import { generate } from "./generators/respec.js";
16 |
17 | const genMap = {
18 | respec: generate,
19 | };
20 |
21 | const app = express();
22 | const BASE_URI = process.env.BASE_URI || "";
23 |
24 | const FORM_HTML = readFileSync("index.html", "utf-8");
25 |
26 | /** Get present date in YYYY-MM-DD format */
27 | const getShortIsoDate = () => new Date().toISOString().slice(0, 10);
28 |
29 | app.use(
30 | fileUpload({
31 | createParentPath: true,
32 | useTempFiles: true,
33 | tempFileDir: "uploads/",
34 | }),
35 | );
36 |
37 | async function extractTar(tarFile) {
38 | const extract = tar.extract();
39 | const uploadPath = await mkdtemp("uploads/");
40 |
41 | function uploadedFileIsAllowed(name) {
42 | if (name.toLowerCase().includes(".htaccess")) return false;
43 | if (name.toLowerCase().includes(".php")) return false;
44 | if (name.includes("CVS")) return false;
45 | if (name.includes("../")) return false;
46 | if (name.includes("://")) return false;
47 | return true;
48 | }
49 |
50 | return new Promise((resolve, reject) => {
51 | let hasIndex = false;
52 | extract.on("entry", (header, stream, next) => {
53 | stream.on("data", async data => {
54 | if (uploadedFileIsAllowed(header.name)) {
55 | if (header.name === "index.html" || header.name === "./index.html") {
56 | hasIndex = true;
57 | }
58 | const filePath = `${uploadPath}/${header.name}`;
59 | mkdirp.sync(dirname(filePath));
60 | await writeFile(filePath, data);
61 | }
62 | });
63 | stream.on("end", () => next());
64 | stream.resume();
65 | });
66 |
67 | extract.on("finish", () => {
68 | if (!hasIndex) {
69 |
70 | reject("No index.html file");
71 | } else {
72 | resolve(uploadPath);
73 | }
74 | });
75 |
76 | extract.end(tarFile);
77 | });
78 | }
79 |
80 | // Listens to GET at the root, expects two required query string parameters:
81 | // type: the type of the generator (case-insensitive)
82 | // url: the URL to the source document
83 | app.get(
84 | "/",
85 | async (req, res, next) => {
86 | const type =
87 | typeof req.query.type === "string"
88 | ? req.query.type.toLowerCase()
89 | : undefined;
90 | const url =
91 | typeof req.query.url === "string"
92 | ? decodeURIComponent(req.query.url)
93 | : undefined;
94 | if (!url || !type) {
95 | if (
96 | req.headers.accept &&
97 | req.headers.accept.includes("text/html")
98 | ) {
99 | return res.send(FORM_HTML);
100 | }
101 | return res
102 | .status(500)
103 | .json({ error: "Both 'type' and 'url' are required." });
104 | }
105 |
106 | if (!genMap.hasOwnProperty(req.query.type)) {
107 | return res
108 | .status(500)
109 | .json({ error: `Unknown generator: ${req.query.type}` });
110 | }
111 | const specURL = new URL(url);
112 | req.targetURL = req.query.url;
113 | if (specURL.hostname === "raw.githubusercontent.com") {
114 | const uploadPath = await mkdtemp("uploads/");
115 | const originalDocument = await fetch(url);
116 | const baseRegex =
117 | /https:\/\/raw.githubusercontent.com\/.+?\/.+?\/.+?\//;
118 | const basePath = req.query.url.match(baseRegex)[0];
119 | const jsdom = new JSDOM(await originalDocument.text());
120 | const refs =
121 | jsdom.window.document.querySelectorAll("[href], [src], [data-include]");
122 | const index = url.replace(/(\?|#).+/, "");
123 | const links = [index];
124 | refs.forEach(ref => {
125 | if (ref && (ref.href || ref.src || ref.dataset.include)) {
126 | const u = new URL(
127 | (ref.href || ref.src || ref.dataset.include)
128 | .replace("about:blank", "")
129 | .replace(/(\?|#).+/, ""),
130 | url.replace(/(\?|#).+/, ""),
131 | );
132 | if (
133 | u.href.startsWith(basePath) &&
134 | !links.includes(u.href)
135 | ) {
136 | links.push(u.href);
137 | }
138 | }
139 | });
140 |
141 | links.forEach(async l => {
142 | const name = l.replace(basePath, "");
143 | mkdirSync(`${uploadPath}/${dirname(name)}`, {
144 | recursive: true,
145 | });
146 | const response = await fetch(l);
147 | response.body.pipe(createWriteStream(`${uploadPath}/${name}`));
148 | });
149 |
150 | const baseUrl = `${req.protocol}://${req.get("host")}/`;
151 | const newPath = url.replace(baseRegex, `${uploadPath}/`);
152 | req.targetURL = `${baseUrl}${newPath}${specURL.search}`;
153 | req.tmpDir = uploadPath;
154 | }
155 | next();
156 | },
157 | async (req, res) => {
158 | const specURL = new URL(req.targetURL);
159 | const publishDate =
160 | specURL.searchParams.get("publishDate") || getShortIsoDate();
161 |
162 | specURL.searchParams.set("publishDate", publishDate);
163 |
164 | // if there's an error we get an err object with status and message, otherwise we get content
165 | try {
166 | const { html, errors, warnings } = await genMap[req.query.type](
167 | specURL.href,
168 | );
169 | res.setHeader("x-errors-count", errors);
170 | res.setHeader("x-warnings-count", warnings);
171 | res.send(html);
172 | } catch (err) {
173 | res.status(err.status).json({ error: err.message });
174 | }
175 | if (req.tmpDir) {
176 | rm(req.tmpDir, { recursive: true });
177 | }
178 | },
179 | );
180 |
181 | app.use(
182 | "/uploads",
183 | express.static("./uploads", {
184 | setHeaders(res, requestPath) {
185 | const noExtension = !extname(requestPath);
186 | if (noExtension) res.setHeader("Content-Type", "text/html");
187 | },
188 | }),
189 | );
190 |
191 | app.post("/", async (req, res) => {
192 | if (!req.files || !req.files.file) {
193 | return res.send({
194 | status: 500,
195 | message: "No file uploaded",
196 | });
197 | }
198 |
199 | try {
200 | const { tempFilePath } = req.files.file;
201 |
202 | // file can be an html file or a tar file
203 | const content = await readFile(tempFilePath);
204 | const type = await fileTypeFromBuffer(content);
205 | const path =
206 | type && type.mime === "application/x-tar"
207 | ? await extractTar(content)
208 | : // assume it's an HTML file
209 | tempFilePath;
210 |
211 | const baseUrl = `${req.protocol}://${req.get("host")}/`;
212 | const params = new URLSearchParams(req.body).toString();
213 | const src = `${baseUrl}${path}?${params}`;
214 | const qs = { url: src, type: "respec" };
215 | request.get({ url: baseUrl, qs }, (err, response, body) => {
216 | if (err) {
217 | res.status(500).send(err);
218 | } else {
219 | res.setHeader(
220 | "x-errors-count",
221 | response.headers["x-errors-count"],
222 | );
223 | res.setHeader(
224 | "x-warnings-count",
225 | response.headers["x-warnings-count"],
226 | );
227 | res.send(body);
228 | // delete temp file
229 | unlink(tempFilePath);
230 | rm(path, { recursive: true });
231 | }
232 | });
233 | } catch (err) {
234 | res.status(500).send(err);
235 | }
236 | });
237 |
238 | /**
239 | * Start listening for HTTP requests.
240 | * @param {number} [port] - port number to use (optional); defaults to environment variable `$PORT` if exists, and to `80` if not
241 | */
242 | app.start = (port = parseInt(process.env.PORT, 10) || 8000) => app.listen(port);
243 |
244 | const server = app.start();
245 | export default app;
246 | export { server };
247 |
--------------------------------------------------------------------------------
/test/mocha.opts:
--------------------------------------------------------------------------------
1 | --colors
2 | --growl
3 | --reporter spec
4 | --timeout 30000
5 |
--------------------------------------------------------------------------------
/test/test.js:
--------------------------------------------------------------------------------
1 | import ASSERT from "assert";
2 | import REQUEST from "request";
3 | import app, { server } from "../server.js";
4 |
5 | const PORT = 3000;
6 | const BASE_URL = "http://localhost:3000/";
7 | const NO_URL = "?type=foo&URL=notice-that-its-in-uppercase";
8 | const NO_TYPE = "?url=foo&TYPE=notice-that-its-in-uppercase";
9 | const BAD_GENERATOR = "?type=fluxor&url=http://example.com/";
10 | const NO_RESPEC = "?type=respec&url=http://example.com/";
11 | const SUCCESS1 = `?type=respec&url=https://w3c.github.io/manifest/`;
12 | const SUCCESS2 = `?type=respec&url=https://w3c.github.io/payment-request/`;
13 | const SUCCESS3 = `?type=respec&url=https://w3c.github.io/vc-di-ecdsa/`;
14 |
15 | const FAILS_WITH =
16 | (
17 | done,
18 | expectedMessage = "{\"error\":\"Both 'type' and 'url' are required.\"}",
19 | expectedCode = 500,
20 | ) =>
21 | (error, response, body) => {
22 | ASSERT.equal(error, null);
23 | ASSERT.equal(response.statusCode, expectedCode);
24 | if (expectedMessage instanceof RegExp)
25 | ASSERT.ok(body.match(expectedMessage));
26 | else ASSERT.equal(body, expectedMessage);
27 | done();
28 | };
29 | const SUCCEEDS = done => (error, response) => {
30 | ASSERT.equal(error, null);
31 | ASSERT.equal(response.statusCode, 200);
32 | ASSERT.equal(response.statusMessage, "OK");
33 | done();
34 | };
35 | let testserver;
36 |
37 | describe("spec-generator", () => {
38 | before(() => {
39 | server.close();
40 | testserver = app.start(PORT);
41 | });
42 |
43 | describe("fails when it should", () => {
44 | it("without parameters", done =>
45 | REQUEST.get(BASE_URL, FAILS_WITH(done)));
46 | it("if there's no URL", done =>
47 | REQUEST.get(BASE_URL + NO_URL, FAILS_WITH(done)));
48 | it("if there's no type", done =>
49 | REQUEST.get(BASE_URL + NO_TYPE, FAILS_WITH(done)));
50 | it("if the generator is not valid", done =>
51 | REQUEST.get(
52 | BASE_URL + BAD_GENERATOR,
53 | FAILS_WITH(done, '{"error":"Unknown generator: fluxor"}'),
54 | ));
55 | it("if the URL does not point to a Respec document", done =>
56 | REQUEST.get(
57 | BASE_URL + NO_RESPEC,
58 | FAILS_WITH(
59 | done,
60 | /That doesn't seem to be a ReSpec document. Please check manually:/,
61 | ),
62 | ));
63 | });
64 |
65 | describe("succeeds when it should", () => {
66 | describe("renders form UI", () => {
67 | it("without parameters", done =>
68 | REQUEST.get(
69 | BASE_URL,
70 | { headers: { Accept: "text/html" } },
71 | SUCCEEDS(done),
72 | ));
73 | it("if there's no URL", done =>
74 | REQUEST.get(
75 | BASE_URL + NO_URL,
76 | { headers: { Accept: "text/html" } },
77 | SUCCEEDS(done),
78 | ));
79 | it("if there's no type", done =>
80 | REQUEST.get(
81 | BASE_URL + NO_TYPE,
82 | { headers: { Accept: "text/html" } },
83 | SUCCEEDS(done),
84 | ));
85 | });
86 | it('Web App Manifest ("appmanifest")', done =>
87 | REQUEST.get(BASE_URL + SUCCESS1, SUCCEEDS(done)));
88 | it('Payment Request API ("payment-request")', done =>
89 | REQUEST.get(BASE_URL + SUCCESS2, SUCCEEDS(done)));
90 | it('Resource Hints ("vc-di-ecdsa")', done =>
91 | REQUEST.get(BASE_URL + SUCCESS3, SUCCEEDS(done)));
92 | });
93 |
94 | after(() => testserver.close());
95 | });
96 |
--------------------------------------------------------------------------------
/w3c.json:
--------------------------------------------------------------------------------
1 | {
2 | "contacts": ["deniak"],
3 | "repo-type": "tool"
4 | }
5 |
--------------------------------------------------------------------------------