├── .prettierrc ├── .gitignore ├── .npmignore ├── src ├── index.ts ├── cheerio.ts ├── logger.ts ├── wiki-api-client.test.ts ├── wiki-api-client.ts ├── types.ts ├── cli.ts ├── wiki-scraper.test.ts └── wiki-scraper.ts ├── .vscode ├── settings.json └── launch.json ├── tsconfig.json ├── LICENSE ├── package.json └── README.md /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "useTabs": true 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | output/ 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | output/ 3 | src/ 4 | .prettierrc 5 | tsconfig.json 6 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./types.js"; 2 | export * from "./wiki-api-client.js"; 3 | export * from "./wiki-scraper.js"; 4 | -------------------------------------------------------------------------------- /src/cheerio.ts: -------------------------------------------------------------------------------- 1 | export function isTagElement(element: unknown): element is cheerio.TagElement { 2 | return ( 3 | typeof element === "object" && 4 | element !== null && 5 | "type" in element && 6 | element.type === "tag" 7 | ); 8 | } 9 | -------------------------------------------------------------------------------- /src/logger.ts: -------------------------------------------------------------------------------- 1 | import { pino } from "pino"; 2 | 3 | const logger = pino({ 4 | base: null, 5 | timestamp: pino.stdTimeFunctions.isoTime, 6 | transport: { 7 | target: "pino-pretty", 8 | }, 9 | }); 10 | 11 | export default logger; 12 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[typescript]": { 3 | "editor.defaultFormatter": "esbenp.prettier-vscode", 4 | "editor.formatOnSave": true 5 | }, 6 | "[json][jsonc]": { 7 | "editor.defaultFormatter": "esbenp.prettier-vscode", 8 | "editor.formatOnSave": true 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "type": "node", 6 | "request": "launch", 7 | "name": "Launch Program", 8 | "skipFiles": ["/**"], 9 | "cwd": "${workspaceFolder}", 10 | "runtimeArgs": ["-r", "ts-node/register"], 11 | "args": ["src/cli.ts"], 12 | "console": "internalConsole", 13 | "outputCapture": "std" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "strictBindCallApply": true, 4 | "strictFunctionTypes": true, 5 | "strictNullChecks": true, 6 | "emitDecoratorMetadata": true, 7 | "esModuleInterop": true, 8 | "experimentalDecorators": true, 9 | "outDir": "dist", 10 | "target": "ES2022", 11 | "moduleResolution": "NodeNext", 12 | "module": "NodeNext", 13 | "declaration": true 14 | }, 15 | "include": ["src/**/*.ts"], 16 | "exclude": ["src/**/*.test.ts"] 17 | } 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Dominique Mattern 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gmod-wiki-scraper", 3 | "version": "2.0.0-rc.1", 4 | "description": "Extract GLua API documentation from the new Garry's Mod wiki", 5 | "exports": "./dist/index.js", 6 | "types": "./dist/index.d.ts", 7 | "type": "module", 8 | "engines": { 9 | "node": ">=18" 10 | }, 11 | "bin": { 12 | "gmod-wiki-scraper": "dist/cli.js" 13 | }, 14 | "scripts": { 15 | "start": "tsx src/cli.ts", 16 | "build": "tsc", 17 | "lint": "eslint \"src/**/*.ts\"", 18 | "prepublishOnly": "npm run build", 19 | "test": "vitest" 20 | }, 21 | "author": "Dominique Mattern ", 22 | "license": "MIT", 23 | "dependencies": { 24 | "cheerio": "^1.0.0", 25 | "cli-progress": "^3.12.0", 26 | "got": "^13.0.0", 27 | "p-limit": "^6.1.0", 28 | "pino": "^9.4.0", 29 | "pino-pretty": "^11.2.2", 30 | "yargs": "^17.7.2" 31 | }, 32 | "devDependencies": { 33 | "@types/cheerio": "^0.22.35", 34 | "@types/cli-progress": "^3.11.6", 35 | "@types/yargs": "^17.0.33", 36 | "prettier": "^3.3.3", 37 | "tsx": "^4.19.1", 38 | "typescript": "^5.6.2", 39 | "vitest": "^2.1.2" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/wiki-api-client.test.ts: -------------------------------------------------------------------------------- 1 | import { beforeAll, describe, expect, it } from "vitest"; 2 | import { WikiApiClient } from "./wiki-api-client"; 3 | 4 | describe("WikiApiClient", () => { 5 | let wikiApiClient: WikiApiClient; 6 | 7 | beforeAll(() => { 8 | wikiApiClient = new WikiApiClient(); 9 | }); 10 | 11 | it("retrieves the page about math.pi", async () => { 12 | const pageUrl = "math.pi"; 13 | 14 | // TODO: The HTTP client should be mocked 15 | const page = await wikiApiClient.retrievePage(pageUrl); 16 | 17 | expect(page.title).toBe("math.pi"); 18 | expect(page.content).toContain("3.1415"); 19 | }); 20 | 21 | it("throws an error when page is not found", async () => { 22 | const pageUrl = "this-page-does-not-exist"; 23 | 24 | await expect(wikiApiClient.retrievePage(pageUrl)).rejects.toThrow( 25 | "Page with url 'this-page-does-not-exist' was not found", 26 | ); 27 | }); 28 | 29 | it("renders a preview of the text", async () => { 30 | const text = '```lua\nprint("Hello, world!")\n```'; 31 | 32 | // TODO: The HTTP client should be mocked 33 | const preview = await wikiApiClient.renderText(text); 34 | 35 | expect(preview.html).toContain("Hello, world!"); 36 | // The rendered code should contain a link to the Global.print page 37 | expect(preview.html).toContain("/gmod/Global.print"); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /src/wiki-api-client.ts: -------------------------------------------------------------------------------- 1 | import got from "got"; 2 | 3 | import { PagePreviewResponse, WikiPage, PageJsonResponse } from "./types.js"; 4 | 5 | export class WikiApiClient { 6 | public static readonly wikiUrl = "https://wiki.facepunch.com"; 7 | public static readonly wikiApiUrl = `${WikiApiClient.wikiUrl}/api`; 8 | 9 | public async retrievePage(pageUrl: string): Promise { 10 | // Remove '/' or '/gmod/' prefixes 11 | let pageUrlNormalized = pageUrl.startsWith("/") 12 | ? pageUrl.substring(1) 13 | : pageUrl; 14 | pageUrlNormalized = pageUrlNormalized.startsWith("gmod/") 15 | ? pageUrlNormalized.substring(5) 16 | : pageUrlNormalized; 17 | 18 | const response = await got( 19 | `${WikiApiClient.wikiUrl}/gmod/${pageUrlNormalized}?format=json`, 20 | ).json(); 21 | 22 | if (response.title === "Page Not Found") { 23 | throw new Error(`Page with url '${pageUrl}' was not found`); 24 | } 25 | 26 | return { 27 | content: response.markup, 28 | title: response.title, 29 | }; 30 | } 31 | 32 | public async renderText(text: string): Promise { 33 | const body = { text: text, realm: "gmod" }; 34 | const response = await got(`${WikiApiClient.wikiApiUrl}/page/preview`, { 35 | method: "POST", 36 | json: body, 37 | }).json(); 38 | 39 | return response; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export type Realm = "client" | "menu" | "server"; 2 | 3 | export interface PagePreviewResponse { 4 | status: string; 5 | html: string; 6 | title: string | null; 7 | } 8 | 9 | export interface PageLink { 10 | url: string; 11 | label: string; 12 | icon: string; 13 | description: string; 14 | } 15 | 16 | export interface PageJsonResponse { 17 | title: string; 18 | wikiName: string; 19 | wikiIcon: string; 20 | wikiUrl: string; 21 | address: string; 22 | createdTime: string; 23 | updatedCount: number; 24 | markup: string; 25 | html: string; 26 | footer: string; 27 | revisionId: number; 28 | pageLinks: Array; 29 | } 30 | 31 | export interface WikiPage { 32 | title: string; 33 | content: string; 34 | } 35 | 36 | export interface Class { 37 | name: string; 38 | parent?: string; 39 | description?: string; 40 | functions?: Array; 41 | } 42 | 43 | export interface Panel { 44 | parent: string; 45 | description?: string; 46 | } 47 | 48 | export interface FunctionArgument { 49 | name: string; 50 | type: string; 51 | default?: string; 52 | description?: string; 53 | } 54 | 55 | export interface FunctionReturnValue { 56 | name?: string; 57 | type: string; 58 | description?: string; 59 | } 60 | 61 | export interface FunctionOverload { 62 | arguments: Array; 63 | returnValues?: Array; 64 | } 65 | 66 | export interface FunctionSource { 67 | file: string; 68 | lineStart: number; 69 | lineEnd?: number; 70 | } 71 | 72 | export interface Function { 73 | name: string; 74 | parent: string; 75 | source?: FunctionSource; 76 | description?: string; 77 | realms: Array; 78 | arguments?: Array; 79 | returnValues?: Array; 80 | overloads?: Array; 81 | } 82 | 83 | export interface Type { 84 | name: string; 85 | description?: string; 86 | } 87 | 88 | export interface EnumField { 89 | name: string; 90 | description?: string; 91 | value: number; 92 | } 93 | 94 | export interface Enum { 95 | name?: string; 96 | description?: string; 97 | fields: Array; 98 | realms: Array; 99 | } 100 | 101 | export interface StructField { 102 | name: string; 103 | type: string; 104 | default?: string; 105 | description?: string; 106 | } 107 | 108 | export interface Struct { 109 | name?: string; 110 | description?: string; 111 | fields: Array; 112 | realms: Array; 113 | } 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GMod Wiki Scraper 2 | 3 | This application allows you to scrape the **new** Garry's Mod wiki which can be 4 | accessed via https://wiki.facepunch.com/gmod/. 5 | 6 | ## Prerequisites 7 | 8 | - Node.js `>= 18.17` 9 | - npm `>= 7` 10 | 11 | npm often comes bundled with Node.js so you probably won't need to install that 12 | separately. 13 | 14 | ## Installation 15 | 16 | ``` 17 | npm install -g gmod-wiki-scraper 18 | ``` 19 | 20 | ## Usage 21 | 22 | ``` 23 | gmod-wiki-scraper 24 | ``` 25 | 26 | This will retrieve all functions, hooks, enums, etc. from the wiki and save 27 | them as JSON files into an output directory in your current working directory. 28 | 29 | gmod-wiki-scraper features a rudimentary cli: 30 | 31 | ``` 32 | $ gmod-wiki-scraper --help 33 | 34 | Usage: gmod-wiki-scraper [OPTIONS] 35 | 36 | Options: 37 | --help Show help [boolean] 38 | --version, -v Print the version of gmod-wiki-scraper [boolean] 39 | --skip-global-functions Do not retrieve global functions [boolean] 40 | --skip-classes Do not retrieve classes [boolean] 41 | --skip-libraries Do not retrieve libraries [boolean] 42 | --skip-hooks Do not retrieve hooks [boolean] 43 | --skip-panels Do not retrieve panels [boolean] 44 | --skip-enums Do not retrieve enums [boolean] 45 | --skip-structs Do not retrieve structs [boolean] 46 | ``` 47 | 48 | ## API 49 | 50 | ### Documentation 51 | 52 | WIP 53 | 54 | ### Examples 55 | 56 | #### Retrieving a single page 57 | 58 | This will retrieve the content of [Global.Entity](https://wiki.facepunch.com/gmod/Global.Entity) and parse it. 59 | 60 | ```typescript 61 | import { WikiApiClient, WikiScraper } from 'gmod-wiki-scraper'; 62 | 63 | const client = new WikiApiClient(); 64 | const scraper = new WikiScraper(client); 65 | 66 | const page = await client.retrievePage('Global.Entity') 67 | const parsedFunctionPage = scraper.parseFunctionPage(page.content); 68 | 69 | console.log(parsedFunctionPage); 70 | ``` 71 | 72 | Output: 73 | 74 | ``` 75 | { 76 | name: 'Entity', 77 | parent: 'Global', 78 | realms: [ 'client', 'server' ], 79 | description: 'Returns the entity with the matching Entity:EntIndex.\n' + 80 | '\n' + 81 | 'Indices 1 through game.MaxPlayers() are always reserved for players.\n' + 82 | '\n' + 83 | "In examples on this wiki, **Entity( 1 )** is used when a player entity is needed (see ). In singleplayer and listen servers, **Entity( 1 )** will always be the first player. In dedicated servers, however, **Entity( 1 )** won't always be a valid player.", 84 | arguments: [ 85 | { 86 | name: 'entityIndex', 87 | type: 'number', 88 | description: 'The entity index.' 89 | } 90 | ], 91 | returnValues: [ 92 | { 93 | type: 'Entity', 94 | description: "The entity if it exists, or NULL if it doesn't." 95 | } 96 | ] 97 | } 98 | ``` 99 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import fs from "node:fs"; 4 | import path from "node:path"; 5 | import yargs from "yargs"; 6 | import { hideBin } from "yargs/helpers"; 7 | 8 | import { WikiScraper } from "./wiki-scraper.js"; 9 | import { WikiApiClient } from "./wiki-api-client.js"; 10 | import logger from "./logger.js"; 11 | 12 | const argv = yargs(hideBin(process.argv)) 13 | .usage("Usage: $0 [OPTIONS]") 14 | .options({ 15 | "skip-global-functions": { 16 | boolean: true, 17 | describe: "Do not retrieve global functions", 18 | }, 19 | "skip-classes": { 20 | boolean: true, 21 | describe: "Do not retrieve classes", 22 | }, 23 | "skip-libraries": { 24 | boolean: true, 25 | describe: "Do not retrieve libraries", 26 | }, 27 | "skip-hooks": { 28 | boolean: true, 29 | describe: "Do not retrieve hooks", 30 | }, 31 | "skip-panels": { 32 | boolean: true, 33 | describe: "Do not retrieve panels", 34 | }, 35 | "skip-enums": { 36 | boolean: true, 37 | describe: "Do not retrieve enums", 38 | }, 39 | "skip-structs": { 40 | boolean: true, 41 | describe: "Do not retrieve structs", 42 | }, 43 | }) 44 | .parseSync(); 45 | 46 | const outputDir = "output"; 47 | 48 | function ensureOutputDirExists(): void { 49 | try { 50 | fs.mkdirSync(outputDir); 51 | } catch (error) { 52 | if (error.code === "EEXIST") { 53 | logger.info(`Output directory '${outputDir}' already exists.`); 54 | } else { 55 | throw error; 56 | } 57 | } 58 | } 59 | 60 | function writeToDisk(filename: string, data: any): void { 61 | fs.writeFileSync( 62 | path.join(outputDir, filename), 63 | JSON.stringify(data, null, 2), 64 | ); 65 | } 66 | 67 | (async (): Promise => { 68 | ensureOutputDirExists(); 69 | 70 | const wikiApiClient = new WikiApiClient(); 71 | const wikiScraper = new WikiScraper(wikiApiClient); 72 | 73 | logger.info(`Starting scraping data from ${WikiApiClient.wikiUrl}`); 74 | logger.info("This might take a few minutes so please be patient"); 75 | 76 | if (argv.skipGlobalFunctions) { 77 | logger.info("Skipping global functions (1 / 7)"); 78 | } else { 79 | logger.info("Retrieving global functions (1 / 7)"); 80 | writeToDisk( 81 | "global-functions.json", 82 | await wikiScraper.getGlobalFunctions(), 83 | ); 84 | } 85 | 86 | if (argv.skipClasses) { 87 | logger.info("Skipping classes (2 / 7)"); 88 | } else { 89 | logger.info("Retrieving classes (2 / 7)"); 90 | writeToDisk("classes.json", await wikiScraper.getClasses()); 91 | } 92 | 93 | if (argv.skipLibraries) { 94 | logger.info("Skipping libraries (3 / 7)"); 95 | } else { 96 | logger.info("Retrieving libraries (3 / 7)"); 97 | writeToDisk("libraries.json", await wikiScraper.getLibraries()); 98 | } 99 | 100 | if (argv.skipHooks) { 101 | logger.info("Skipping hooks (4 / 7)"); 102 | } else { 103 | logger.info("Retrieving hooks (4 / 7)"); 104 | writeToDisk("hooks.json", await wikiScraper.getHooks()); 105 | } 106 | 107 | if (argv.skipPanels) { 108 | logger.info("Skipping panels (5 / 7)"); 109 | } else { 110 | logger.info("Retrieving panels (5 / 7)"); 111 | writeToDisk("panels.json", await wikiScraper.getPanels()); 112 | } 113 | 114 | if (argv.skipEnums) { 115 | logger.info("Skipping enums (6 / 7)"); 116 | } else { 117 | logger.info("Retrieving enums (6 / 7)"); 118 | writeToDisk("enums.json", await wikiScraper.getEnums()); 119 | } 120 | 121 | if (argv.skipStructs) { 122 | logger.info("Skipping structs (7 / 7)"); 123 | } else { 124 | logger.info("Retrieving structs (7 / 7)"); 125 | writeToDisk("structs.json", await wikiScraper.getStructs()); 126 | } 127 | })(); 128 | -------------------------------------------------------------------------------- /src/wiki-scraper.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeAll } from "vitest"; 2 | import { WikiScraper } from "./wiki-scraper"; 3 | import { WikiApiClient } from "./wiki-api-client"; 4 | import { WikiPage } from "./types"; 5 | 6 | describe("WikiScraper", () => { 7 | let wikiApiClient: WikiApiClient; 8 | let wikiScraper: WikiScraper; 9 | 10 | beforeAll(() => { 11 | wikiApiClient = new WikiApiClient(); 12 | wikiScraper = new WikiScraper(wikiApiClient); 13 | }); 14 | 15 | it("parses a function page", () => { 16 | const mathPiPageContent = 17 | '\r\n' + 18 | "\t\r\n" + 19 | "A variable containing the mathematical constant pi. (`3.1415926535898`)\r\n" + 20 | "\r\n" + 21 | "See also: Trigonometry\r\n" + 22 | "\r\n" + 23 | "It should be noted that due to the nature of floating point numbers, results of calculations with `math.pi` may not be what you expect. See second example below.\r\n" + 24 | "\r\n" + 25 | "\tShared and Menu\r\n" + 26 | "\t\r\n" + 27 | '\t\tThe mathematical constant, Pi.\r\n' + 28 | "\t\r\n" + 29 | "\r\n" + 30 | "\r\n" + 31 | "\r\n" + 32 | "\r\n" + 33 | "print( math.cos( math.pi ) )\r\n" + 34 | "\r\n" + 35 | "\r\n" + 36 | "```\r\n" + 37 | "-1\r\n" + 38 | "```\r\n" + 39 | "\r\n" + 40 | "\r\n" + 41 | "\r\n" + 42 | "\r\n" + 43 | "\r\n" + 44 | "\r\n" + 45 | "`sin(π) = 0`, but because floating point precision is not unlimited it cannot be calculated as exactly `0`.\r\n" + 46 | "\r\n" + 47 | "\r\n" + 48 | "print( math.sin( math.pi ), math.sin( math.pi ) == 0 )\r\n" + 49 | "\r\n" + 50 | "\r\n" + 51 | "```\r\n" + 52 | "1.2246467991474e-16 false\r\n" + 53 | "```\r\n" + 54 | "\r\n" + 55 | "\r\n"; 56 | 57 | const mathPiFunction = wikiScraper.parseFunctionPage(mathPiPageContent); 58 | 59 | expect(mathPiFunction.name).toBe("pi"); 60 | expect(mathPiFunction.parent).toBe("math"); 61 | expect(mathPiFunction.realms).toEqual( 62 | expect.arrayContaining(["client", "server", "menu"]), 63 | ); 64 | expect(mathPiFunction.returnValues).toEqual([ 65 | { type: "number", description: "The mathematical constant, Pi." }, 66 | ]); 67 | }); 68 | 69 | it("parses a panel page", () => { 70 | const dbuttonPageContent = 71 | "\r\n" + 72 | "\tDLabel\r\n" + 73 | "\tDButton_small.png\r\n" + 74 | "\tClient and Menu\r\n" + 75 | "\tlua/vgui/dbutton.lua\r\n" + 76 | "\t\r\n" + 77 | "A standard Derma button.\r\n" + 78 | "\r\n" + 79 | "By default, a DButton is 22px tall.\r\n" + 80 | "\t\r\n" + 81 | "\t\r\n" + 82 | "PANEL:Init\r\n" + 83 | "PANEL:Paint\r\n" + 84 | "PANEL:PerformLayout\r\n" + 85 | "PANEL:GenerateExample\r\n" + 86 | "\t\r\n" + 87 | "\r\n" + 88 | "\r\n" + 89 | "\r\n" + 90 | "\tThe DButton is exactly what you think it is - a button!\r\n" + 91 | "\t\r\n" + 92 | 'local frame = vgui.Create( "DFrame" )\r\n' + 93 | "frame:SetSize( 300, 250 )\r\n" + 94 | "frame:Center()\r\n" + 95 | "frame:MakePopup()\r\n" + 96 | "\r\n" + 97 | 'local DermaButton = vgui.Create( "DButton", frame ) // Create the button and parent it to the frame\r\n' + 98 | 'DermaButton:SetText( "Say hi" )\t\t\t\t\t// Set the text on the button\r\n' + 99 | "DermaButton:SetPos( 25, 50 )\t\t\t\t\t// Set the position on the frame\r\n" + 100 | "DermaButton:SetSize( 250, 30 )\t\t\t\t\t// Set the size\r\n" + 101 | "DermaButton.DoClick = function()\t\t\t\t// A custom function run when clicked ( note the . instead of : )\r\n" + 102 | '\tRunConsoleCommand( "say", "Hi" )\t\t\t// Run the console command "say hi" when you click it ( command, args )\r\n' + 103 | "end\r\n" + 104 | "\r\n" + 105 | "DermaButton.DoRightClick = function()\r\n" + 106 | '\tRunConsoleCommand( "say", "Hello World" )\r\n' + 107 | "end\r\n" + 108 | "\t\r\n" + 109 | "\r\n" + 110 | "\r\n" + 111 | ''; 112 | 113 | const dbuttonPanel = wikiScraper.parsePanelPage(dbuttonPageContent); 114 | 115 | expect(dbuttonPanel.parent).toBe("DLabel"); 116 | expect(dbuttonPanel.description).toContain("A standard Derma button."); 117 | }); 118 | 119 | it("parses a type page", () => { 120 | const entityPageContent = 121 | '\r\n' + 122 | '\tThis is a list of all available methods for all entities, which includes Player, Weapon, NPC and Vehicle.\r\n' + 123 | "\r\n" + 124 | 'For a list of possible members of Scripted Entities see Structures/ENT\r\n' + 125 | "\r\n" + 126 | ""; 127 | 128 | const entityType = wikiScraper.parseTypePage(entityPageContent); 129 | 130 | expect(entityType.name).toBe("Entity"); 131 | expect(entityType.description).toContain( 132 | "This is a list of all available methods for all entities", 133 | ); 134 | }); 135 | 136 | it("parses an enum page", () => { 137 | const useEnumPageContent = 138 | "\r\n" + 139 | "\tShared\r\n" + 140 | "\t\r\n" + 141 | "Enumerations used by Entity:SetUseType. Affects when ENTITY:Use is triggered.\r\n" + 142 | "\r\n" + 143 | "Not to be confused with Enums/USE used for ENTITY:Use and others.\r\n" + 144 | "\t\r\n" + 145 | "\t\r\n" + 146 | 'Fire a Enums/USE signal every tick as long as the player holds their use key and aims at the target.\r\n' + 147 | 'Fires a Enums/USE signal when starting to use an entity, and a Enums/USE signal when letting go.\r\n' + 148 | "\r\n" + 149 | "There is no guarantee to receive both ON and OFF signals. A signal will only be sent when pushing or letting go of the use key while actually aiming at the entity, so an ON signal might not be followed by an OFF signal if the player is aiming somewhere else when releasing the key, and similarly, an OFF signal may not be preceded by an ON signal if the player started aiming at the entity only after pressing the key.\r\n" + 150 | "\r\n" + 151 | "Therefore, this method of input is unreliable and should not be used.\r\n" + 152 | 'Like a wheel turning.\r\n' + 153 | 'Fire a Enums/USE signal only once when player presses their use key while aiming at the target.\r\n' + 154 | "\t\r\n" + 155 | "\r\n" + 156 | "\r\n" + 157 | "\r\n"; 158 | 159 | const useEnum = wikiScraper.parseEnumPage(useEnumPageContent); 160 | 161 | expect(useEnum.name).toBeUndefined(); 162 | expect(useEnum.fields).toEqual( 163 | expect.arrayContaining([ 164 | expect.objectContaining({ 165 | name: "CONTINUOUS_USE", 166 | value: 0, 167 | description: expect.stringContaining( 168 | 'Fire a Enums/USE signal every tick', 169 | ), 170 | }), 171 | expect.objectContaining({ 172 | name: "ONOFF_USE", 173 | value: 1, 174 | description: expect.stringContaining( 175 | 'Fires a Enums/USE signal when starting to use an entity', 176 | ), 177 | }), 178 | expect.objectContaining({ 179 | name: "DIRECTIONAL_USE", 180 | value: 2, 181 | description: "Like a wheel turning.", 182 | }), 183 | expect.objectContaining({ 184 | name: "SIMPLE_USE", 185 | value: 3, 186 | description: expect.stringContaining( 187 | 'Fire a Enums/USE signal only once', 188 | ), 189 | }), 190 | ]), 191 | ); 192 | expect(useEnum.realms).toEqual( 193 | expect.arrayContaining(["client", "server"]), 194 | ); 195 | expect(useEnum.description).toContain( 196 | "Enumerations used by Entity:SetUseType", 197 | ); 198 | }); 199 | 200 | it("parses a struct page", () => { 201 | const angPosStructPageContent = 202 | "\r\n" + 203 | "\tShared\r\n" + 204 | "\tTable used by various functions, such as Entity:GetAttachment.\r\n" + 205 | "\t\r\n" + 206 | 'Angle object\r\n' + 207 | 'Vector object\r\n' + 208 | 'The bone ID the attachment point is parented to.\r\n' + 209 | "\t\r\n" + 210 | "\r\n" + 211 | "\r\n" + 212 | "\r\n"; 213 | 214 | const angPosStruct = wikiScraper.parseStructPage(angPosStructPageContent); 215 | 216 | expect(angPosStruct.name).toBeUndefined(); 217 | expect(angPosStruct.fields).toEqual( 218 | expect.arrayContaining([ 219 | expect.objectContaining({ 220 | name: "Ang", 221 | type: "Angle", 222 | description: "Angle object", 223 | }), 224 | expect.objectContaining({ 225 | name: "Pos", 226 | type: "Vector", 227 | description: "Vector object", 228 | }), 229 | expect.objectContaining({ 230 | name: "Bone", 231 | type: "number", 232 | description: "The bone ID the attachment point is parented to.", 233 | }), 234 | ]), 235 | ); 236 | expect(angPosStruct.realms).toEqual( 237 | expect.arrayContaining(["client", "server"]), 238 | ); 239 | }); 240 | 241 | it("gets pages in a category", async () => { 242 | const wikiApiClientMock = { 243 | renderText: vi.fn().mockResolvedValue({ 244 | status: "ok", 245 | html: 246 | "\n", 257 | title: null, 258 | }), 259 | }; 260 | const wikiScraper = new WikiScraper(wikiApiClientMock as any); 261 | 262 | const enumPagePaths = await wikiScraper.getPagesInCategory("enum"); 263 | 264 | expect(wikiApiClientMock.renderText).toHaveBeenCalledOnce(); 265 | expect(enumPagePaths).toEqual( 266 | expect.arrayContaining([ 267 | "/gmod/Enums/_USE", 268 | "/gmod/Enums/ACT", 269 | "/gmod/Enums/AIMR", 270 | "/gmod/Enums/AMMO", 271 | "/gmod/Enums/ANALOG", 272 | "/gmod/Enums/BLEND", 273 | "/gmod/Enums/BLENDFUNC", 274 | "/gmod/Enums/BLOOD_COLOR", 275 | ]), 276 | ); 277 | }); 278 | 279 | it("builds the class for a wiki page", () => { 280 | const wikiPages: Array = [ 281 | { 282 | title: "Angle", 283 | content: 284 | "# Angle\r\n" + 285 | "\r\n" + 286 | '\r\n' + 287 | "\t\r\n" + 288 | "\r\n" + 289 | "\t\tList of all possible functions to manipulate angles.\r\n" + 290 | "\r\n" + 291 | "\t\tCreated by Global.Angle.\r\n" + 292 | "\r\n" + 293 | "\t\t| Type | Name | Description |\r\n" + 294 | "\t\t| ------------------- | ------------------------------------ | -------------------------------- |\r\n" + 295 | "\t\t| number | `p` or `pitch` or `x` or `1` | The pitch component of the angle. |\r\n" + 296 | "\t\t| number | `y` or `yaw` or `y` or `2` | The yaw component of the angle. |\r\n" + 297 | "\t\t| number | `r` or `roll` or `z` or `3` | The roll component of the angle. |\r\n" + 298 | "\r\n" + 299 | "\t\tMetamethod | Second Operand | Description\r\n" + 300 | "\t\t---------- | -------------- | -----------\r\n" + 301 | "\t\t`__add` | Angle | Returns new Angle with the result of addition.\r\n" + 302 | "\t\t`__div` | number | Returns new Angle with the result of division.\r\n" + 303 | "\t\t`__eq` | any | Compares 2 operands, if they both are Angle, compares each individual component.
Doesn't normalize the angles (360 is not equal to 0).\r\n" + 304 | "\t\t`__index` | number or string | Gets the component of the Angle. Returns a number.\r\n" + 305 | "\t\t`__mul` | number | Returns new Angle with the result of multiplication.\r\n" + 306 | "\t\t`__newindex` | number or string | Sets the component of the Angle. Accepts number and string.\r\n" + 307 | "\t\t`__sub` | Angle | Returns new Angle with the result of subtraction.\r\n" + 308 | "\t\t`__tostring` | | Returns `p y r`.\r\n" + 309 | "\t\t`__unm` | | Returns new Angle with the result of negation.\r\n" + 310 | "\r\n" + 311 | "\t\r\n" + 312 | "\r\n", 313 | // shortened for brevity 314 | }, 315 | { 316 | title: "Angle:Add", 317 | content: 318 | '\r\n' + 319 | "\tAdds the values of the argument angle to the orignal angle. \r\n" + 320 | "\r\n" + 321 | "This functions the same as angle1 + angle2 without creating a new angle object, skipping object construction and garbage collection.\r\n" + 322 | "\tShared and Menu\r\n" + 323 | "\t\r\n" + 324 | '\t\tThe angle to add.\r\n' + 325 | "\t\r\n" + 326 | "\r\n" + 327 | "\r\n", 328 | }, 329 | ]; 330 | 331 | const classes = wikiScraper.buildClasses(wikiPages); 332 | const angleClass = classes[0]; 333 | 334 | expect(classes.length).toBe(1); 335 | expect(angleClass.name).toBe("Angle"); 336 | expect(angleClass.description).toContain( 337 | "List of all possible functions to manipulate angles", 338 | ); 339 | expect(angleClass.functions?.length).toBe(1); 340 | expect(angleClass.functions?.[0].name).toBe("Add"); 341 | expect(angleClass.functions?.[0].parent).toBe("Angle"); 342 | expect(angleClass.functions?.[0].realms).toEqual( 343 | expect.arrayContaining(["client", "server", "menu"]), 344 | ); 345 | expect(angleClass.functions?.[0].arguments).toEqual([ 346 | { name: "angle", type: "Angle", description: "The angle to add." }, 347 | ]); 348 | expect(angleClass.functions?.[0].description).toContain( 349 | "Adds the values of the argument angle to the orignal angle.", 350 | ); 351 | }); 352 | }); 353 | -------------------------------------------------------------------------------- /src/wiki-scraper.ts: -------------------------------------------------------------------------------- 1 | import * as cheerio from "cheerio"; 2 | import { SingleBar, Presets } from "cli-progress"; 3 | import pLimit from "p-limit"; 4 | 5 | import { 6 | Function, 7 | FunctionSource, 8 | FunctionArgument, 9 | FunctionReturnValue, 10 | Realm, 11 | Class, 12 | Panel, 13 | WikiPage, 14 | Type, 15 | Enum, 16 | EnumField, 17 | StructField, 18 | Struct, 19 | } from "./types.js"; 20 | import { WikiApiClient } from "./wiki-api-client.js"; 21 | import logger from "./logger.js"; 22 | import { isTagElement } from "./cheerio.js"; 23 | 24 | export class WikiScraper { 25 | private static readonly limit = pLimit(8); 26 | private readonly progressBar = new SingleBar({}, Presets.shades_classic); 27 | 28 | constructor(private wikiApiClient: WikiApiClient) {} 29 | 30 | public async getGlobalFunctions(): Promise> { 31 | const globalFunctionPageUrls = await this.getPagesInCategory("Global"); 32 | this.progressBar.start(globalFunctionPageUrls.length, 0); 33 | 34 | const globalFunctionPages = await Promise.all( 35 | globalFunctionPageUrls.map(async (pageUrl) => { 36 | const page = await WikiScraper.limit(() => 37 | this.wikiApiClient.retrievePage(pageUrl), 38 | ); 39 | this.progressBar.increment(); 40 | 41 | return page; 42 | }), 43 | ); 44 | 45 | this.progressBar.stop(); 46 | 47 | const globalFunctions: Array = []; 48 | 49 | globalFunctionPages.forEach((globalFunctionPage) => { 50 | if (this.isFunctionPage(globalFunctionPage.content)) { 51 | const globalFunction = this.parseFunctionPage( 52 | globalFunctionPage.content, 53 | ); 54 | globalFunctions.push(globalFunction); 55 | } else { 56 | logger.warn( 57 | `Unknown page type encountered on page '${globalFunctionPage.title}'`, 58 | ); 59 | } 60 | }); 61 | 62 | return globalFunctions; 63 | } 64 | 65 | public async getClasses(): Promise> { 66 | const classPageUrls = await this.getPagesInCategory("classfunc"); 67 | this.progressBar.start(classPageUrls.length, 0); 68 | 69 | const classPages = await Promise.all( 70 | classPageUrls.map(async (pageUrl) => { 71 | const page = await WikiScraper.limit(() => 72 | this.wikiApiClient.retrievePage(pageUrl), 73 | ); 74 | this.progressBar.increment(); 75 | 76 | return page; 77 | }), 78 | ); 79 | 80 | this.progressBar.stop(); 81 | 82 | return this.buildClasses(classPages); 83 | } 84 | 85 | public async getLibraries(): Promise> { 86 | const libraryPageUrls = await this.getPagesInCategory("libraryfunc"); 87 | this.progressBar.start(libraryPageUrls.length, 0); 88 | 89 | const libraryPages = await Promise.all( 90 | libraryPageUrls.map(async (pageUrl) => { 91 | const page = await WikiScraper.limit(() => 92 | this.wikiApiClient.retrievePage(pageUrl), 93 | ); 94 | this.progressBar.increment(); 95 | 96 | return page; 97 | }), 98 | ); 99 | 100 | this.progressBar.stop(); 101 | 102 | return this.buildClasses(libraryPages); 103 | } 104 | 105 | public async getHooks(): Promise> { 106 | const hookPageUrls = await this.getPagesInCategory("hook", ".*:"); 107 | this.progressBar.start(hookPageUrls.length, 0); 108 | 109 | const hookPages = await Promise.all( 110 | hookPageUrls.map(async (pageUrl) => { 111 | const page = await WikiScraper.limit(() => 112 | this.wikiApiClient.retrievePage(pageUrl), 113 | ); 114 | this.progressBar.increment(); 115 | 116 | return page; 117 | }), 118 | ); 119 | 120 | this.progressBar.stop(); 121 | 122 | return this.buildClasses(hookPages); 123 | } 124 | 125 | public async getPanels(): Promise> { 126 | const panelPageUrls = await this.getPagesInCategory("panelfunc"); 127 | this.progressBar.start(panelPageUrls.length, 0); 128 | 129 | const panelPages = await Promise.all( 130 | panelPageUrls.map(async (pageUrl) => { 131 | const page = await WikiScraper.limit(() => 132 | this.wikiApiClient.retrievePage(pageUrl), 133 | ); 134 | this.progressBar.increment(); 135 | 136 | return page; 137 | }), 138 | ); 139 | 140 | this.progressBar.stop(); 141 | 142 | return this.buildClasses(panelPages); 143 | } 144 | 145 | public async getEnums(): Promise> { 146 | const enumPageUrls = await this.getPagesInCategory("enum"); 147 | this.progressBar.start(enumPageUrls.length, 0); 148 | 149 | const enumPages = await Promise.all( 150 | enumPageUrls.map(async (pageUrl) => { 151 | const page = await WikiScraper.limit(() => 152 | this.wikiApiClient.retrievePage(pageUrl), 153 | ); 154 | this.progressBar.increment(); 155 | 156 | return page; 157 | }), 158 | ); 159 | 160 | this.progressBar.stop(); 161 | 162 | const enums: Array = []; 163 | 164 | enumPages.forEach((enumPage) => { 165 | if (this.isEnumPage(enumPage.content)) { 166 | const _enum = this.parseEnumPage(enumPage.content); 167 | _enum.name = enumPage.title; 168 | 169 | enums.push(_enum); 170 | } else { 171 | logger.warn( 172 | `Unknown page type encountered on page '${enumPage.title}'`, 173 | ); 174 | } 175 | }); 176 | 177 | return enums; 178 | } 179 | 180 | public async getStructs(): Promise> { 181 | const structPageUrls = await this.getPagesInCategory("struct"); 182 | this.progressBar.start(structPageUrls.length, 0); 183 | 184 | const structPages = await Promise.all( 185 | structPageUrls.map(async (pageUrl) => { 186 | const page = await WikiScraper.limit(() => 187 | this.wikiApiClient.retrievePage(pageUrl), 188 | ); 189 | this.progressBar.increment(); 190 | 191 | return page; 192 | }), 193 | ); 194 | 195 | this.progressBar.stop(); 196 | 197 | const structs: Array = []; 198 | 199 | structPages.forEach((structPage) => { 200 | if (this.isStructPage(structPage.content)) { 201 | const struct = this.parseStructPage(structPage.content); 202 | struct.name = structPage.title; 203 | 204 | structs.push(struct); 205 | } else { 206 | logger.warn( 207 | `Unknown page type encountered on page '${structPage.title}'`, 208 | ); 209 | } 210 | }); 211 | 212 | return structs; 213 | } 214 | 215 | public async getPagesInCategory( 216 | category: string, 217 | filter = "", 218 | ): Promise> { 219 | const response = await this.wikiApiClient.renderText( 220 | ``, 221 | ); 222 | 223 | if (!response.html || response.html === "") { 224 | throw new Error( 225 | `Could not get pages in category '${category}' with filter '${filter}'`, 226 | ); 227 | } 228 | 229 | const pageUrls: Array = []; 230 | const $ = this.parseContent(response.html); 231 | 232 | $("ul > li > a").each((i, element) => { 233 | if (element.type !== "tag") { 234 | return; 235 | } 236 | pageUrls.push(element.attribs.href); 237 | }); 238 | 239 | return pageUrls; 240 | } 241 | 242 | public buildClasses(wikiPages: Array): Array { 243 | const classes = new Map(); 244 | 245 | wikiPages.forEach((wikiPage) => { 246 | // Examples: 247 | // ContentIcon 248 | // ContentIcon:GetColor 249 | // achievements 250 | // achievements.BalloonPopped 251 | const className = wikiPage.title.includes(":") 252 | ? wikiPage.title.split(":")[0] 253 | : wikiPage.title.includes(".") 254 | ? wikiPage.title.split(".")[0] 255 | : wikiPage.title; 256 | 257 | const _class: Class = classes.get(className) ?? { name: className }; 258 | 259 | if (this.isPanelPage(wikiPage.content)) { 260 | const panel = this.parsePanelPage(wikiPage.content); 261 | _class.parent = panel.parent; 262 | 263 | if (panel.description) { 264 | _class.description = panel.description; 265 | } 266 | } else if (this.isTypePage(wikiPage.content)) { 267 | const type = this.parseTypePage(wikiPage.content); 268 | 269 | if (type.description) { 270 | _class.description = type.description; 271 | } 272 | } else if (this.isFunctionPage(wikiPage.content)) { 273 | const _function = this.parseFunctionPage(wikiPage.content); 274 | 275 | _class.functions = _class.functions ?? []; 276 | _class.functions.push(_function); 277 | } else { 278 | logger.warn( 279 | `Unknown page type encountered on page '${wikiPage.title}'`, 280 | ); 281 | } 282 | 283 | classes.set(className, _class); 284 | }); 285 | 286 | return Array.from(classes.values()); 287 | } 288 | 289 | public parseFunctionPage(pageContent: string): Function { 290 | const $ = this.parseContent(pageContent); 291 | const name = $("function").attr().name; 292 | const parent = $("function").attr().parent; 293 | const description = $("function > description").html(); 294 | const $sourceFile = $("function > file"); 295 | const realmsRaw = this.trimMultiLineString($("function > realm").text()); 296 | const realms = this.parseRealms(realmsRaw); 297 | const args: Array = []; 298 | const overloadedArgs: Array> = []; 299 | const returnValues: Array = []; 300 | 301 | const [argsElement, ...overloadArgsElements] = $("function > args"); 302 | 303 | if (argsElement != null) { 304 | args.push(...this.parseFunctionArguments(argsElement)); 305 | } 306 | 307 | if (overloadArgsElements.length > 0) { 308 | for (const overloadArgsElement of overloadArgsElements) { 309 | overloadedArgs.push(this.parseFunctionArguments(overloadArgsElement)); 310 | } 311 | } 312 | 313 | $("function > rets") 314 | .children() 315 | .each((i, element) => { 316 | if (element.type !== "tag") { 317 | return; 318 | } 319 | 320 | const description = $(element).html(); 321 | const name = element.attribs.name; 322 | 323 | const returnValue: FunctionReturnValue = { 324 | type: element.attribs.type, 325 | }; 326 | 327 | if (name && name !== "") { 328 | returnValue.name = name; 329 | } 330 | 331 | if (description && description !== "") { 332 | returnValue.description = this.trimMultiLineString(description); 333 | } 334 | 335 | returnValues.push(returnValue); 336 | }); 337 | 338 | const _function: Function = { 339 | name: name, 340 | parent: parent, 341 | realms: realms, 342 | }; 343 | 344 | if (description && description !== "") { 345 | _function.description = this.trimMultiLineString(description); 346 | } 347 | 348 | if (args.length > 0) { 349 | _function.arguments = args; 350 | } 351 | 352 | if (returnValues.length > 0) { 353 | _function.returnValues = returnValues; 354 | } 355 | 356 | if (overloadedArgs.length > 0) { 357 | _function.overloads = overloadedArgs.map((args) => { 358 | return { 359 | arguments: args, 360 | returnValues: returnValues.length > 0 ? returnValues : undefined, 361 | }; 362 | }); 363 | } 364 | 365 | if ($sourceFile.length > 0) { 366 | const file = $sourceFile.text(); 367 | 368 | const line = $sourceFile.attr().line.replace("L", ""); 369 | const lines = line.split("-"); 370 | const lineStart = lines[0]; 371 | const lineEnd = lines[1]; 372 | 373 | const source: FunctionSource = { 374 | file: file, 375 | lineStart: Number(lineStart), 376 | }; 377 | 378 | if (lineEnd) { 379 | source.lineEnd = Number(lineEnd); 380 | } 381 | 382 | _function.source = source; 383 | } 384 | 385 | return _function; 386 | } 387 | 388 | public parsePanelPage(pageContent: string): Panel { 389 | const $ = this.parseContent(pageContent); 390 | const parent = this.trimMultiLineString($("panel > parent").text()); 391 | const description = $("panel > description").html(); 392 | 393 | const panel: Panel = { 394 | parent: parent, 395 | }; 396 | 397 | if (description && description !== "") { 398 | panel.description = this.trimMultiLineString(description); 399 | } 400 | 401 | return panel; 402 | } 403 | 404 | public parseTypePage(pageContent: string): Type { 405 | const $ = this.parseContent(pageContent); 406 | const name = $("type").attr().name; 407 | const description = $("type > summary").html(); 408 | 409 | const type: Type = { 410 | name: name, 411 | }; 412 | 413 | if (description && description !== "") { 414 | type.description = this.trimMultiLineString(description); 415 | } 416 | 417 | return type; 418 | } 419 | 420 | public parseEnumPage(pageContent: string): Enum { 421 | const $ = this.parseContent(pageContent); 422 | const realmsRaw = this.trimMultiLineString($("enum > realm").text()); 423 | const realms = this.parseRealms(realmsRaw); 424 | const description = $("enum > description").html(); 425 | const enumFields: Array = []; 426 | 427 | $("enum > items") 428 | .children() 429 | .each((i, element) => { 430 | if (element.type !== "tag") { 431 | return; 432 | } 433 | 434 | const name = element.attribs.key; 435 | const value = element.attribs.value; 436 | const description = $(element).html(); 437 | 438 | const enumField: EnumField = { 439 | name: name, 440 | value: Number(value), 441 | }; 442 | 443 | if (description && description !== "") { 444 | enumField.description = this.trimMultiLineString(description); 445 | } 446 | 447 | enumFields.push(enumField); 448 | }); 449 | 450 | const _enum: Enum = { 451 | fields: enumFields, 452 | realms: realms, 453 | }; 454 | 455 | if (description && description !== "") { 456 | _enum.description = this.trimMultiLineString(description); 457 | } 458 | 459 | return _enum; 460 | } 461 | 462 | public parseStructPage(pageContent: string): Struct { 463 | const $ = this.parseContent(pageContent); 464 | const realmsRaw = this.trimMultiLineString($("structure > realm").text()); 465 | const realms = this.parseRealms(realmsRaw); 466 | const description = $("structure > description").html(); 467 | const structFields: Array = []; 468 | 469 | $("structure > fields") 470 | .children() 471 | .each((i, element) => { 472 | if (element.type !== "tag") { 473 | return; 474 | } 475 | 476 | const name = element.attribs.name; 477 | const type = element.attribs.type; 478 | const description = $(element).html(); 479 | 480 | const structField: StructField = { 481 | name: name, 482 | type: type, 483 | }; 484 | 485 | if (element.attribs.default) { 486 | structField.default = element.attribs.default; 487 | } 488 | 489 | if (description && description !== "") { 490 | structField.description = this.trimMultiLineString(description); 491 | } 492 | 493 | structFields.push(structField); 494 | }); 495 | 496 | const struct: Struct = { 497 | fields: structFields, 498 | realms: realms, 499 | }; 500 | 501 | if (description && description !== "") { 502 | struct.description = this.trimMultiLineString(description); 503 | } 504 | 505 | return struct; 506 | } 507 | 508 | public parseRealms(realmsRaw: string): Array { 509 | const realms = new Set(); 510 | const realmsRawLower = realmsRaw.toLowerCase(); 511 | 512 | if (realmsRawLower.includes("client")) { 513 | realms.add("client"); 514 | } 515 | 516 | if (realmsRawLower.includes("menu")) { 517 | realms.add("menu"); 518 | } 519 | 520 | if (realmsRawLower.includes("server")) { 521 | realms.add("server"); 522 | } 523 | 524 | if (realmsRawLower.includes("shared")) { 525 | realms.add("client"); 526 | realms.add("server"); 527 | } 528 | 529 | return Array.from(realms); 530 | } 531 | 532 | public isPanelPage(pageContent: string): boolean { 533 | const $ = this.parseContent(pageContent); 534 | 535 | return $("panel").length > 0; 536 | } 537 | 538 | public isFunctionPage(pageContent: string): boolean { 539 | const $ = this.parseContent(pageContent); 540 | 541 | return $("function").length > 0; 542 | } 543 | 544 | public isTypePage(pageContent: string): boolean { 545 | const $ = this.parseContent(pageContent); 546 | 547 | return $("type").length > 0; 548 | } 549 | 550 | public isEnumPage(pageContent: string): boolean { 551 | const $ = this.parseContent(pageContent); 552 | 553 | return $("enum").length > 0; 554 | } 555 | 556 | public isStructPage(pageContent: string): boolean { 557 | const $ = this.parseContent(pageContent); 558 | 559 | return $("structure").length > 0; 560 | } 561 | 562 | private parseFunctionArguments(argsElement: cheerio.Element) { 563 | if (!isTagElement(argsElement)) { 564 | throw new Error(`Expected a tag element, got ${argsElement.type}`); 565 | } 566 | 567 | return argsElement.children 568 | .filter(isTagElement) 569 | .map((element): FunctionArgument => { 570 | const name = element.attribs.name; 571 | const type = element.attribs.type; 572 | const defaultValue = element.attribs.default; 573 | 574 | let description = cheerio 575 | .load(element.children, { decodeEntities: false }) 576 | .html(); 577 | if (description != null && description !== "") { 578 | description = this.trimMultiLineString(description); 579 | } 580 | 581 | return { 582 | name: name, 583 | type: type, 584 | default: defaultValue, 585 | description: description, 586 | }; 587 | }); 588 | } 589 | 590 | private parseContent(content: string) { 591 | return cheerio.load(content, { decodeEntities: false }); 592 | } 593 | 594 | private trimMultiLineString(str: string) { 595 | return str 596 | .split("\n") 597 | .map((line) => line.trim()) 598 | .join("\n") 599 | .trim(); 600 | } 601 | } 602 | --------------------------------------------------------------------------------