├── .npmignore ├── prepublish.sh ├── postbuild.sh ├── src ├── ast │ ├── index.ts │ ├── ast.ts │ ├── visit.ts │ ├── ast.test.ts │ └── ast-types.ts ├── parser │ ├── glsltest.glsl │ ├── index.ts │ ├── parser.d.ts │ ├── utils.ts │ ├── test-helpers.ts │ ├── scope.ts │ ├── generator.ts │ ├── parse.test.ts │ ├── scope.test.ts │ └── grammar.ts ├── preprocessor │ ├── preprocess-test-grammar.glsl │ ├── preprocessor-parser.d.ts │ ├── index.ts │ ├── generator.ts │ ├── preprocessor-node.ts │ ├── preprocessor-grammar.pegjs │ ├── preprocessor.test.ts │ └── preprocessor.ts ├── index.ts └── error.ts ├── .gitignore ├── babel.config.cjs ├── .prettierrc ├── jest.config.js ├── .github └── workflows │ └── main.yml ├── tsconfig.json ├── package.json └── README.md /.npmignore: -------------------------------------------------------------------------------- 1 | *.test.ts 2 | *.test.js 3 | -------------------------------------------------------------------------------- /prepublish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cp -r dist/* . 5 | -------------------------------------------------------------------------------- /postbuild.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | rm -rf ast index.d.ts error.js index.js parser preprocessor 5 | -------------------------------------------------------------------------------- /src/ast/index.ts: -------------------------------------------------------------------------------- 1 | export * from './ast.js'; 2 | export * from './visit.js'; 3 | export * from './ast-types.js'; 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | .vscode 4 | .DS_Store 5 | tmp 6 | src/parser/parser.js 7 | src/preprocessor/preprocessor-parser.js 8 | tsconfig.tsbuildinfo 9 | -------------------------------------------------------------------------------- /babel.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | ['@babel/preset-env', { targets: { node: 'current' } }], 4 | '@babel/preset-typescript', 5 | ], 6 | }; 7 | -------------------------------------------------------------------------------- /src/parser/glsltest.glsl: -------------------------------------------------------------------------------- 1 | #version 300 es 2 | out highp vec4 pc_fragColor; 3 | precision highp float; 4 | precision highp int; 5 | 6 | highp float rand( const in vec2 uv ) { 7 | } -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "overrides": [ 3 | { 4 | "files": ["*.mjs", "*.js", ".prettierrc", "*.json"] 5 | } 6 | ], 7 | "trailingComma": "es5", 8 | "singleQuote": true 9 | } 10 | -------------------------------------------------------------------------------- /src/preprocessor/preprocess-test-grammar.glsl: -------------------------------------------------------------------------------- 1 | #ifdef HIGH_PRECISION 2 | float precisionSafeLength( vec3 v ) { return length( v ); } 3 | #else 4 | float precisionSafeLength( vec3 v ) { 5 | float maxComponent = max3( abs( v ) ); 6 | return length( v / maxComponent ) * maxComponent; 7 | } 8 | #endif 9 | hi -------------------------------------------------------------------------------- /src/preprocessor/preprocessor-parser.d.ts: -------------------------------------------------------------------------------- 1 | import type { PreprocessorProgram, PreprocessorOptions } from './preprocessor'; 2 | import { SyntaxError } from '../error.ts'; 3 | 4 | export function parse( 5 | input: string, 6 | options?: PreprocessorOptions 7 | ): PreprocessorProgram; 8 | 9 | export const SyntaxError: typeof SyntaxError; 10 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import generate from './parser/generator.js'; 2 | import * as parser from './parser/parser.js'; 3 | import parse from './parser/index.js'; 4 | 5 | // I tried "export * from './error.js'" here but it doesn't seem to make 6 | // GlslSyntaxError available to consumers of the module? 7 | export { GlslSyntaxError } from './error.js'; 8 | 9 | export { generate, parser, parse }; 10 | -------------------------------------------------------------------------------- /src/parser/index.ts: -------------------------------------------------------------------------------- 1 | import { formatError } from '../error.js'; 2 | import * as parser from './parser.js'; 3 | 4 | /** 5 | * This is the main entry point for the parser. It parses the source 6 | * code and returns an AST. It protects the user from the horrific peggy 7 | * SyntaxError, by wrapping it in a nicer custom error. 8 | */ 9 | const parse = formatError(parser); 10 | 11 | export default parse; 12 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | moduleFileExtensions: ['ts', 'tsx', 'js', 'json', 'pegjs', 'glsl'], 3 | modulePathIgnorePatterns: ['src/parser/parser.js'], 4 | testPathIgnorePatterns: ['dist', 'src/parser/parser.js'], 5 | preset: 'ts-jest', 6 | resolver: 'ts-jest-resolver', 7 | // ts-jest is horrifically slow https://github.com/kulshekhar/ts-jest/issues/259#issuecomment-1332269911 8 | transform: { 9 | '^.+\\.(t|j)sx?$': '@swc/jest', 10 | }, 11 | // In CI, it builds the test files into JS files, which we don't want to run 12 | testMatch: ['**/*.test.ts'], 13 | }; 14 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | 12 | name: Run tests 13 | runs-on: ubuntu-latest 14 | 15 | strategy: 16 | matrix: 17 | node-version: [18.x] 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Use Node.js ${{ matrix.node-version }} 22 | uses: actions/setup-node@v3 23 | with: 24 | node-version: ${{ matrix.node-version }} 25 | - run: npm ci 26 | - name: Run tests 27 | run: npm test 28 | - name: Typecheck 29 | run: npx tsc --noEmit 30 | -------------------------------------------------------------------------------- /src/preprocessor/index.ts: -------------------------------------------------------------------------------- 1 | import generate from './generator.js'; 2 | import { 3 | preprocessAst, 4 | preprocessComments, 5 | PreprocessorOptions, 6 | visitPreprocessedAst, 7 | } from './preprocessor.js'; 8 | import { formatError } from '../error.js'; 9 | 10 | // This index file is currently only for package publishing, where the whole 11 | // library exists in the dist/ folder, so the below import is relative to dist/ 12 | import * as parser from './preprocessor-parser.js'; 13 | 14 | /** 15 | * This is the main entry point for the preprocessor. It parses the source 16 | * code and returns an AST. It protects the user from the horrific peggy 17 | * SyntaxError, by wrapping it in a nicer custom error. 18 | */ 19 | const parse = (src: string, options?: PreprocessorOptions) => 20 | formatError(parser)( 21 | options?.preserveComments ? src : preprocessComments(src), 22 | options 23 | ); 24 | 25 | const preprocess = (src: string, options?: PreprocessorOptions) => 26 | generate(preprocessAst(parse(src, options), options)); 27 | 28 | export default preprocess; 29 | 30 | export { 31 | parse, 32 | preprocessAst, 33 | preprocessComments, 34 | generate, 35 | preprocess, 36 | parser, 37 | visitPreprocessedAst, 38 | }; 39 | -------------------------------------------------------------------------------- /src/parser/parser.d.ts: -------------------------------------------------------------------------------- 1 | import type { AstNode, Program } from '../ast'; 2 | import { SyntaxError } from '../error.ts'; 3 | 4 | export type ParserOptions = Partial<{ 5 | quiet: boolean; 6 | grammarSource: string; 7 | includeLocation: boolean; 8 | failOnWarn: boolean; 9 | stage: 'vertex' | 'fragment' | 'either'; 10 | tracer: { 11 | trace: (e: { 12 | type: 'rule.enter' | 'rule.match' | 'rule.fail'; 13 | rule: string; 14 | result: any; 15 | }) => void; 16 | }; 17 | }>; 18 | 19 | // Allow to fetch util functions from parser directly. I'd like to inline those 20 | // functions directly in this file, but then the tests can't find it since jest 21 | // can't import from .d.ts files as there's no accompanying ts/js file 22 | export { 23 | renameBinding, 24 | renameBindings, 25 | renameType, 26 | renameTypes, 27 | renameFunction, 28 | renameFunctions, 29 | debugEntry, 30 | debugFunctionEntry, 31 | debugScopes, 32 | } from './utils'; 33 | 34 | export type Parse = { 35 | (input: string, options?: ParserOptions): Program; 36 | }; 37 | 38 | // Convenience export to cast the parser in tests 39 | export type Parser = { 40 | parse: Parse; 41 | }; 42 | 43 | export const parse: Parse; 44 | 45 | export const SyntaxError: typeof SyntaxError; 46 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "incremental": true, 4 | "target": "es5", 5 | 6 | // This is for VSCode. Without this line, VSCode's Typescript server 7 | // includes *DOM* types in typechecks, and complains that location() is 8 | // window.location, when in fact it's peggy's location() function. 9 | "lib": ["ESNext"], 10 | 11 | // Create ESM modules 12 | "module": "NodeNext", 13 | // Specify multiple folders that act like `./node_modules/@types` 14 | "typeRoots": [ 15 | "node_modules/@types", 16 | "./preprocessor", 17 | "./parser" 18 | ], 19 | 20 | "moduleResolution": "NodeNext", 21 | 22 | // Generate .d.ts files from TypeScript and JavaScript files in your project 23 | "declaration": true, 24 | // Specify an output folder for all emitted files. 25 | "outDir": "./dist", 26 | // Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */ 27 | "esModuleInterop": true, 28 | // Ensure that casing is correct in imports 29 | "forceConsistentCasingInFileNames": true, 30 | // Enable all strict type-checking options. 31 | "strict": true, 32 | // Skip type checking all .d.ts files 33 | "skipLibCheck": true 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ast/ast.ts: -------------------------------------------------------------------------------- 1 | import type { AstNode, Program } from './ast-types.js'; 2 | 3 | type NodeGenerator = (node: NodeType) => string; 4 | 5 | export type NodeGenerators = { 6 | [NodeType in AstNode['type']]: NodeGenerator< 7 | Extract 8 | >; 9 | } & { program?: NodeGenerator }; 10 | 11 | export type Generator = ( 12 | ast: Program | AstNode | AstNode[] | string | string[] | undefined | null 13 | ) => string; 14 | 15 | /** 16 | * Stringify an AST 17 | */ 18 | export const makeGenerator = (generators: NodeGenerators): Generator => { 19 | const gen = ( 20 | ast: Program | AstNode | AstNode[] | string | string[] | undefined | null 21 | ): string => 22 | typeof ast === 'string' 23 | ? ast 24 | : ast === null || ast === undefined 25 | ? '' 26 | : Array.isArray(ast) 27 | ? ast.map(gen).join('') 28 | : ast.type in generators 29 | ? (generators[ast.type] as Generator)(ast) 30 | : `NO GENERATOR FOR ${ast.type}` + ast; 31 | return gen; 32 | }; 33 | 34 | export type EveryOtherGenerator = (nodes: AstNode[], eo: AstNode[]) => string; 35 | 36 | export const makeEveryOtherGenerator = ( 37 | generate: Generator 38 | ): EveryOtherGenerator => { 39 | const everyOther = (nodes: AstNode[], eo: AstNode[]) => 40 | nodes.reduce( 41 | (output, node, index) => 42 | output + 43 | generate(node) + 44 | (index === nodes.length - 1 ? '' : generate(eo[index])), 45 | '' 46 | ); 47 | return everyOther; 48 | }; 49 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@shaderfrog/glsl-parser", 3 | "engines": { 4 | "node": ">=16" 5 | }, 6 | "version": "6.1.0", 7 | "type": "module", 8 | "description": "A GLSL ES 1.0 and 3.0 parser and preprocessor that can preserve whitespace and comments", 9 | "scripts": { 10 | "prepare": "npm run build && ./prepublish.sh", 11 | "postpublish": "./postbuild.sh", 12 | "build": "./build.sh", 13 | "watch-test": "jest --watch", 14 | "test": "jest --colors" 15 | }, 16 | "files": [ 17 | "ast", 18 | "index.d.ts", 19 | "index.js", 20 | "error.js", 21 | "parser", 22 | "preprocessor" 23 | ], 24 | "repository": { 25 | "type": "git", 26 | "url": "git+https://github.com/ShaderFrog/glsl-parser.git" 27 | }, 28 | "keywords": [ 29 | "glsl", 30 | "parser", 31 | "shaderfrog" 32 | ], 33 | "author": "Andrew Ray", 34 | "license": "ISC", 35 | "bugs": { 36 | "url": "https://github.com/ShaderFrog/glsl-parser/issues" 37 | }, 38 | "homepage": "https://github.com/ShaderFrog/glsl-parser#readme", 39 | "devDependencies": { 40 | "@babel/core": "^7.15.5", 41 | "@babel/preset-env": "^7.15.6", 42 | "@babel/preset-typescript": "^7.15.0", 43 | "@swc/core": "^1.4.11", 44 | "@swc/jest": "^0.2.36", 45 | "@types/jest": "^27.0.2", 46 | "@types/node": "^16.10.2", 47 | "jest": "^29.7.0", 48 | "peggy": "^1.2.0", 49 | "prettier": "^2.1.2", 50 | "ts-jest": "^29.1.2", 51 | "ts-jest-resolver": "^2.0.1", 52 | "typescript": "^5.5.3" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/preprocessor/generator.ts: -------------------------------------------------------------------------------- 1 | import { makeGenerator, NodeGenerators } from '../ast/index.js'; 2 | import { PreprocessorProgram } from './preprocessor.js'; 3 | import { PreprocessorAstNode } from './preprocessor-node.js'; 4 | 5 | type NodeGenerator = (node: NodeType) => string; 6 | 7 | type NodePreprocessorGenerators = { 8 | [NodeType in PreprocessorAstNode['type']]: NodeGenerator< 9 | Extract 10 | >; 11 | } & { program?: NodeGenerator }; 12 | 13 | type Generator = ( 14 | ast: 15 | | PreprocessorProgram 16 | | PreprocessorAstNode 17 | | PreprocessorAstNode[] 18 | | string 19 | | string[] 20 | | undefined 21 | | null 22 | ) => string; 23 | 24 | /** 25 | * Stringify an AST 26 | */ 27 | // @ts-ignore 28 | const makeGeneratorPreprocessor = makeGenerator as ( 29 | generators: NodePreprocessorGenerators 30 | ) => Generator; 31 | 32 | const generators: NodePreprocessorGenerators = { 33 | program: (node) => generate(node.program) + generate(node.wsEnd), 34 | segment: (node) => generate(node.blocks), 35 | text: (node) => generate(node.text), 36 | literal: (node) => 37 | generate(node.wsStart) + generate(node.literal) + generate(node.wsEnd), 38 | identifier: (node) => generate(node.identifier) + generate(node.wsEnd), 39 | 40 | binary: (node) => 41 | generate(node.left) + generate(node.operator) + generate(node.right), 42 | group: (node) => 43 | generate(node.lp) + generate(node.expression) + generate(node.rp), 44 | unary: (node) => generate(node.operator) + generate(node.expression), 45 | unary_defined: (node) => 46 | generate(node.operator) + 47 | generate(node.lp) + 48 | generate(node.identifier) + 49 | generate(node.rp), 50 | int_constant: (node) => generate(node.token) + generate(node.wsEnd), 51 | 52 | elseif: (node) => 53 | generate(node.token) + 54 | generate(node.expression) + 55 | generate(node.wsEnd) + 56 | generate(node.body), 57 | if: (node) => 58 | generate(node.token) + 59 | generate(node.expression) + 60 | generate(node.wsEnd) + 61 | generate(node.body), 62 | ifdef: (node) => 63 | generate(node.token) + 64 | generate(node.identifier) + 65 | generate(node.wsEnd) + 66 | generate(node.body), 67 | ifndef: (node) => 68 | generate(node.token) + 69 | generate(node.identifier) + 70 | generate(node.wsEnd) + 71 | generate(node.body), 72 | else: (node) => 73 | generate(node.token) + generate(node.wsEnd) + generate(node.body), 74 | error: (node) => 75 | generate(node.error) + generate(node.message) + generate(node.wsEnd), 76 | 77 | undef: (node) => 78 | generate(node.undef) + generate(node.identifier) + generate(node.wsEnd), 79 | define: (node) => 80 | generate(node.wsStart) + 81 | generate(node.define) + 82 | generate(node.identifier) + 83 | generate(node.body) + 84 | generate(node.wsEnd), 85 | define_arguments: (node) => 86 | generate(node.wsStart) + 87 | generate(node.define) + 88 | generate(node.identifier) + 89 | generate(node.lp) + 90 | generate(node.args) + 91 | generate(node.rp) + 92 | generate(node.body) + 93 | generate(node.wsEnd), 94 | 95 | conditional: (node) => 96 | generate(node.wsStart) + 97 | generate(node.ifPart) + 98 | // generate(node.body) + 99 | generate(node.elseIfParts) + 100 | generate(node.elsePart) + 101 | generate(node.endif) + 102 | generate(node.wsEnd), 103 | 104 | version: (node) => 105 | generate(node.version) + 106 | generate(node.value) + 107 | generate(node.profile) + 108 | generate(node.wsEnd), 109 | pragma: (node) => 110 | generate(node.pragma) + generate(node.body) + generate(node.wsEnd), 111 | line: (node) => 112 | generate(node.line) + generate(node.value) + generate(node.wsEnd), 113 | extension: (node) => 114 | generate(node.extension) + 115 | generate(node.name) + 116 | generate(node.colon) + 117 | generate(node.behavior) + 118 | generate(node.wsEnd), 119 | }; 120 | 121 | const generate = makeGeneratorPreprocessor(generators); 122 | 123 | export default generate; 124 | -------------------------------------------------------------------------------- /src/ast/visit.ts: -------------------------------------------------------------------------------- 1 | import type { AstNode, Program } from './ast-types.js'; 2 | 3 | const isNode = (node: AstNode) => !!node?.type; 4 | const isTraversable = (node: any) => isNode(node) || Array.isArray(node); 5 | 6 | export type Path = { 7 | node: NodeType; 8 | parent: Program | AstNode | undefined; 9 | parentPath: Path | undefined; 10 | key: string | undefined; 11 | index: number | undefined; 12 | stop: () => void; 13 | skip: () => void; 14 | remove: () => void; 15 | replaceWith: (replacer: AstNode) => void; 16 | findParent: (test: (p: Path) => boolean) => Path | undefined; 17 | 18 | stopped?: boolean; 19 | skipped?: boolean; 20 | removed?: boolean; 21 | replaced?: any; 22 | }; 23 | 24 | const makePath = ( 25 | node: NodeType, 26 | parent: AstNode | Program | undefined, 27 | parentPath: Path | undefined, 28 | key: string | undefined, 29 | index: number | undefined 30 | ): Path => ({ 31 | node, 32 | parent, 33 | parentPath, 34 | key, 35 | index, 36 | stop: function () { 37 | this.stopped = true; 38 | }, 39 | skip: function () { 40 | this.skipped = true; 41 | }, 42 | remove: function () { 43 | this.removed = true; 44 | }, 45 | replaceWith: function (replacer) { 46 | this.replaced = replacer; 47 | }, 48 | findParent: function (test) { 49 | return !parentPath 50 | ? parentPath 51 | : test(parentPath) 52 | ? parentPath 53 | : parentPath.findParent(test); 54 | }, 55 | }); 56 | 57 | export type NodeVisitor = { 58 | enter?: (p: Path) => void; 59 | exit?: (p: Path) => void; 60 | }; 61 | 62 | // This builds a type of all AST types to a visitor type. Aka it builds 63 | // { 64 | // function_call: NodeVisitor, 65 | // ... 66 | // } 67 | // AstNode['type'] is the union of all the type properties of all AST nodes. 68 | // Extract pulls out the type from the AstNode union where the "type" 69 | // property matches the NodeType (like "function_call"). Pretty sweet! 70 | export type NodeVisitors = { 71 | [NodeType in AstNode['type']]?: NodeVisitor< 72 | Extract 73 | >; 74 | } & { program?: NodeVisitor }; 75 | 76 | /** 77 | * Apply the visitor pattern to an AST that conforms to this compiler's spec 78 | */ 79 | export const visit = (ast: Program | AstNode, visitors: NodeVisitors) => { 80 | let stopped = false; 81 | 82 | const visitNode = ( 83 | node: AstNode | Program, 84 | parent?: AstNode | Program, 85 | parentPath?: Path, 86 | key?: string, 87 | index?: number 88 | ) => { 89 | // Handle case where stop happened at exit 90 | if (stopped) { 91 | return; 92 | } 93 | 94 | const visitor = visitors[node.type]; 95 | const path = makePath(node, parent, parentPath, key, index); 96 | const parentNode = parent as any; 97 | 98 | if (visitor?.enter) { 99 | visitor.enter(path as any); 100 | if (path.removed) { 101 | if (!key || !parent) { 102 | throw new Error( 103 | `Asked to remove ${node} but no parent key was present in ${parent}` 104 | ); 105 | } 106 | if (typeof index === 'number') { 107 | parentNode[key].splice(index, 1); 108 | } else { 109 | parentNode[key] = null; 110 | } 111 | return path; 112 | } 113 | if (path.replaced) { 114 | if (!key || !parent) { 115 | throw new Error( 116 | `Asked to remove ${node} but no parent key was present in ${parent}` 117 | ); 118 | } 119 | if (typeof index === 'number') { 120 | parentNode[key].splice(index, 1, path.replaced); 121 | } else { 122 | parentNode[key] = path.replaced; 123 | } 124 | } 125 | if (path.skipped) { 126 | return path; 127 | } 128 | } 129 | 130 | if (path.stopped) { 131 | stopped = true; 132 | return; 133 | } 134 | 135 | if (path.replaced) { 136 | const replacedNode = path.replaced as AstNode; 137 | visitNode(replacedNode, parent, parentPath, key, index); 138 | } else { 139 | Object.entries(node) 140 | .filter(([_, nodeValue]) => isTraversable(nodeValue)) 141 | .forEach(([nodeKey, nodeValue]) => { 142 | if (Array.isArray(nodeValue)) { 143 | for ( 144 | let i = 0, offset = 0; 145 | i - offset < nodeValue.length && !stopped; 146 | i++ 147 | ) { 148 | const child = nodeValue[i - offset]; 149 | const res = visitNode(child, node, path, nodeKey, i - offset); 150 | if (res?.removed) { 151 | offset += 1; 152 | } 153 | } 154 | } else { 155 | if (!stopped) { 156 | visitNode(nodeValue, node, path, nodeKey); 157 | } 158 | } 159 | }); 160 | 161 | if (!stopped) { 162 | visitor?.exit?.(path as any); 163 | } 164 | } 165 | }; 166 | 167 | visitNode(ast); 168 | }; 169 | -------------------------------------------------------------------------------- /src/parser/utils.ts: -------------------------------------------------------------------------------- 1 | import { Program } from '../ast/ast-types.js'; 2 | import { 3 | FunctionOverloadIndex, 4 | FunctionScopeIndex, 5 | Scope, 6 | ScopeEntry, 7 | ScopeIndex, 8 | TypeScopeEntry, 9 | TypeScopeIndex, 10 | } from './scope.js'; 11 | 12 | export const renameBinding = (binding: ScopeEntry, newName: string) => { 13 | binding.references.forEach((node) => { 14 | if (node.type === 'declaration') { 15 | node.identifier.identifier = newName; 16 | } else if (node.type === 'identifier') { 17 | node.identifier = newName; 18 | } else if (node.type === 'parameter_declaration' && node.identifier) { 19 | node.identifier.identifier = newName; 20 | /* Ignore case of: 21 | layout(std140,column_major) uniform; 22 | uniform Material { 23 | uniform vec2 prop; 24 | } 25 | */ 26 | } else if (node.type !== 'interface_declarator') { 27 | console.warn('Unknown binding node', node); 28 | throw new Error(`Binding for type ${node.type} not recognized`); 29 | } 30 | }); 31 | return binding; 32 | }; 33 | 34 | export const renameBindings = ( 35 | bindings: ScopeIndex, 36 | mangle: (name: string) => string 37 | ) => 38 | Object.entries(bindings).reduce((acc, [name, binding]) => { 39 | const mangled = mangle(name); 40 | return { 41 | ...acc, 42 | [mangled]: renameBinding(binding, mangled), 43 | }; 44 | }, {}); 45 | 46 | export const renameType = (type: TypeScopeEntry, newName: string) => { 47 | type.references.forEach((node) => { 48 | if (node.type === 'type_name') { 49 | node.identifier = newName; 50 | } else { 51 | console.warn('Unknown type node', node); 52 | throw new Error(`Type ${node.type} not recognized`); 53 | } 54 | }); 55 | return type; 56 | }; 57 | 58 | export const renameTypes = ( 59 | types: TypeScopeIndex, 60 | mangle: (name: string) => string 61 | ) => 62 | Object.entries(types).reduce((acc, [name, type]) => { 63 | const mangled = mangle(name); 64 | return { 65 | ...acc, 66 | [mangled]: renameType(type, mangled), 67 | }; 68 | }, {}); 69 | 70 | export const renameFunction = ( 71 | overloadIndex: FunctionOverloadIndex, 72 | newName: string 73 | ) => { 74 | Object.entries(overloadIndex).forEach(([signature, overload]) => { 75 | overload.references.forEach((node) => { 76 | if (node.type === 'function') { 77 | node['prototype'].header.name.identifier = newName; 78 | } else if ( 79 | node.type === 'function_call' && 80 | node.identifier.type === 'postfix' 81 | ) { 82 | // @ts-ignore 83 | const specifier = node.identifier.expression.identifier.specifier; 84 | if (specifier) { 85 | specifier.identifier = newName; 86 | } else { 87 | console.warn('Unknown function node to rename', node); 88 | throw new Error( 89 | `Function specifier type ${node.type} not recognized` 90 | ); 91 | } 92 | } else if ( 93 | node.type === 'function_call' && 94 | 'specifier' in node.identifier && 95 | 'identifier' in node.identifier.specifier 96 | ) { 97 | node.identifier.specifier.identifier = newName; 98 | } else if ( 99 | node.type === 'function_call' && 100 | node.identifier.type === 'identifier' 101 | ) { 102 | node.identifier.identifier = newName; 103 | } else if (node.type === 'function_prototype') { 104 | node.header.name.identifier = newName; 105 | } else { 106 | console.warn('Unknown function node to rename', node); 107 | throw new Error(`Function for type ${node.type} not recognized`); 108 | } 109 | }); 110 | }); 111 | return overloadIndex; 112 | }; 113 | 114 | export const renameFunctions = ( 115 | functions: FunctionScopeIndex, 116 | mangle: (name: string) => string 117 | ) => 118 | Object.entries(functions).reduce( 119 | (acc, [fnName, overloads]) => { 120 | const mangled = mangle(fnName); 121 | return { 122 | ...acc, 123 | [mangled]: renameFunction(overloads, mangled), 124 | }; 125 | }, 126 | {} 127 | ); 128 | 129 | export const xor = (a: any, b: any): boolean => (a || b) && !(a && b); 130 | 131 | export const debugEntry = (bindings: ScopeIndex) => { 132 | return Object.entries(bindings).map( 133 | ([k, v]) => 134 | `${k}: (${v.references.length} references, ${ 135 | v.declaration ? '' : 'un' 136 | }declared): ${v.references.map((r) => r.type).join(', ')}` 137 | ); 138 | }; 139 | export const debugFunctionEntry = (bindings: FunctionScopeIndex) => 140 | Object.entries(bindings).flatMap(([name, overloads]) => 141 | Object.entries(overloads).map( 142 | ([signature, overload]) => 143 | `${name} (${signature}): (${overload.references.length} references, ${ 144 | overload.declaration ? '' : 'un' 145 | }declared): ${overload.references.map((r) => r.type).join(', ')}` 146 | ) 147 | ); 148 | 149 | export const debugScopes = (astOrScopes: Program | Scope[]) => 150 | console.log( 151 | 'Scopes:', 152 | 'scopes' in astOrScopes 153 | ? astOrScopes.scopes 154 | : astOrScopes.map((s) => ({ 155 | name: s.name, 156 | types: debugEntry(s.types), 157 | bindings: debugEntry(s.bindings), 158 | functions: debugFunctionEntry(s.functions), 159 | })) 160 | ); 161 | -------------------------------------------------------------------------------- /src/preprocessor/preprocessor-node.ts: -------------------------------------------------------------------------------- 1 | export interface IPreprocessorNode { 2 | // Only used on preprocessor nodes 3 | wsStart?: string; 4 | wsEnd?: string; 5 | } 6 | 7 | export interface PreprocessorBinaryNode extends IPreprocessorNode { 8 | type: 'binary'; 9 | left: PreprocessorAstNode; 10 | right: PreprocessorAstNode; 11 | operator: PreprocessorLiteralNode; 12 | } 13 | 14 | export type PreprocessorIfPart = 15 | | PreprocessorIfNode 16 | | PreprocessorIfDefNode 17 | | PreprocessorIfndefNode; 18 | 19 | export interface PreprocessorConditionalNode extends IPreprocessorNode { 20 | type: 'conditional'; 21 | ifPart: PreprocessorIfPart; 22 | elseIfParts: PreprocessorElseIfNode[]; 23 | elsePart: PreprocessorElseNode; 24 | endif: PreprocessorLiteralNode; 25 | wsEnd: string; 26 | } 27 | 28 | export interface PreprocessorDefineArgumentsNode extends IPreprocessorNode { 29 | type: 'define_arguments'; 30 | define: string; 31 | identifier: PreprocessorIdentifierNode; 32 | lp: PreprocessorLiteralNode; 33 | args: PreprocessorLiteralNode[]; 34 | rp: PreprocessorLiteralNode; 35 | body: string; 36 | } 37 | 38 | export interface PreprocessorDefineNode extends IPreprocessorNode { 39 | type: 'define'; 40 | define: string; 41 | identifier: PreprocessorIdentifierNode; 42 | body: string; 43 | } 44 | 45 | export interface PreprocessorElseNode extends IPreprocessorNode { 46 | type: 'else'; 47 | token: string; 48 | wsEnd: string; 49 | body: PreprocessorAstNode; 50 | } 51 | 52 | export interface PreprocessorElseIfNode extends IPreprocessorNode { 53 | type: 'elseif'; 54 | token: string; 55 | expression: PreprocessorAstNode; 56 | wsEnd: string; 57 | body: PreprocessorAstNode; 58 | } 59 | 60 | export interface PreprocessorErrorNode extends IPreprocessorNode { 61 | type: 'error'; 62 | error: string; 63 | message: string; 64 | } 65 | 66 | export interface PreprocessorExtensionNode extends IPreprocessorNode { 67 | type: 'extension'; 68 | extension: string; 69 | name: string; 70 | colon: string; 71 | behavior: string; 72 | } 73 | 74 | export interface PreprocessorGroupNode extends IPreprocessorNode { 75 | type: 'group'; 76 | lp: PreprocessorLiteralNode; 77 | expression: PreprocessorAstNode; 78 | rp: PreprocessorLiteralNode; 79 | } 80 | 81 | export interface PreprocessorIdentifierNode extends IPreprocessorNode { 82 | type: 'identifier'; 83 | identifier: string; 84 | wsEnd: string; 85 | } 86 | 87 | export interface PreprocessorIfNode extends IPreprocessorNode { 88 | type: 'if'; 89 | token: string; 90 | expression: PreprocessorAstNode; 91 | body: PreprocessorAstNode; 92 | } 93 | 94 | export interface PreprocessorIfDefNode extends IPreprocessorNode { 95 | type: 'ifdef'; 96 | token: string; 97 | identifier: PreprocessorIdentifierNode; 98 | body: PreprocessorAstNode; 99 | } 100 | 101 | export interface PreprocessorIfndefNode extends IPreprocessorNode { 102 | type: 'ifndef'; 103 | token: string; 104 | identifier: PreprocessorIdentifierNode; 105 | body: PreprocessorAstNode; 106 | } 107 | 108 | export interface PreprocessorIntConstantNode extends IPreprocessorNode { 109 | type: 'int_constant'; 110 | token: string; 111 | wsEnd: string; 112 | } 113 | 114 | export interface PreprocessorLineNode extends IPreprocessorNode { 115 | type: 'line'; 116 | line: string; 117 | value: string; 118 | } 119 | 120 | export interface PreprocessorLiteralNode extends IPreprocessorNode { 121 | type: 'literal'; 122 | literal: string; 123 | wsStart: string; 124 | wsEnd: string; 125 | } 126 | 127 | export interface PreprocessorPragmaNode extends IPreprocessorNode { 128 | type: 'pragma'; 129 | pragma: string; 130 | body: string; 131 | } 132 | 133 | export interface PreprocessorSegmentNode extends IPreprocessorNode { 134 | type: 'segment'; 135 | blocks: PreprocessorAstNode[]; 136 | } 137 | 138 | export interface PreprocessorTextNode extends IPreprocessorNode { 139 | type: 'text'; 140 | text: string; 141 | } 142 | 143 | export interface PreprocessorUnaryDefinedNode extends IPreprocessorNode { 144 | type: 'unary_defined'; 145 | operator: PreprocessorLiteralNode; 146 | lp: PreprocessorLiteralNode; 147 | identifier: PreprocessorIdentifierNode; 148 | rp: PreprocessorLiteralNode; 149 | } 150 | 151 | export interface PreprocessorUnaryNode extends IPreprocessorNode { 152 | type: 'unary'; 153 | operator: PreprocessorLiteralNode; 154 | expression: PreprocessorAstNode; 155 | } 156 | 157 | export interface PreprocessorUndefNode extends IPreprocessorNode { 158 | type: 'undef'; 159 | undef: string; 160 | identifier: PreprocessorIdentifierNode; 161 | } 162 | 163 | export interface PreprocessorVersionNode extends IPreprocessorNode { 164 | type: 'version'; 165 | version: PreprocessorLiteralNode; 166 | value: string; 167 | profile: string; 168 | } 169 | 170 | export type PreprocessorAstNode = 171 | | PreprocessorBinaryNode 172 | | PreprocessorConditionalNode 173 | | PreprocessorDefineArgumentsNode 174 | | PreprocessorDefineNode 175 | | PreprocessorElseNode 176 | | PreprocessorElseIfNode 177 | | PreprocessorErrorNode 178 | | PreprocessorExtensionNode 179 | | PreprocessorGroupNode 180 | | PreprocessorIdentifierNode 181 | | PreprocessorIfNode 182 | | PreprocessorIfDefNode 183 | | PreprocessorIfndefNode 184 | | PreprocessorIntConstantNode 185 | | PreprocessorLineNode 186 | | PreprocessorLiteralNode 187 | | PreprocessorPragmaNode 188 | | PreprocessorSegmentNode 189 | | PreprocessorTextNode 190 | | PreprocessorUnaryDefinedNode 191 | | PreprocessorUnaryNode 192 | | PreprocessorUndefNode 193 | | PreprocessorVersionNode; 194 | -------------------------------------------------------------------------------- /src/ast/ast.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AstNode, 3 | BinaryNode, 4 | IdentifierNode, 5 | LiteralNode, 6 | } from './ast-types.js'; 7 | import { Path, visit } from './visit.js'; 8 | 9 | const visitLogger = () => { 10 | const visitLog: Array<['enter' | 'exit', AstNode['type']]> = []; 11 | const track = (type: 'enter' | 'exit') => (path: Path) => 12 | visitLog.push([type, path.node.type]); 13 | const enter = track('enter'); 14 | const exit = track('exit'); 15 | return [visitLog, enter, exit, track] as const; 16 | }; 17 | 18 | const literal = (literal: T): LiteralNode => ({ 19 | type: 'literal', 20 | literal, 21 | whitespace: '', 22 | }); 23 | const identifier = (identifier: string): IdentifierNode => ({ 24 | type: 'identifier', 25 | identifier, 26 | whitespace: '', 27 | }); 28 | 29 | test('visit()', () => { 30 | const tree: BinaryNode = { 31 | type: 'binary', 32 | operator: literal('-'), 33 | // mock location data 34 | location: { 35 | start: { line: 0, column: 0, offset: 0 }, 36 | end: { line: 0, column: 0, offset: 0 }, 37 | }, 38 | left: { 39 | type: 'binary', 40 | operator: literal('+'), 41 | left: identifier('foo'), 42 | right: identifier('bar'), 43 | }, 44 | right: { 45 | type: 'group', 46 | lp: literal('('), 47 | rp: literal(')'), 48 | expression: identifier('baz'), 49 | }, 50 | }; 51 | 52 | let grandparent: AstNode | undefined; 53 | let parent: AstNode | undefined; 54 | let unfound; 55 | 56 | visit(tree, { 57 | identifier: { 58 | enter: (path) => { 59 | const { node } = path; 60 | if (node.identifier === 'foo') { 61 | grandparent = path.findParent( 62 | ({ node }) => node.operator.literal === '-' 63 | )?.node; 64 | parent = path.findParent(({ node }) => node.operator.literal === '+') 65 | ?.node; 66 | unfound = path.findParent(({ node }) => node.operator.literal === '*') 67 | ?.node; 68 | } 69 | }, 70 | }, 71 | }); 72 | 73 | expect(grandparent).not.toBeNull(); 74 | expect(grandparent?.type).toBe('binary'); 75 | expect(parent).not.toBeNull(); 76 | expect(parent?.type).toBe('binary'); 77 | expect(unfound).not.toBeDefined(); 78 | }); 79 | 80 | test('visit with replace', () => { 81 | const [visitLog, enter, exit] = visitLogger(); 82 | 83 | const tree: BinaryNode = { 84 | type: 'binary', 85 | operator: literal('-'), 86 | // mock location data 87 | location: { 88 | start: { line: 0, column: 0, offset: 0 }, 89 | end: { line: 0, column: 0, offset: 0 }, 90 | }, 91 | left: identifier('foo'), 92 | right: { 93 | type: 'group', 94 | lp: literal('('), 95 | rp: literal(')'), 96 | expression: identifier('bar'), 97 | }, 98 | }; 99 | 100 | let sawBar = false; 101 | let sawBaz = false; 102 | 103 | visit(tree, { 104 | group: { 105 | enter: (path) => { 106 | enter(path); 107 | path.replaceWith(identifier('baz')); 108 | }, 109 | exit, 110 | }, 111 | binary: { 112 | enter, 113 | exit, 114 | }, 115 | literal: { 116 | enter, 117 | exit, 118 | }, 119 | identifier: { 120 | enter: (path) => { 121 | enter(path); 122 | if (path.node.identifier === 'baz') { 123 | sawBaz = true; 124 | } 125 | if (path.node.identifier === 'bar') { 126 | sawBar = true; 127 | } 128 | }, 129 | exit, 130 | }, 131 | }); 132 | 133 | expect(visitLog).toEqual([ 134 | ['enter', 'binary'], 135 | 136 | // tree.operator 137 | ['enter', 'literal'], 138 | ['exit', 'literal'], 139 | 140 | // tree.left 141 | ['enter', 'identifier'], 142 | ['exit', 'identifier'], 143 | 144 | // tree.right 145 | ['enter', 'group'], 146 | // No exit because it got replaced 147 | 148 | // Replaced tree.right 149 | ['enter', 'identifier'], 150 | ['exit', 'identifier'], 151 | 152 | ['exit', 'binary'], 153 | ]); 154 | 155 | // The children of the node that got replaced should not be visited 156 | expect(sawBar).toBeFalsy(); 157 | 158 | // The children of the new replacement node should be visited 159 | expect(sawBaz).toBeTruthy(); 160 | }); 161 | 162 | test('visit stop()', () => { 163 | const [visitLog, enter, exit] = visitLogger(); 164 | 165 | const tree: BinaryNode = { 166 | type: 'binary', 167 | operator: literal('-'), 168 | left: { 169 | type: 'binary', 170 | operator: literal('+'), 171 | left: identifier('foo'), 172 | right: identifier('bar'), 173 | }, 174 | right: { 175 | type: 'group', 176 | lp: literal('('), 177 | rp: literal(')'), 178 | expression: identifier('baz'), 179 | }, 180 | }; 181 | 182 | visit(tree, { 183 | group: { 184 | enter, 185 | exit, 186 | }, 187 | binary: { 188 | enter, 189 | exit, 190 | }, 191 | literal: { 192 | enter, 193 | exit, 194 | }, 195 | identifier: { 196 | enter: (path) => { 197 | enter(path); 198 | if (path.node.identifier === 'foo') { 199 | path.stop(); 200 | } 201 | }, 202 | exit, 203 | }, 204 | }); 205 | 206 | expect(visitLog).toEqual([ 207 | ['enter', 'binary'], 208 | 209 | // tree.operator 210 | ['enter', 'literal'], 211 | ['exit', 'literal'], 212 | 213 | // tree.left 214 | ['enter', 'binary'], 215 | ['enter', 'literal'], 216 | ['exit', 'literal'], 217 | 218 | // stop on first identifier! 219 | ['enter', 'identifier'], 220 | ]); 221 | }); 222 | -------------------------------------------------------------------------------- /src/parser/test-helpers.ts: -------------------------------------------------------------------------------- 1 | import { execSync } from 'child_process'; 2 | import { GrammarError } from 'peggy'; 3 | import util from 'util'; 4 | import generate from './generator.js'; 5 | import { AstNode, FunctionNode, Program } from '../ast/index.js'; 6 | import { Parse, ParserOptions } from './parser.js'; 7 | import { FunctionScopeIndex, Scope, ScopeIndex } from './scope.js'; 8 | 9 | export const inspect = (arg: any) => 10 | console.log(util.inspect(arg, false, null, true)); 11 | 12 | export const nextWarn = () => { 13 | console.warn = jest.fn(); 14 | let i = 0; 15 | // @ts-ignore 16 | const mock = console.warn.mock; 17 | return () => mock.calls[i++][0]; 18 | }; 19 | 20 | type Context = { 21 | parse: Parse; 22 | parseSrc: ParseSrc; 23 | }; 24 | 25 | export const buildParser = () => { 26 | execSync( 27 | 'npx peggy --cache --format es -o src/parser/parser.js src/parser/glsl-grammar.pegjs' 28 | ); 29 | const parser = require('./parser'); 30 | const parse = parser.parse as Parse; 31 | const ps = parseSrc(parse); 32 | const ctx: Context = { 33 | parse, 34 | parseSrc: ps, 35 | }; 36 | return { 37 | parse, 38 | parser, 39 | parseSrc: ps, 40 | debugSrc: debugSrc(ctx), 41 | debugStatement: debugStatement(ctx), 42 | expectParsedStatement: expectParsedStatement(ctx), 43 | parseStatement: parseStatement(ctx), 44 | expectParsedProgram: expectParsedProgram(ctx), 45 | }; 46 | }; 47 | 48 | export const buildPreprocessorParser = () => { 49 | execSync( 50 | 'npx peggy --cache --format es -o src/preprocessor/preprocessor-parser.js src/preprocessor/preprocessor-grammar.pegjs' 51 | ); 52 | const { parse, parser } = require('../preprocessor'); 53 | return { 54 | parse, 55 | parser, 56 | }; 57 | }; 58 | 59 | // Keeping this around in case I need to figure out how to do tracing again 60 | // Most of this ceremony around building a parser is dealing with Peggy's error 61 | // format() function, where the grammarSource has to line up in generate() and 62 | // format() to get nicely formatted errors if there's a syntax error in the 63 | // grammar 64 | // const buildParser = (file: string) => { 65 | // const grammar = fileContents(file); 66 | // try { 67 | // return peggy.generate(grammar, { 68 | // grammarSource: file, 69 | // cache: true, 70 | // trace: false, 71 | // }); 72 | // } catch (e) { 73 | // const err = e as SyntaxError; 74 | // if ('format' in err && typeof err.format === 'function') { 75 | // console.error(err.format([{ source: file, text: grammar }])); 76 | // } 77 | // throw e; 78 | // } 79 | // }; 80 | 81 | const middle = /\/\* start \*\/((.|[\r\n])+)(\/\* end \*\/)?/m; 82 | 83 | type ParseSrc = (src: string, options?: ParserOptions) => Program; 84 | const parseSrc = (parse: Parse): ParseSrc => (src, options = {}) => { 85 | const grammarSource = ''; 86 | try { 87 | return parse(src, { 88 | ...options, 89 | grammarSource, 90 | tracer: { 91 | trace: (type) => { 92 | if ( 93 | type.type === 'rule.match' && 94 | type.rule !== 'whitespace' && 95 | type.rule !== 'single_comment' && 96 | type.rule !== 'comment' && 97 | type.rule !== 'digit_sequence' && 98 | type.rule !== 'digit' && 99 | type.rule !== 'fractional_constant' && 100 | type.rule !== 'floating_constant' && 101 | type.rule !== 'translation_unit' && 102 | type.rule !== 'start' && 103 | type.rule !== 'external_declaration' && 104 | type.rule !== 'SEMICOLON' && 105 | type.rule !== 'terminal' && 106 | type.rule !== '_' 107 | ) { 108 | if (type.rule === 'IDENTIFIER' || type.rule === 'TYPE_NAME') { 109 | console.log( 110 | '\x1b[35mMatch literal\x1b[0m', 111 | type.rule, 112 | type.result 113 | ); 114 | } else { 115 | console.log('\x1b[35mMatch\x1b[0m', type.rule); 116 | } 117 | } 118 | // if (type.type === 'rule.fail') { 119 | // console.log('fail', type.rule); 120 | // } 121 | }, 122 | }, 123 | }); 124 | } catch (e) { 125 | const err = e as GrammarError; 126 | if ('format' in err) { 127 | console.error(err.format([{ source: grammarSource, text: src }])); 128 | } 129 | console.error(`Error parsing lexeme!\n"${src}"`); 130 | throw err; 131 | } 132 | }; 133 | 134 | const debugSrc = ({ parseSrc }: Context) => (src: string) => { 135 | inspect(parseSrc(src).program); 136 | }; 137 | 138 | const debugStatement = ({ parseSrc }: Context) => (stmt: AstNode) => { 139 | const program = `void main() {/* start */${stmt}/* end */}`; 140 | const ast = parseSrc(program); 141 | inspect((ast.program[0] as FunctionNode).body.statements[0]); 142 | }; 143 | 144 | const expectParsedStatement = ({ parseSrc }: Context) => ( 145 | src: string, 146 | options = {} 147 | ) => { 148 | const program = `void main() {/* start */${src}/* end */}`; 149 | const ast = parseSrc(program, options); 150 | const glsl = generate(ast); 151 | if (glsl !== program) { 152 | inspect(ast.program[0]); 153 | // @ts-ignore 154 | expect(glsl.match(middle)[1]).toBe(src); 155 | } 156 | }; 157 | 158 | const parseStatement = ({ parseSrc }: Context) => ( 159 | src: string, 160 | options: ParserOptions = {} 161 | ) => { 162 | const program = `void main() {${src}}`; 163 | return parseSrc(program, options); 164 | }; 165 | 166 | const expectParsedProgram = ({ parseSrc }: Context) => ( 167 | src: string, 168 | options?: ParserOptions 169 | ) => { 170 | const ast = parseSrc(src, options); 171 | const glsl = generate(ast); 172 | if (glsl !== src) { 173 | inspect(ast); 174 | expect(glsl).toBe(src); 175 | } 176 | }; 177 | -------------------------------------------------------------------------------- /src/parser/scope.ts: -------------------------------------------------------------------------------- 1 | // This file is compiled and inlined in /glsl-grammar.pegjs. See build-parser.sh 2 | // and note that file is called in parse.test.ts 3 | import { 4 | AstNode, 5 | LocationObject, 6 | ArraySpecifierNode, 7 | FunctionPrototypeNode, 8 | KeywordNode, 9 | FunctionNode, 10 | FunctionCallNode, 11 | TypeNameNode, 12 | } from '../ast/index.js'; 13 | import { xor } from './utils.js'; 14 | 15 | export type TypeScopeEntry = { 16 | declaration?: TypeNameNode; 17 | references: TypeNameNode[]; 18 | }; 19 | export type TypeScopeIndex = { 20 | [name: string]: TypeScopeEntry; 21 | }; 22 | export type ScopeEntry = { declaration?: AstNode; references: AstNode[] }; 23 | export type ScopeIndex = { 24 | [name: string]: ScopeEntry; 25 | }; 26 | export type FunctionOverloadDefinition = { 27 | returnType: string; 28 | parameterTypes: string[]; 29 | declaration?: FunctionNode; 30 | references: (FunctionNode | FunctionCallNode | FunctionPrototypeNode)[]; 31 | }; 32 | export type FunctionOverloadIndex = { 33 | [signature: string]: FunctionOverloadDefinition; 34 | }; 35 | export type FunctionScopeIndex = { 36 | [name: string]: FunctionOverloadIndex; 37 | }; 38 | 39 | export type Scope = { 40 | name: string; 41 | parent?: Scope; 42 | bindings: ScopeIndex; 43 | types: TypeScopeIndex; 44 | functions: FunctionScopeIndex; 45 | location?: LocationObject; 46 | }; 47 | 48 | export const UNKNOWN_TYPE = 'UNKNOWN TYPE'; 49 | 50 | export type FunctionSignature = [ 51 | returnType: string, 52 | parameterTypes: string[], 53 | signature: string 54 | ]; 55 | 56 | export const makeScopeIndex = ( 57 | firstReference: AstNode, 58 | declaration?: AstNode 59 | ): ScopeEntry => ({ 60 | declaration, 61 | references: [firstReference], 62 | }); 63 | 64 | export const findTypeScope = ( 65 | scope: Scope | undefined, 66 | typeName: string 67 | ): Scope | null => { 68 | if (!scope) { 69 | return null; 70 | } 71 | if (typeName in scope.types) { 72 | return scope; 73 | } 74 | return findTypeScope(scope.parent, typeName); 75 | }; 76 | 77 | export const isDeclaredType = (scope: Scope, typeName: string) => 78 | findTypeScope(scope, typeName) !== null; 79 | 80 | export const findBindingScope = ( 81 | scope: Scope | undefined, 82 | name: string 83 | ): Scope | null => { 84 | if (!scope) { 85 | return null; 86 | } 87 | if (name in scope.bindings) { 88 | return scope; 89 | } 90 | return findBindingScope(scope.parent, name); 91 | }; 92 | 93 | export const extractConstant = (expression: AstNode): string => { 94 | let result = UNKNOWN_TYPE; 95 | // Keyword case, like float 96 | if ('token' in expression) { 97 | result = expression.token; 98 | // User defined type 99 | } else if ( 100 | 'identifier' in expression && 101 | typeof expression.identifier === 'string' 102 | ) { 103 | result = expression.identifier; 104 | } else { 105 | console.warn(result, expression); 106 | } 107 | return result; 108 | }; 109 | 110 | export const quantifiersSignature = (quantifier: ArraySpecifierNode[]) => 111 | quantifier.map((q) => `[${extractConstant(q.expression)}]`).join(''); 112 | 113 | export const functionDeclarationSignature = ( 114 | node: FunctionNode | FunctionPrototypeNode 115 | ): FunctionSignature => { 116 | const proto = node.type === 'function' ? node.prototype : node; 117 | const { specifier } = proto.header.returnType; 118 | const quantifiers = specifier.quantifier || []; 119 | 120 | const parameterTypes = proto?.parameters?.map(({ specifier, quantifier }) => { 121 | const quantifiers = 122 | // vec4[1][2] param 123 | specifier.quantifier || 124 | // vec4 param[1][3] 125 | quantifier || 126 | []; 127 | return `${extractConstant(specifier.specifier)}${quantifiersSignature( 128 | quantifiers 129 | )}`; 130 | }) || ['void']; 131 | 132 | const returnType = `${ 133 | (specifier.specifier as KeywordNode).token 134 | }${quantifiersSignature(quantifiers)}`; 135 | 136 | return [ 137 | returnType, 138 | parameterTypes, 139 | `${returnType}: ${parameterTypes.join(', ')}`, 140 | ]; 141 | }; 142 | 143 | export const doSignaturesMatch = ( 144 | definitionSignature: string, 145 | definition: FunctionOverloadDefinition, 146 | callSignature: FunctionSignature 147 | ) => { 148 | if (definitionSignature === callSignature[0]) { 149 | return true; 150 | } 151 | const left = [definition.returnType, ...definition.parameterTypes]; 152 | const right = [callSignature[0], ...callSignature[1]]; 153 | 154 | // Special case. When comparing "a()" to "a(1)", a() has paramater VOID, and 155 | // a(1) has type UNKNOWN. This will pass as true in the final check of this 156 | // function, even though it's not. 157 | if (left.length === 2 && xor(left[1] === 'void', right[1] === 'void')) { 158 | return false; 159 | } 160 | 161 | return ( 162 | left.length === right.length && 163 | left.every( 164 | (type, index) => 165 | type === right[index] || 166 | type === UNKNOWN_TYPE || 167 | right[index] === UNKNOWN_TYPE 168 | ) 169 | ); 170 | }; 171 | 172 | export const findOverloadDefinition = ( 173 | signature: FunctionSignature, 174 | index: FunctionOverloadIndex 175 | ): FunctionOverloadDefinition | undefined => { 176 | return Object.entries(index).reduce< 177 | ReturnType 178 | >((found, [overloadSignature, overloadDefinition]) => { 179 | return ( 180 | found || 181 | (doSignaturesMatch(overloadSignature, overloadDefinition, signature) 182 | ? overloadDefinition 183 | : undefined) 184 | ); 185 | }, undefined); 186 | }; 187 | 188 | export const functionUseSignature = ( 189 | node: FunctionCallNode 190 | ): FunctionSignature => { 191 | const parameterTypes = 192 | node.args.length === 0 193 | ? ['void'] 194 | : node.args 195 | .filter((arg) => (arg as any).literal !== ',') 196 | .map(() => UNKNOWN_TYPE); 197 | const returnType = UNKNOWN_TYPE; 198 | return [ 199 | returnType, 200 | parameterTypes, 201 | `${returnType}: ${parameterTypes.join(', ')}`, 202 | ]; 203 | }; 204 | 205 | export const newOverloadIndex = ( 206 | returnType: string, 207 | parameterTypes: string[], 208 | firstReference: FunctionNode | FunctionCallNode | FunctionPrototypeNode, 209 | declaration?: FunctionNode 210 | ): FunctionOverloadDefinition => ({ 211 | returnType, 212 | parameterTypes, 213 | declaration, 214 | references: [firstReference], 215 | }); 216 | 217 | export const findGlobalScope = (scope: Scope): Scope => 218 | scope.parent ? findGlobalScope(scope.parent) : scope; 219 | 220 | export const isDeclaredFunction = (scope: Scope, fnName: string) => 221 | fnName in findGlobalScope(scope).functions; 222 | -------------------------------------------------------------------------------- /src/error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * The types here are hand copied from peggy's peg.d.ts file so that end 3 | * consumers of the GLSL parser can use the error type without me having to 4 | * fully publish peggy as a dependency of this module. 5 | * 6 | * The primary exported type is GlslSyntaxError. 7 | */ 8 | 9 | /** Provides information pointing to a location within a source. */ 10 | export interface Location { 11 | /** Line in the parsed source (1-based). */ 12 | line: number; 13 | /** Column in the parsed source (1-based). */ 14 | column: number; 15 | /** Offset in the parsed source (0-based). */ 16 | offset: number; 17 | } 18 | 19 | /** The `start` and `end` position's of an object within the source. */ 20 | export interface LocationRange { 21 | /** Any object that was supplied to the `parse()` call as the `grammarSource` option. */ 22 | source: any; 23 | /** Position at the beginning of the expression. */ 24 | start: Location; 25 | /** Position after the end of the expression. */ 26 | end: Location; 27 | } 28 | 29 | /** Specific sequence of symbols is expected in the parsed source. */ 30 | interface LiteralExpectation { 31 | type: 'literal'; 32 | /** Expected sequence of symbols. */ 33 | text: string; 34 | /** If `true`, symbols of any case is expected. `text` in that case in lower case */ 35 | ignoreCase: boolean; 36 | } 37 | 38 | /** One of the specified symbols is expected in the parse position. */ 39 | interface ClassExpectation { 40 | type: 'class'; 41 | /** List of symbols and symbol ranges expected in the parse position. */ 42 | parts: (string[] | string)[]; 43 | /** 44 | * If `true`, meaning of `parts` is inverted: symbols that NOT expected in 45 | * the parse position. 46 | */ 47 | inverted: boolean; 48 | /** If `true`, symbols of any case is expected. `text` in that case in lower case */ 49 | ignoreCase: boolean; 50 | } 51 | 52 | /** Any symbol is expected in the parse position. */ 53 | interface AnyExpectation { 54 | type: 'any'; 55 | } 56 | 57 | /** EOF is expected in the parse position. */ 58 | interface EndExpectation { 59 | type: 'end'; 60 | } 61 | 62 | /** 63 | * Something other is expected in the parse position. That expectation is 64 | * generated by call of the `expected()` function in the parser code, as 65 | * well as rules with human-readable names. 66 | */ 67 | interface OtherExpectation { 68 | type: 'other'; 69 | /** 70 | * Depending on the origin of this expectation, can be: 71 | * - text, supplied to the `expected()` function 72 | * - human-readable name of the rule 73 | */ 74 | description: string; 75 | } 76 | 77 | type Expectation = 78 | | LiteralExpectation 79 | | ClassExpectation 80 | | AnyExpectation 81 | | EndExpectation 82 | | OtherExpectation; 83 | 84 | /** 85 | * The entry that maps object in the `source` property of error locations 86 | * to the actual source text of a grammar. That entries is necessary for 87 | * formatting errors. 88 | */ 89 | export interface SourceText { 90 | /** 91 | * Identifier of a grammar that stored in the `location().source` property 92 | * of error and diagnostic messages. 93 | * 94 | * This one should be the same object that used in the `location().source`, 95 | * because their compared using `===`. 96 | */ 97 | source: any; 98 | /** Source text of a grammar. */ 99 | text: string; 100 | } 101 | 102 | export interface SyntaxError extends Error { 103 | /** Location where error was originated. */ 104 | location: LocationRange; 105 | /** 106 | * List of possible tokens in the parse position, or `null` if error was 107 | * created by the `error()` call. 108 | */ 109 | expected: Expectation[] | null; 110 | /** 111 | * Character in the current parse position, or `null` if error was created 112 | * by the `error()` call. 113 | */ 114 | found: string | null; 115 | 116 | /** 117 | * Format the error with associated sources. The `location.source` should have 118 | * a `toString()` representation in order the result to look nice. If source 119 | * is `null` or `undefined`, it is skipped from the output 120 | * 121 | * Sample output: 122 | * ``` 123 | * Error: Expected "!", "$", "&", "(", ".", "@", character class, comment, end of line, identifier, literal, or whitespace but "#" found. 124 | * --> my grammar:3:9 125 | * | 126 | * 3 | start = # 'a'; 127 | * | ^ 128 | * ``` 129 | * 130 | * @param sources mapping from location source to source text 131 | * 132 | * @returns the formatted error 133 | */ 134 | format(sources: SourceText[]): string; 135 | } 136 | 137 | /** 138 | * Peggy's default error type is complete nonsense. It has the horrific 139 | * format() API to get a useful error message. 140 | */ 141 | export class GlslSyntaxError extends Error { 142 | /** Location where error was originated. */ 143 | location: LocationRange; 144 | 145 | /** 146 | * List of possible tokens in the parse position, or `null` if error was 147 | * created by the `error()` call. 148 | */ 149 | expected: Expectation[] | null; 150 | /** 151 | * Character in the current parse position, or `null` if error was created 152 | * by the `error()` call. 153 | */ 154 | found: string | null; 155 | 156 | constructor(source: string, grammarSource: string, error: SyntaxError) { 157 | // End users shouldn't have to deal with this - this line is the main 158 | // purpose of this class. #format() is what gives an ASCII formatted error 159 | // message with ASCII arrows pointing to the location of the source. For 160 | // example, this format method produces something like 161 | // Error: Expected end of input but "#" found. 162 | // --> location:3:5 163 | // | 164 | // 3 | #ifdef RENORMALZE_REFLECTANCE 165 | // | ^ 166 | super(error.format([{ source: grammarSource, text: source }])); 167 | this.location = error.location; 168 | this.expected = error.expected; 169 | this.found = error.found; 170 | } 171 | } 172 | 173 | // When the error is formatted, this is the string that shows before the 174 | // location text. For example this becomes "location:3:5". 175 | export const DEFAULT_GRAMMAR_SOURCE = 'location'; 176 | 177 | /** 178 | * Wrap the peggy parser to catch the built in SyntaxError and throw a 179 | * formatted GlslSyntaxError instead. 180 | */ 181 | export const formatError = < 182 | T extends { 183 | parse: (...args: any[]) => any; 184 | SyntaxError: new () => SyntaxError; 185 | } 186 | >( 187 | parser: T, 188 | grammarSource = DEFAULT_GRAMMAR_SOURCE 189 | // Some gymanastics to forward the return type of the parser so the exported 190 | // parse() function has the right types 191 | ): T['parse'] => (...args: Parameters) => { 192 | const [src, options] = args; 193 | try { 194 | return parser.parse(src, { 195 | grammarSource, 196 | ...options, 197 | }); 198 | } catch (e) { 199 | if (e instanceof parser.SyntaxError) { 200 | throw new GlslSyntaxError(src, grammarSource, e as SyntaxError); 201 | } 202 | throw e; 203 | } 204 | }; 205 | -------------------------------------------------------------------------------- /src/parser/generator.ts: -------------------------------------------------------------------------------- 1 | import { 2 | makeGenerator, 3 | makeEveryOtherGenerator, 4 | NodeGenerators, 5 | } from '../ast/index.js'; 6 | 7 | const generators: NodeGenerators = { 8 | program: (node) => generate(node.wsStart) + generate(node.program), 9 | preprocessor: (node) => generate(node.line) + generate(node._), 10 | keyword: (node) => generate(node.token) + generate(node.whitespace), 11 | 12 | precision: (node) => 13 | generate(node.prefix) + generate(node.qualifier) + generate(node.specifier), 14 | 15 | // Statements 16 | expression_statement: (node) => 17 | generate(node.expression) + generate(node.semi), 18 | if_statement: (node) => 19 | generate(node.if) + 20 | generate(node.lp) + 21 | generate(node.condition) + 22 | generate(node.rp) + 23 | generate(node.body) + 24 | generate(node.else), 25 | switch_statement: (node) => 26 | generate(node.switch) + 27 | generate(node.lp) + 28 | generate(node.expression) + 29 | generate(node.rp) + 30 | generate(node.lb) + 31 | generate(node.cases) + 32 | generate(node.rb), 33 | break_statement: (node) => generate(node.break) + generate(node.semi), 34 | do_statement: (node) => 35 | generate(node.do) + 36 | generate(node.body) + 37 | generate(node.while) + 38 | generate(node.lp) + 39 | generate(node.expression) + 40 | generate(node.rp) + 41 | generate(node.semi), 42 | continue_statement: (node) => generate(node.continue) + generate(node.semi), 43 | return_statement: (node) => 44 | generate(node.return) + generate(node.expression) + generate(node.semi), 45 | discard_statement: (node) => generate(node.discard) + generate(node.semi), 46 | while_statement: (node) => 47 | generate(node.while) + 48 | generate(node.lp) + 49 | generate(node.condition) + 50 | generate(node.rp) + 51 | generate(node.body), 52 | for_statement: (node) => 53 | generate(node.for) + 54 | generate(node.lp) + 55 | generate(node.init) + 56 | generate(node.initSemi) + 57 | generate(node.condition) + 58 | generate(node.conditionSemi) + 59 | generate(node.operation) + 60 | generate(node.rp) + 61 | generate(node.body), 62 | condition_expression: (node) => 63 | generate(node.specified_type) + 64 | generate(node.identifier) + 65 | generate(node.operator) + 66 | generate(node.initializer), 67 | declaration_statement: (node) => 68 | generate(node.declaration) + generate(node.semi), 69 | fully_specified_type: (node) => 70 | generate(node.qualifiers) + generate(node.specifier), 71 | layout_qualifier: (node) => 72 | generate(node.layout) + 73 | generate(node.lp) + 74 | generateWithEveryOther(node.qualifiers, node.commas) + 75 | generate(node.rp), 76 | layout_qualifier_id: (node) => 77 | generate(node.identifier) + 78 | generate(node.operator) + 79 | generate(node.expression), 80 | 81 | switch_case: (node) => 82 | generate(node.case) + 83 | generate(node.test) + 84 | generate(node.colon) + 85 | generate(node.statements), 86 | default_case: (node) => 87 | generate(node.default) + generate(node.colon) + generate(node.statements), 88 | 89 | declaration: (node) => 90 | generate(node.identifier) + 91 | generate(node.quantifier) + 92 | generate(node.equal) + 93 | generate(node.initializer), 94 | declarator_list: (node) => 95 | generate(node.specified_type) + 96 | generateWithEveryOther(node.declarations, node.commas), 97 | type_specifier: (node) => 98 | generate(node.specifier) + generate(node.quantifier), 99 | array_specifier: (node) => 100 | generate(node.lb) + generate(node.expression) + generate(node.rb), 101 | identifier: (node) => node.identifier + generate(node.whitespace), 102 | type_name: (node) => node.identifier + generate(node.whitespace), 103 | function_header: (node) => 104 | generate(node.returnType) + generate(node.name) + generate(node.lp), 105 | function_prototype: (node) => 106 | generate(node.header.returnType) + 107 | generate(node.header.name) + 108 | generate(node.header.lp) + 109 | (node.parameters 110 | ? generateWithEveryOther(node.parameters, node.commas) 111 | : '') + 112 | generate(node.rp), 113 | parameter_declaration: (node) => 114 | generate(node.qualifier) + 115 | generate(node.specifier) + 116 | generate(node.identifier) + 117 | generate(node.quantifier), 118 | compound_statement: (node) => 119 | generate(node.lb) + generate(node.statements) + generate(node.rb), 120 | function: (node) => generate(node['prototype']) + generate(node.body), 121 | function_call: (node) => 122 | generate(node.identifier) + 123 | generate(node.lp) + 124 | generate(node.args) + 125 | generate(node.rp), 126 | postfix: (node) => generate(node.expression) + generate(node.postfix), 127 | quantifier: (node) => 128 | generate(node.lb) + generate(node.expression) + generate(node.rb), 129 | quantified_identifier: (node) => 130 | generate(node.identifier) + generate(node.quantifier), 131 | field_selection: (node) => generate(node.dot) + generate(node.selection), 132 | 133 | subroutine_qualifier: (node) => 134 | generate(node.subroutine) + 135 | generate(node.lp) + 136 | generate(node.type_names) + 137 | generate(node.commas) + 138 | generate(node.rp), 139 | 140 | assignment: (node) => 141 | generate(node.left) + generate(node.operator) + generate(node.right), 142 | 143 | ternary: (node) => 144 | generate(node.expression) + 145 | generate(node.question) + 146 | generate(node.left) + 147 | generate(node.colon) + 148 | generate(node.right), 149 | 150 | binary: (node) => 151 | generate(node.left) + generate(node.operator) + generate(node.right), 152 | group: (node) => 153 | generate(node.lp) + generate(node.expression) + generate(node.rp), 154 | unary: (node) => generate(node.operator) + generate(node.expression), 155 | 156 | float_constant: (node) => generate(node.token) + generate(node.whitespace), 157 | double_constant: (node) => generate(node.token) + generate(node.whitespace), 158 | int_constant: (node) => generate(node.token) + generate(node.whitespace), 159 | uint_constant: (node) => generate(node.token) + generate(node.whitespace), 160 | bool_constant: (node) => generate(node.token) + generate(node.whitespace), 161 | 162 | literal: (node) => generate(node.literal) + generate(node.whitespace), 163 | 164 | struct: (node) => 165 | generate(node.struct) + 166 | generate(node.typeName) + 167 | generate(node.lb) + 168 | generate(node.declarations) + 169 | generate(node.rb), 170 | 171 | struct_declaration: (node) => 172 | generate(node.declaration) + generate(node.semi), 173 | 174 | interface_declarator: (node) => 175 | generate(node.qualifiers) + 176 | generate(node.interface_type) + 177 | generate(node.lp) + 178 | generate(node.declarations) + 179 | generate(node.rp) + 180 | generate(node.identifier), 181 | 182 | struct_declarator: (node) => 183 | generate(node.specified_type) + 184 | generateWithEveryOther(node.declarations, node.commas), 185 | 186 | initializer_list: (node) => 187 | generate(node.lb) + 188 | generateWithEveryOther(node.initializers, node.commas) + 189 | generate(node.rb), 190 | 191 | qualifier_declarator: (node) => 192 | generate(node.qualifiers) + 193 | generateWithEveryOther(node.declarations, node.commas), 194 | }; 195 | 196 | const generate = makeGenerator(generators); 197 | const generateWithEveryOther = makeEveryOtherGenerator(generate); 198 | 199 | export default generate; 200 | -------------------------------------------------------------------------------- /src/parser/parse.test.ts: -------------------------------------------------------------------------------- 1 | import { FunctionCallNode, visit } from '../ast/index.js'; 2 | import { GlslSyntaxError } from '../error.js'; 3 | import { buildParser } from './test-helpers.js'; 4 | 5 | let c!: ReturnType; 6 | beforeAll(() => (c = buildParser())); 7 | 8 | test('parse error', () => { 9 | let error: GlslSyntaxError | undefined; 10 | // Missing a semicolon 11 | const text = `float a 12 | float b`; 13 | try { 14 | c.parse(text); 15 | } catch (e) { 16 | error = e as GlslSyntaxError; 17 | } 18 | 19 | expect(error).toBeInstanceOf(c.parser.SyntaxError); 20 | expect(error!.location.start.line).toBe(2); 21 | expect(error!.location.end.line).toBe(2); 22 | }); 23 | 24 | test('declarations', () => { 25 | c.expectParsedProgram(` 26 | float a, b = 1.0, c = a; 27 | vec2 texcoord1, texcoord2; 28 | vec3 position; 29 | vec4 myRGBA; 30 | ivec2 textureLookup; 31 | bvec3 less; 32 | `); 33 | }); 34 | 35 | test('headers', () => { 36 | // The following includes the varying/attribute case which only works in GL 37 | // ES 1.00, and will need to be updated when the switch is implemented 38 | c.expectParsedProgram(` 39 | precision mediump float; 40 | precision highp int; 41 | 42 | in vec4 varName1; 43 | out vec4 varName2; 44 | 45 | varying vec4 varName3, blarName; 46 | uniform vec4 varName4; 47 | attribute vec4 varName5; 48 | `); 49 | }); 50 | 51 | test('if statement', () => { 52 | c.expectParsedStatement( 53 | `if(i != 0) { aFunction(); } 54 | else if(i == 2) { bFunction(); } 55 | else { cFunction(); }`, 56 | { 57 | quiet: true, 58 | } 59 | ); 60 | }); 61 | 62 | test('do while loop', () => { 63 | c.expectParsedStatement( 64 | ` 65 | do { 66 | aFunction(); 67 | break; 68 | continue; 69 | return; 70 | } while(i <= 99); 71 | `, 72 | { quiet: true } 73 | ); 74 | }); 75 | 76 | test('standard while loop', () => { 77 | c.expectParsedStatement( 78 | ` 79 | while(i <= 99) { 80 | aFunction(); 81 | break; 82 | continue; 83 | return; 84 | } 85 | `, 86 | { quiet: true } 87 | ); 88 | }); 89 | 90 | test('for loops', () => { 91 | // Infinite for loop 92 | c.expectParsedStatement(` 93 | for(;;) { 94 | } 95 | `); 96 | // For loop with jump statements 97 | c.expectParsedStatement( 98 | ` 99 | for(int a = 0; b <= 99; c++) { 100 | break; 101 | continue; 102 | return; 103 | aFunction(); 104 | } 105 | `, 106 | { quiet: true } 107 | ); 108 | // Loop with condition variable declaration (GLSL ES 3.00 only) 109 | c.expectParsedStatement(` 110 | for(int i = 0; bool x = false; i++) {} 111 | `); 112 | }); 113 | 114 | test('switch error', () => { 115 | // Test the semantic analysis case 116 | expect(() => 117 | c.parse( 118 | `void main() { 119 | switch (easingId) { 120 | result = cubicIn(); 121 | } 122 | }`, 123 | { quiet: true } 124 | ) 125 | ).toThrow(/must start with a case or default label/); 126 | }); 127 | 128 | test('switch statement', () => { 129 | c.expectParsedStatement( 130 | ` 131 | switch (easingId) { 132 | case 0: 133 | result = cubicIn(); 134 | break; 135 | case 1: 136 | result = cubicOut(); 137 | break; 138 | default: 139 | result = 1.0; 140 | } 141 | `, 142 | { quiet: true } 143 | ); 144 | }); 145 | 146 | test('qualifier declarations', () => { 147 | // The expected node here is "qualifier_declarator", which would be nice to 148 | // test for at some point, maybe when doing more AST analysis 149 | c.expectParsedProgram(` 150 | invariant precise in a, b,c; 151 | `); 152 | }); 153 | 154 | test('number notations', () => { 155 | // Integer hex notation 156 | c.expectParsedStatement(`highp uint value = 0x1234u;`); 157 | c.expectParsedStatement(`uint c = 0xffffffff;`); 158 | c.expectParsedStatement(`uint d = 0xffffffffU;`); 159 | // Octal 160 | c.expectParsedStatement(`uint d = 021234;`); 161 | // Double precision floating point 162 | c.expectParsedStatement(`double c, d = 2.0LF;`); 163 | // uint 164 | c.expectParsedStatement(`uint k = 3u;`); 165 | c.expectParsedStatement(`uint f = -1u;`); 166 | }); 167 | 168 | test('layout', () => { 169 | c.expectParsedProgram(` 170 | layout(location = 4, component = 2) in vec2 a; 171 | layout(location = 3) in vec4 normal1; 172 | layout(location = 9) in mat4 transforms[2]; 173 | layout(location = 3) in vec4 normal2; 174 | 175 | const int start = 6; 176 | layout(location = start + 2) in vec4 p; 177 | 178 | layout(location = 3) in struct S 179 | { 180 | vec3 a; // gets location 3 181 | mat2 b; // gets locations 4 and 5 182 | vec4 c[2]; // gets locations 6 and 7 183 | layout(location = 8) vec2 A; // ERROR, can't use on struct member 184 | } s; 185 | 186 | layout(location = 4) in block 187 | { 188 | vec4 d; // gets location 4 189 | vec4 e; // gets location 5 190 | layout(location = 7) vec4 f; // gets location 7 191 | vec4 g; // gets location 8 192 | layout(location = 1) vec4 h; // gets location 1 193 | vec4 i; // gets location 2 194 | vec4 j; // gets location 3 195 | vec4 k; // ERROR, location 4 already used 196 | }; 197 | 198 | // From the grammar but I think it's a typo 199 | // https://github.com/KhronosGroup/GLSL/issues/161 200 | // layout(location = start + 2) int vec4 p; 201 | 202 | layout(std140,column_major) uniform; 203 | `); 204 | }); 205 | 206 | test('parses comments', () => { 207 | c.expectParsedProgram( 208 | ` 209 | /* starting comment */ 210 | // hi 211 | void main(float x) { 212 | /* comment */// hi 213 | /* comment */ // hi 214 | statement(); // hi 215 | /* start */ statement(); /* end */ 216 | } 217 | `, 218 | { quiet: true } 219 | ); 220 | }); 221 | 222 | test('parses functions', () => { 223 | c.expectParsedProgram(` 224 | // Prototypes 225 | vec4 f(in vec4 x, out vec4 y); 226 | int newFunction(in bvec4 aBvec4, // read-only 227 | out vec3 aVec3, // write-only 228 | inout int aInt); // read-write 229 | highp float rand( const in vec2 uv ) {} 230 | highp float otherFn( const in vec3 rectCoords[ 4 ] ) {} 231 | `); 232 | }); 233 | 234 | test('parses function_call . postfix_expression', () => { 235 | c.expectParsedStatement('texture().rgb;', { quiet: true }); 236 | }); 237 | 238 | test('parses postfix_expression as function_identifier', () => { 239 | c.expectParsedStatement('a().length();', { quiet: true }); 240 | }); 241 | 242 | test('parses postfix expressions after non-function calls (aka map.length())', () => { 243 | c.expectParsedProgram( 244 | ` 245 | void main() { 246 | float y = x().length(); 247 | float x = map.length(); 248 | for (int i = 0; i < map.length(); i++) { 249 | } 250 | } 251 | `, 252 | { quiet: true } 253 | ); 254 | }); 255 | 256 | test('postfix, unary, binary expressions', () => { 257 | c.expectParsedStatement('x ++ + 1.0 + + 2.0;', { quiet: true }); 258 | }); 259 | 260 | test('operators', () => { 261 | c.expectParsedStatement('1 || 2 && 2 ^^ 3 >> 4 << 5;'); 262 | }); 263 | 264 | test('declaration', () => { 265 | c.expectParsedStatement('const float x = 1.0, y = 2.0;'); 266 | }); 267 | 268 | test('assignment', () => { 269 | c.expectParsedStatement('x |= 1.0;', { quiet: true }); 270 | }); 271 | 272 | test('ternary', () => { 273 | c.expectParsedStatement( 274 | 'float y = x == 1.0 ? x == 2.0 ? 1.0 : 3.0 : x == 3.0 ? 4.0 : 5.0;', 275 | { quiet: true } 276 | ); 277 | }); 278 | 279 | test('structs', () => { 280 | c.expectParsedProgram(` 281 | struct light { 282 | float intensity; 283 | vec3 position, color; 284 | } lightVar; 285 | light lightVar2; 286 | 287 | struct S { float f; }; 288 | `); 289 | }); 290 | 291 | test('buffer variables', () => { 292 | c.expectParsedProgram(` 293 | buffer b { 294 | float u[]; 295 | vec4 v[]; 296 | } name[3]; 297 | `); 298 | }); 299 | 300 | test('arrays', () => { 301 | c.expectParsedProgram(` 302 | float frequencies[3]; 303 | uniform vec4 lightPosition[4]; 304 | struct light { int a; }; 305 | light lights[]; 306 | const int numLights = 2; 307 | light lights2[numLights]; 308 | 309 | buffer b { 310 | float u[]; 311 | vec4 v[]; 312 | } name[3]; 313 | 314 | // Array initializers 315 | float array[3] = float[3](1.0, 2.0, 3.0); 316 | float array2[3] = float[](1.0, 2.0, 3.0); 317 | 318 | // Function with array as return type 319 | float[5] foo() { } 320 | `); 321 | }); 322 | 323 | test('initializer list', () => { 324 | c.expectParsedProgram(` 325 | vec4 a[3][2] = { 326 | vec4[2](vec4(0.0), vec4(1.0)), 327 | vec4[2](vec4(0.0), vec4(1.0)), 328 | vec4[2](vec4(0.0), vec4(1.0)) 329 | }; 330 | `); 331 | }); 332 | 333 | test('subroutines', () => { 334 | c.expectParsedProgram(` 335 | subroutine vec4 colorRedBlue(); 336 | 337 | // option 1 338 | subroutine (colorRedBlue ) vec4 redColor() { 339 | return vec4(1.0, 0.0, 0.0, 1.0); 340 | } 341 | 342 | // // option 2 343 | subroutine (colorRedBlue ) vec4 blueColor() { 344 | return vec4(0.0, 0.0, 1.0, 1.0); 345 | } 346 | `); 347 | }); 348 | 349 | test('Locations with location disabled', () => { 350 | const src = `void main() {}`; 351 | const ast = c.parseSrc(src); // default argument is no location information 352 | expect(ast.program[0].location).toBe(undefined); 353 | expect(ast.scopes[0].location).toBe(undefined); 354 | }); 355 | 356 | test('built-in function names should be identified as keywords', () => { 357 | console.warn = jest.fn(); 358 | 359 | const src = ` 360 | void main() { 361 | void x = texture2D(); 362 | }`; 363 | const ast = c.parseSrc(src); 364 | 365 | // Built-ins should not appear in scope 366 | expect(ast.scopes[0].functions).not.toHaveProperty('texture2D'); 367 | expect(ast.scopes[1].functions).not.toHaveProperty('texture2D'); 368 | 369 | let call: FunctionCallNode; 370 | visit(ast, { 371 | function_call: { 372 | enter: (path) => { 373 | call = path.node; 374 | }, 375 | }, 376 | }); 377 | 378 | // Builtins like texture2D should be recognized as a identifier since that's 379 | // how user defined functions are treated 380 | expect(call!.identifier.type).toBe('identifier'); 381 | 382 | // Should not warn about built in function call being undefined 383 | expect(console.warn).not.toHaveBeenCalled(); 384 | }); 385 | 386 | test('Parser locations', () => { 387 | const src = `// Some comment 388 | void main() { 389 | float x = 1.0; 390 | 391 | { 392 | float x = 1.0; 393 | } 394 | }`; 395 | const ast = c.parseSrc(src, { includeLocation: true }); 396 | // The main fn location should start at "void" 397 | expect(ast.program[0].location).toStrictEqual({ 398 | start: { line: 2, column: 1, offset: 16 }, 399 | end: { line: 8, column: 2, offset: 76 }, 400 | }); 401 | 402 | // The global scope is the entire program 403 | expect(ast.scopes[0].location).toStrictEqual({ 404 | start: { line: 1, column: 1, offset: 0 }, 405 | end: { line: 8, column: 2, offset: 76 }, 406 | }); 407 | 408 | // The scope created by the main fn should start at the open paren of the fn 409 | // header, because fn scopes include fn arguments 410 | expect(ast.scopes[1].location).toStrictEqual({ 411 | start: { line: 2, column: 10, offset: 25 }, 412 | end: { line: 8, column: 1, offset: 75 }, 413 | }); 414 | 415 | // The inner compound statement { scope } 416 | expect(ast.scopes[2].location).toStrictEqual({ 417 | start: { line: 5, column: 3, offset: 50 }, 418 | end: { line: 7, column: 3, offset: 73 }, 419 | }); 420 | }); 421 | 422 | test('fails on error', () => { 423 | expect(() => 424 | c.parse( 425 | `float a; 426 | float a;`, 427 | { failOnWarn: true } 428 | ) 429 | ).toThrow(/duplicate variable declaration: "a"/); 430 | }); 431 | 432 | test('exotic precision statements', () => { 433 | // Regression to test for upper/loweracse typos in specific keywords 434 | expect( 435 | c.parse(`precision highp sampler2DArrayShadow;`).program[0].declaration 436 | .specifier.specifier.token 437 | ).toBe('sampler2DArrayShadow'); 438 | expect( 439 | c.parse(`precision highp sampler2DRectShadow;`).program[0].declaration 440 | .specifier.specifier.token 441 | ).toBe('sampler2DRectShadow'); 442 | }); 443 | 444 | test('warns when grammar stage is unknown', () => { 445 | const consoleWarnMock = jest 446 | .spyOn(console, 'warn') 447 | .mockImplementation(() => {}); 448 | 449 | // we don't know if this is vertex or fragment so it should warn 450 | c.parseSrc(` 451 | void main() { 452 | gl_Position = vec4(0.0); 453 | } 454 | `); 455 | 456 | expect(consoleWarnMock).toHaveBeenCalled(); 457 | consoleWarnMock.mockRestore(); 458 | }); 459 | 460 | test('does not warn on built in stage variable', () => { 461 | const consoleWarnMock = jest 462 | .spyOn(console, 'warn') 463 | .mockImplementation(() => {}); 464 | 465 | c.parseSrc( 466 | ` 467 | void main() { 468 | gl_Position = vec4(0.0); 469 | } 470 | `, 471 | { stage: 'vertex' } 472 | ); 473 | 474 | expect(consoleWarnMock).not.toHaveBeenCalled(); 475 | consoleWarnMock.mockRestore(); 476 | }); 477 | 478 | test('does not warn on built in either stage variable', () => { 479 | const consoleWarnMock = jest 480 | .spyOn(console, 'warn') 481 | .mockImplementation(() => {}); 482 | 483 | c.parseSrc( 484 | ` 485 | void main() { 486 | gl_Position = vec4(0.0); 487 | gl_FragColor = vec4(0.0); 488 | } 489 | `, 490 | { stage: 'either' } 491 | ); 492 | 493 | expect(consoleWarnMock).not.toHaveBeenCalled(); 494 | consoleWarnMock.mockRestore(); 495 | }); 496 | 497 | test('warn on variable from wrong stage', () => { 498 | const consoleWarnMock = jest 499 | .spyOn(console, 'warn') 500 | .mockImplementation(() => {}); 501 | 502 | c.parseSrc( 503 | ` 504 | void main() { 505 | gl_Position = vec4(0.0); 506 | gl_FragColor = vec4(0.0); 507 | } 508 | `, 509 | { stage: 'fragment' } 510 | ); 511 | 512 | expect(consoleWarnMock).toHaveBeenCalled(); 513 | consoleWarnMock.mockRestore(); 514 | }); 515 | -------------------------------------------------------------------------------- /src/preprocessor/preprocessor-grammar.pegjs: -------------------------------------------------------------------------------- 1 | /** 2 | * This grammar is based on: 3 | * Khronos Shading Language Version 4.60.7 4 | * https://www.khronos.org/registry/OpenGL/specs/gl/GLSLangSpec.4.60.pdf 5 | * 6 | * And I used some Microsoft preprocessor documentation for the grammar base: 7 | * https://docs.microsoft.com/en-us/cpp/preprocessor/grammar-summary-c-cpp?view=msvc-160 8 | */ 9 | 10 | {{ 11 | import { 12 | unescapeSrc 13 | } from './preprocessor.js'; 14 | }} 15 | 16 | { 17 | // Remove escaped newlines before parsing 18 | input = unescapeSrc(input); 19 | 20 | const node = (type, attrs) => ({ 21 | type, 22 | ...attrs 23 | }); 24 | 25 | // Filter out "empty" elements from an array 26 | const xnil = (...args) => args.flat().filter(e => 27 | e !== undefined && e !== null && e !== '' && e.length !== 0 28 | ) 29 | 30 | const ifOnly = arr => arr.length > 1 ? arr : arr[0]; 31 | 32 | // Remove empty elements and return value if only 1 element remains 33 | const collapse = (...args) => ifOnly(xnil(args.flat())); 34 | 35 | // Create a left associative tree of nodes 36 | const leftAssociate = (...nodes) => 37 | nodes.flat().reduce((current, [operator, expr]) => ({ 38 | type: 'binary', 39 | operator: operator, 40 | left: current, 41 | right: expr 42 | })); 43 | } 44 | 45 | start = program 46 | 47 | program = 48 | program:text_or_control_lines 49 | wsEnd:_? { 50 | return node('program', { program: program.blocks, wsEnd }); 51 | } 52 | 53 | // GLSL only allows for integers in constant expressions 54 | INTCONSTANT = token:integer_constant _:_? { return node('int_constant', { token, wsEnd: _ }); } 55 | 56 | LEFT_OP = token:"<<" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 57 | RIGHT_OP = token:">>" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 58 | LE_OP = token:"<=" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 59 | GE_OP = token:">=" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 60 | EQ_OP = token:"==" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 61 | NE_OP = token:"!=" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 62 | AND_OP = token:"&&" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 63 | OR_OP = token:"||" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 64 | 65 | LEFT_PAREN = token:"(" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 66 | RIGHT_PAREN = token:")" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 67 | COMMA = token:"," _:_? { return node('literal', { literal: token, wsEnd: _ }); } 68 | BANG = token:"!" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 69 | DASH = token:"-" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 70 | TILDE = token:"~" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 71 | PLUS = token:"+" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 72 | STAR = token:"*" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 73 | SLASH = token:"/" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 74 | PERCENT = token:"%" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 75 | LEFT_ANGLE = token:"<" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 76 | RIGHT_ANGLE = token:">" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 77 | VERTICAL_BAR = token:"|" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 78 | CARET = token:"^" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 79 | AMPERSAND = token:"&" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 80 | COLON = token:":" _:_? { return node('literal', { literal: token, wsEnd: _ }); } 81 | 82 | // From the grammar spec: "Each number sign (#) can be preceded in its line only 83 | // by spaces or horizontal tabs. It may also be followed by spaces and 84 | // horizontal tabs, preceding the directive." 85 | HASH = $(token:"#") whitespace:_? { return "#"; } 86 | 87 | DEFINE = wsStart:_? token:$(HASH "define") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 88 | INCLUDE = wsStart:_? token:$(HASH "include") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 89 | LINE = wsStart:_? token:$(HASH "line") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 90 | UNDEF = wsStart:_? token:$(HASH "undef") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 91 | ERROR = wsStart:_? token:$(HASH "error") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 92 | PRAGMA = wsStart:_? token:$(HASH "pragma") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 93 | DEFINED = wsStart:_? token:"defined" wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 94 | IF = wsStart:_? token:$(HASH "if") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 95 | IFDEF = wsStart:_? token:$(HASH "ifdef") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 96 | IFNDEF = wsStart:_? token:$(HASH "ifndef") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 97 | ELIF = wsStart:_? token:$(HASH "elif") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 98 | ELSE = wsStart:_? token:$(HASH "else") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 99 | ENDIF = wsStart:_? token:$(HASH "endif") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 100 | VERSION = wsStart:_? token:$(HASH "version") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 101 | EXTENSION = wsStart:_? token:$(HASH "extension") wsEnd:terminal { return node('literal', { literal: token, wsStart, wsEnd }); } 102 | 103 | IDENTIFIER = identifier:$([A-Za-z_] [A-Za-z_0-9]*) _:_? { return node('identifier', { identifier, wsEnd: _ }); } 104 | IDENTIFIER_NO_WS = identifier:$([A-Za-z_] [A-Za-z_0-9]*) { return node('identifier', { identifier }); } 105 | 106 | // Integers 107 | integer_constant "number" 108 | = $(decimal_constant integer_suffix?) 109 | / $(octal_constant integer_suffix?) 110 | / $(hexadecimal_constant integer_suffix?) 111 | 112 | integer_suffix = [uU] 113 | 114 | // Collapsing the above becomes 115 | decimal_constant = $([1-9] digit*) 116 | octal_constant = "0" [0-7]* 117 | hexadecimal_constant = "0" [xX] [0-9a-fA-F]* 118 | 119 | digit = [0-9] 120 | 121 | // Basically any valid source code 122 | text_or_control_lines = 123 | blocks:( 124 | control_line 125 | / text:text+ { 126 | return node('text', { text: text.join('') }); 127 | } 128 | )+ { 129 | return node('segment', { blocks }); 130 | } 131 | 132 | // Any preprocessor or directive line 133 | control_line "control line" 134 | = conditional 135 | / line:( 136 | define:DEFINE 137 | identifier:IDENTIFIER_NO_WS 138 | lp:LEFT_PAREN 139 | args:( 140 | head:IDENTIFIER 141 | tail:(COMMA IDENTIFIER)* { 142 | return [head, ...tail.flat()]; 143 | } 144 | )? 145 | rp:RIGHT_PAREN 146 | body:token_string? { 147 | return node('define_arguments', { define, identifier, lp, args: args || [], rp, body } ) 148 | } 149 | / define:DEFINE identifier:IDENTIFIER body:token_string? { 150 | return node('define', { define, identifier, body } ) 151 | } 152 | / line:LINE value:$digit+ { 153 | return node('line', { line, value }); 154 | } 155 | / undef:UNDEF identifier:IDENTIFIER { 156 | return node('undef', { undef, identifier }); 157 | } 158 | / error:ERROR message:token_string { 159 | return node('error', { error, message }); 160 | } 161 | / pragma:PRAGMA body:token_string { 162 | return node('pragma', { pragma, body }); 163 | } 164 | // The cpp preprocessor spec doesn't have version in it, I added it. 165 | // "profile" is defined on page 14 of GLSL spec 166 | / version:VERSION value:integer_constant profile:token_string? { 167 | return node('version', { version, value, profile }); 168 | } 169 | / extension:EXTENSION name:IDENTIFIER colon:COLON behavior:token_string { 170 | return node('extension', { extension, name, colon, behavior }); 171 | } 172 | ) 173 | wsEnd:[\n\r]? { 174 | return { ...line, wsEnd }; 175 | } 176 | 177 | // Any series of characters on the same line, 178 | // for example "abc 123" in "#define A abc 123" 179 | token_string "token string" = $([^\n\r]+) 180 | 181 | // Any non-control line. Ending newline for text is optional because program 182 | // might end on a non-newline 183 | text "text" = $(!(whitespace? "#") [^\n\r]+ [\n\r]? / [\n\r]) 184 | 185 | conditional 186 | = ifPart:( 187 | ifLine:if_line 188 | wsEnd:[\n\r] 189 | body:text_or_control_lines? { 190 | return { ...ifLine, body, wsEnd }; 191 | } 192 | ) 193 | elseIfParts:( 194 | token:ELIF 195 | expression:constant_expression 196 | wsEnd: [\n\r] 197 | elseIfBody:text_or_control_lines? { 198 | return node('elseif', { token, expression, wsEnd, body: elseIfBody }); 199 | } 200 | )* 201 | elsePart:( 202 | token:ELSE 203 | wsEnd: [\n\r] 204 | elseBody:text_or_control_lines? { 205 | return node('else', { token, wsEnd, body: elseBody }); 206 | } 207 | )? 208 | endif:ENDIF 209 | wsEnd:[\n\r]? { // optional because the program can end with endif 210 | return node('conditional', { ifPart, elseIfParts, elsePart, endif, wsEnd, }); 211 | } 212 | 213 | if_line "if" 214 | = token:IFDEF identifier:IDENTIFIER { 215 | return node('ifdef', { token, identifier }); 216 | } 217 | / token:IFNDEF identifier:IDENTIFIER { 218 | return node('ifndef', { token, identifier }); 219 | } 220 | / token:IF expression:constant_expression? { 221 | return node('if', { token, expression }); 222 | } 223 | 224 | // The following encodes the operator precedence for preprocessor #if 225 | // expressions, as defined on page 12 of 226 | // https://www.khronos.org/registry/OpenGL/specs/gl/GLSLangSpec.4.60.pdf 227 | 228 | primary_expression "primary expression" 229 | = INTCONSTANT 230 | / lp:LEFT_PAREN expression:constant_expression rp:RIGHT_PAREN { 231 | return node('group', { lp, expression, rp }); 232 | } 233 | / IDENTIFIER 234 | 235 | unary_expression "unary expression" 236 | // "defined" is a unary operator, it can appear with optional parens. I'm not 237 | // sure if it makes sense to have it in the unary_expression section 238 | = operator:DEFINED lp:LEFT_PAREN? identifier:IDENTIFIER rp:RIGHT_PAREN? { 239 | return node('unary_defined', { operator, lp, identifier, rp, }); 240 | } 241 | / operator:(PLUS / DASH / BANG / TILDE) 242 | expression:unary_expression { 243 | return node('unary', { operator, expression }); 244 | } 245 | / primary_expression 246 | 247 | multiplicative_expression "multiplicative expression" 248 | = head:unary_expression 249 | tail:( 250 | op:(STAR / SLASH / PERCENT) 251 | expr:unary_expression 252 | )* { 253 | return leftAssociate(head, tail); 254 | } 255 | 256 | additive_expression "additive expression" 257 | = head:multiplicative_expression 258 | tail:( 259 | op:(PLUS / DASH) 260 | expr:multiplicative_expression 261 | )* { 262 | return leftAssociate(head, tail); 263 | } 264 | 265 | shift_expression "shift expression" 266 | = head:additive_expression 267 | tail:( 268 | op:(RIGHT_OP / LEFT_OP) 269 | expr:additive_expression 270 | )* { 271 | return leftAssociate(head, tail); 272 | } 273 | 274 | relational_expression "relational expression" 275 | = head:shift_expression 276 | tail:( 277 | op:(LE_OP / GE_OP / LEFT_ANGLE / RIGHT_ANGLE) 278 | expr:shift_expression 279 | )* { 280 | return leftAssociate(head, tail); 281 | } 282 | 283 | equality_expression "equality expression" 284 | = head:relational_expression 285 | tail:( 286 | op:(EQ_OP / NE_OP) 287 | expr:relational_expression 288 | )* { 289 | return leftAssociate(head, tail); 290 | } 291 | 292 | and_expression "and expression" 293 | = head:equality_expression 294 | tail:( 295 | op:AMPERSAND 296 | expr:equality_expression 297 | )* { 298 | return leftAssociate(head, tail); 299 | } 300 | 301 | exclusive_or_expression "exclusive or expression" 302 | = head:and_expression 303 | tail:( 304 | op:CARET 305 | expr:and_expression 306 | )* { 307 | return leftAssociate(head, tail); 308 | } 309 | 310 | inclusive_or_expression "inclusive or expression" 311 | = head:exclusive_or_expression 312 | tail:( 313 | op:VERTICAL_BAR 314 | expr:exclusive_or_expression 315 | )* { 316 | return leftAssociate(head, tail); 317 | } 318 | 319 | logical_and_expression "logical and expression" 320 | = head:inclusive_or_expression 321 | tail:( 322 | op:AND_OP 323 | expr:inclusive_or_expression 324 | )* { 325 | return leftAssociate(head, tail); 326 | } 327 | 328 | logical_or_expression "logical or expression" 329 | = head:logical_and_expression 330 | tail:( 331 | op:OR_OP 332 | expr:logical_and_expression 333 | )* { 334 | return leftAssociate(head, tail); 335 | } 336 | 337 | // I added this as a maybe entry point to expressions 338 | constant_expression "constant expression" = logical_or_expression 339 | 340 | // The whitespace is optional so that we can put comments immediately after 341 | // terminals, like void/* comment */ 342 | // The ending whitespace is so that linebreaks can happen after comments 343 | _ "whitespace or comment" = w:whitespace? rest:(comment whitespace?)* { 344 | return collapse(w, rest); 345 | } 346 | 347 | comment 348 | = single_comment 349 | // Intention is to handle any type of comment case. A multiline comment 350 | // can be followed by more multiline comments, or a single comment, and 351 | // collapse everything into one array 352 | / a:multiline_comment d:( 353 | x:whitespace cc:comment { return xnil(x, cc); } 354 | )* { return xnil(a, d.flat()); } 355 | 356 | single_comment = $('//' [^\n\r]*) 357 | multiline_comment = $("/*" inner:(!"*/" i:. { return i; })* "*/") 358 | 359 | whitespace "whitespace" = $[ \t]+ 360 | 361 | terminal = ![A-Za-z_0-9] _:_? { return _; } 362 | -------------------------------------------------------------------------------- /src/ast/ast-types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is written by hand, to map to the parser expression results in 3 | * parser/glsl-grammar.pegjs. It very, very likely contains errors. I put in 4 | * *AstNode* types where I was lazy or didn't know the core type. 5 | */ 6 | 7 | import { Scope } from '../parser/scope.js'; 8 | 9 | // Valid top level program lines 10 | export type ProgramStatement = 11 | | PreprocessorNode 12 | | DeclarationStatementNode 13 | | FunctionNode; 14 | 15 | // The overall result of parsing, which incldues the AST and scopes 16 | export interface Program { 17 | type: 'program'; 18 | program: ProgramStatement[]; 19 | scopes: Scope[]; 20 | wsStart?: string; 21 | wsEnd?: string; 22 | } 23 | 24 | // Optional source code location info, set by { includeLocation: true } 25 | export type LocationInfo = { offset: number; line: number; column: number }; 26 | 27 | export type LocationObject = { 28 | start: LocationInfo; 29 | end: LocationInfo; 30 | }; 31 | 32 | export interface BaseNode { 33 | location?: LocationObject; 34 | } 35 | 36 | export type Whitespace = string | string[]; 37 | 38 | // Types reused across nodes 39 | export type TypeQualifiers = ( 40 | | KeywordNode 41 | | SubroutineQualifierNode 42 | | LayoutQualifierNode 43 | )[]; 44 | export type Semicolon = LiteralNode<';'>; 45 | export type Comma = LiteralNode<','>; 46 | 47 | // This is my best guess at what can be in an expression. It's probably wrong! 48 | export type Expression = 49 | | LiteralNode 50 | | KeywordNode 51 | | IdentifierNode 52 | | TypeNameNode 53 | | ArraySpecifierNode 54 | | AssignmentNode 55 | | BinaryNode 56 | | BoolConstantNode 57 | | ConditionExpressionNode 58 | | DefaultCaseNode 59 | | DoubleConstantNode 60 | | FieldSelectionNode 61 | | FloatConstantNode 62 | | FullySpecifiedTypeNode 63 | | FunctionCallNode 64 | | GroupNode 65 | | InitializerListNode 66 | | IntConstantNode 67 | | PostfixNode 68 | | PreprocessorNode 69 | | QuantifiedIdentifierNode 70 | | QuantifierNode 71 | | SwitchCaseNode 72 | | TernaryNode 73 | | TypeSpecifierNode 74 | | UintConstantNode 75 | | UnaryNode; 76 | 77 | export interface LiteralNode extends BaseNode { 78 | type: 'literal'; 79 | literal: Literal; 80 | whitespace: Whitespace; 81 | } 82 | 83 | export interface KeywordNode extends BaseNode { 84 | type: 'keyword'; 85 | token: Token; 86 | whitespace: Whitespace; 87 | } 88 | 89 | export interface IdentifierNode extends BaseNode { 90 | type: 'identifier'; 91 | identifier: string; 92 | whitespace: Whitespace; 93 | } 94 | 95 | export interface TypeNameNode extends BaseNode { 96 | type: 'type_name'; 97 | identifier: string; 98 | whitespace: Whitespace; 99 | } 100 | 101 | export interface ArraySpecifierNode extends BaseNode { 102 | type: 'array_specifier'; 103 | lb: LiteralNode<'['>; 104 | expression: Expression; 105 | rb: LiteralNode<']'>; 106 | } 107 | 108 | export interface AssignmentNode extends BaseNode { 109 | type: 'assignment'; 110 | left: AstNode; 111 | operator: LiteralNode< 112 | '=' | '*=' | '/=' | '%=' | '+=' | '-=' | '<<="' | '>>=' | '&=' | '^=' | '|=' 113 | >; 114 | right: AstNode; 115 | } 116 | 117 | export interface BinaryNode extends BaseNode { 118 | type: 'binary'; 119 | operator: LiteralNode; 120 | left: AstNode; 121 | right: AstNode; 122 | } 123 | 124 | export interface BoolConstantNode extends BaseNode { 125 | type: 'bool_constant'; 126 | token: string; 127 | whitespace: Whitespace; 128 | } 129 | 130 | export interface BreakStatementNode extends BaseNode { 131 | type: 'break_statement'; 132 | break: KeywordNode<'break'>; 133 | semi: Semicolon; 134 | } 135 | 136 | export interface CompoundStatementNode extends BaseNode { 137 | type: 'compound_statement'; 138 | lb: LiteralNode<'['>; 139 | statements: AstNode[]; 140 | rb: LiteralNode<']'>; 141 | } 142 | 143 | export interface ConditionExpressionNode extends BaseNode { 144 | type: 'condition_expression'; 145 | specified_type: FullySpecifiedTypeNode; 146 | identifier: IdentifierNode; 147 | operator: LiteralNode; 148 | initializer: InitializerListNode; 149 | } 150 | 151 | export interface ContinueStatementNode extends BaseNode { 152 | type: 'continue_statement'; 153 | continue: KeywordNode<'continue'>; 154 | semi: Semicolon; 155 | } 156 | 157 | export interface DeclarationStatementNode extends BaseNode { 158 | type: 'declaration_statement'; 159 | declaration: 160 | | PrecisionNode 161 | | InterfaceDeclaratorNode 162 | | QualifierDeclaratorNode 163 | | DeclaratorListNode 164 | | FunctionHeaderNode; 165 | semi: Semicolon; 166 | } 167 | 168 | export interface DeclarationNode extends BaseNode { 169 | type: 'declaration'; 170 | identifier: IdentifierNode; 171 | quantifier: ArraySpecifierNode[]; 172 | equal?: LiteralNode<'='>; 173 | initializer?: AstNode; 174 | } 175 | 176 | export interface DeclaratorListNode extends BaseNode { 177 | type: 'declarator_list'; 178 | specified_type: FullySpecifiedTypeNode; 179 | declarations: DeclarationNode[]; 180 | commas: Comma[]; 181 | } 182 | 183 | export interface DefaultCaseNode extends BaseNode { 184 | type: 'default_case'; 185 | statements: []; 186 | default: KeywordNode<'default'>; 187 | colon: LiteralNode<':'>; 188 | } 189 | 190 | export interface DiscardStatementNode extends BaseNode { 191 | type: 'discard_statement'; 192 | discard: KeywordNode<'discard'>; 193 | semi: Semicolon; 194 | } 195 | 196 | export interface DoStatementNode extends BaseNode { 197 | type: 'do_statement'; 198 | do: KeywordNode<'do'>; 199 | body: AstNode; 200 | while: KeywordNode<'while'>; 201 | lp: LiteralNode<'('>; 202 | expression: Expression; 203 | rp: LiteralNode<')'>; 204 | semi: Semicolon; 205 | } 206 | 207 | export interface DoubleConstantNode extends BaseNode { 208 | type: 'double_constant'; 209 | token: string; 210 | whitespace: Whitespace; 211 | } 212 | 213 | export interface ExpressionStatementNode extends BaseNode { 214 | type: 'expression_statement'; 215 | expression: Expression; 216 | semi: Semicolon; 217 | } 218 | 219 | export interface FieldSelectionNode extends BaseNode { 220 | type: 'field_selection'; 221 | dot: LiteralNode; 222 | selection: LiteralNode; 223 | } 224 | 225 | export interface FloatConstantNode extends BaseNode { 226 | type: 'float_constant'; 227 | token: string; 228 | whitespace: Whitespace; 229 | } 230 | 231 | export type SimpleStatement = 232 | | ContinueStatementNode 233 | | BreakStatementNode 234 | | ReturnStatementNode 235 | | DiscardStatementNode 236 | | DeclarationStatementNode 237 | | ExpressionStatementNode 238 | | IfStatementNode 239 | | SwitchStatementNode 240 | | WhileStatementNode; 241 | 242 | export interface ForStatementNode extends BaseNode { 243 | type: 'for_statement'; 244 | for: KeywordNode<'for'>; 245 | body: CompoundStatementNode | SimpleStatement; 246 | lp: LiteralNode<'('>; 247 | init: AstNode; 248 | initSemi: Semicolon; 249 | condition: ConditionExpressionNode; 250 | conditionSemi: Semicolon; 251 | operation: AstNode; 252 | rp: LiteralNode<')'>; 253 | } 254 | 255 | export interface FullySpecifiedTypeNode extends BaseNode { 256 | type: 'fully_specified_type'; 257 | qualifiers?: TypeQualifiers; 258 | specifier: TypeSpecifierNode; 259 | } 260 | 261 | export interface FunctionNode extends BaseNode { 262 | type: 'function'; 263 | prototype: FunctionPrototypeNode; 264 | body: CompoundStatementNode; 265 | } 266 | 267 | export interface FunctionCallNode extends BaseNode { 268 | type: 'function_call'; 269 | identifier: IdentifierNode | TypeSpecifierNode | PostfixNode; 270 | lp: LiteralNode<'('>; 271 | args: AstNode[]; 272 | rp: LiteralNode<')'>; 273 | } 274 | 275 | export interface FunctionHeaderNode extends BaseNode { 276 | type: 'function_header'; 277 | returnType: FullySpecifiedTypeNode; 278 | name: IdentifierNode; 279 | lp: LiteralNode<'('>; 280 | } 281 | 282 | export interface FunctionPrototypeNode extends BaseNode { 283 | type: 'function_prototype'; 284 | header: FunctionHeaderNode; 285 | parameters: ParameterDeclarationNode[]; 286 | commas: Comma[]; 287 | rp: LiteralNode<')'>; 288 | } 289 | 290 | export interface GroupNode extends BaseNode { 291 | type: 'group'; 292 | lp: LiteralNode<'('>; 293 | expression: Expression; 294 | rp: LiteralNode<')'>; 295 | } 296 | 297 | export interface IfStatementNode extends BaseNode { 298 | type: 'if_statement'; 299 | if: KeywordNode<'if'>; 300 | body: AstNode; 301 | lp: LiteralNode<'('>; 302 | condition: AstNode; 303 | rp: LiteralNode<')'>; 304 | else: AstNode[]; 305 | } 306 | 307 | export interface InitializerListNode extends BaseNode { 308 | type: 'initializer_list'; 309 | lb: LiteralNode<'['>; 310 | initializers: AstNode[]; 311 | commas: Comma[]; 312 | rb: LiteralNode<']'>; 313 | } 314 | 315 | export interface IntConstantNode extends BaseNode { 316 | type: 'int_constant'; 317 | token: string; 318 | whitespace: Whitespace; 319 | } 320 | 321 | export interface InterfaceDeclaratorNode extends BaseNode { 322 | type: 'interface_declarator'; 323 | qualifiers: TypeQualifiers; 324 | interface_type: IdentifierNode; 325 | lp: LiteralNode<'('>; 326 | declarations: StructDeclarationNode[]; 327 | rp: LiteralNode<')'>; 328 | identifier?: QuantifiedIdentifierNode; 329 | } 330 | 331 | export interface LayoutQualifierIdNode extends BaseNode { 332 | type: 'layout_qualifier_id'; 333 | identifier: IdentifierNode; 334 | operator: LiteralNode; 335 | expression: Expression; 336 | } 337 | 338 | export interface LayoutQualifierNode extends BaseNode { 339 | type: 'layout_qualifier'; 340 | layout: KeywordNode<'layout'>; 341 | lp: LiteralNode<'('>; 342 | qualifiers: LayoutQualifierIdNode[]; 343 | commas: Comma[]; 344 | rp: LiteralNode<')'>; 345 | } 346 | 347 | export interface ParameterDeclarationNode extends BaseNode { 348 | type: 'parameter_declaration'; 349 | qualifier: KeywordNode[]; 350 | specifier: TypeSpecifierNode; 351 | identifier: IdentifierNode; 352 | quantifier: ArraySpecifierNode[]; 353 | } 354 | 355 | export interface PostfixNode extends BaseNode { 356 | type: 'postfix'; 357 | expression: Expression; 358 | postfix: AstNode; 359 | } 360 | 361 | export interface PrecisionNode extends BaseNode { 362 | type: 'precision'; 363 | prefix: KeywordNode<'prefix'>; 364 | qualifier: KeywordNode<'highp' | 'mediump' | 'lowp'>; 365 | specifier: TypeSpecifierNode; 366 | } 367 | 368 | export interface PreprocessorNode extends BaseNode { 369 | type: 'preprocessor'; 370 | line: string; 371 | _: string | string[]; 372 | } 373 | 374 | export interface QualifierDeclaratorNode extends BaseNode { 375 | type: 'qualifier_declarator'; 376 | qualifiers: TypeQualifiers; 377 | declarations: IdentifierNode[]; 378 | commas: Comma[]; 379 | } 380 | 381 | export interface QuantifiedIdentifierNode extends BaseNode { 382 | type: 'quantified_identifier'; 383 | identifier: IdentifierNode; 384 | quantifier: ArraySpecifierNode[]; 385 | } 386 | 387 | export interface QuantifierNode extends BaseNode { 388 | type: 'quantifier'; 389 | lb: LiteralNode<'['>; 390 | expression: Expression; 391 | rb: LiteralNode<']'>; 392 | } 393 | 394 | export interface ReturnStatementNode extends BaseNode { 395 | type: 'return_statement'; 396 | return: KeywordNode<'return'>; 397 | expression: Expression; 398 | semi: Semicolon; 399 | } 400 | 401 | export interface StructNode extends BaseNode { 402 | type: 'struct'; 403 | lb: LiteralNode<'['>; 404 | declarations: StructDeclarationNode[]; 405 | rb: LiteralNode<']'>; 406 | struct: KeywordNode<'struct'>; 407 | typeName: TypeNameNode; 408 | } 409 | 410 | export interface StructDeclarationNode extends BaseNode { 411 | type: 'struct_declaration'; 412 | declaration: StructDeclaratorNode; 413 | semi: Semicolon; 414 | } 415 | 416 | export interface StructDeclaratorNode extends BaseNode { 417 | type: 'struct_declarator'; 418 | specified_type: FullySpecifiedTypeNode; 419 | declarations: QuantifiedIdentifierNode[]; 420 | commas: Comma[]; 421 | } 422 | 423 | export interface SubroutineQualifierNode extends BaseNode { 424 | type: 'subroutine_qualifier'; 425 | subroutine: KeywordNode<'subroutine'>; 426 | lp: LiteralNode<'('>; 427 | type_names: TypeNameNode[]; 428 | commas: Comma[]; 429 | rp: LiteralNode<')'>; 430 | } 431 | 432 | export interface SwitchCaseNode extends BaseNode { 433 | type: 'switch_case'; 434 | statements: []; 435 | case: KeywordNode<'case'>; 436 | test: AstNode; 437 | colon: LiteralNode<':'>; 438 | } 439 | 440 | export interface SwitchStatementNode extends BaseNode { 441 | type: 'switch_statement'; 442 | switch: KeywordNode<'switch'>; 443 | lp: LiteralNode<'('>; 444 | expression: Expression; 445 | rp: LiteralNode<')'>; 446 | lb: LiteralNode<'['>; 447 | cases: AstNode[]; 448 | rb: LiteralNode<']'>; 449 | } 450 | 451 | export interface TernaryNode extends BaseNode { 452 | type: 'ternary'; 453 | expression: Expression; 454 | question: LiteralNode<'?'>; 455 | left: AstNode; 456 | right: AstNode; 457 | colon: LiteralNode<':'>; 458 | } 459 | 460 | export interface TypeSpecifierNode extends BaseNode { 461 | type: 'type_specifier'; 462 | specifier: KeywordNode | IdentifierNode | StructNode | TypeNameNode; 463 | quantifier: ArraySpecifierNode[] | null; 464 | } 465 | 466 | export interface UintConstantNode extends BaseNode { 467 | type: 'uint_constant'; 468 | token: string; 469 | whitespace: Whitespace; 470 | } 471 | 472 | export interface UnaryNode extends BaseNode { 473 | type: 'unary'; 474 | operator: LiteralNode<'++' | '--' | '+' | '-' | '!' | '~'>; 475 | expression: Expression; 476 | } 477 | 478 | export interface WhileStatementNode extends BaseNode { 479 | type: 'while_statement'; 480 | while: KeywordNode<'while'>; 481 | lp: LiteralNode<'('>; 482 | condition: AstNode; 483 | rp: LiteralNode<')'>; 484 | body: AstNode; 485 | } 486 | 487 | export type AstNode = 488 | | LiteralNode 489 | | KeywordNode 490 | | IdentifierNode 491 | | TypeNameNode 492 | | ArraySpecifierNode 493 | | AssignmentNode 494 | | BinaryNode 495 | | BoolConstantNode 496 | | BreakStatementNode 497 | | CompoundStatementNode 498 | | ConditionExpressionNode 499 | | ContinueStatementNode 500 | | DeclarationStatementNode 501 | | DeclarationNode 502 | | DeclaratorListNode 503 | | DefaultCaseNode 504 | | DiscardStatementNode 505 | | DoStatementNode 506 | | DoubleConstantNode 507 | | ExpressionStatementNode 508 | | FieldSelectionNode 509 | | FloatConstantNode 510 | | ForStatementNode 511 | | FullySpecifiedTypeNode 512 | | FunctionNode 513 | | FunctionCallNode 514 | | FunctionHeaderNode 515 | | FunctionPrototypeNode 516 | | GroupNode 517 | | IfStatementNode 518 | | InitializerListNode 519 | | IntConstantNode 520 | | InterfaceDeclaratorNode 521 | | LayoutQualifierIdNode 522 | | LayoutQualifierNode 523 | | ParameterDeclarationNode 524 | | PostfixNode 525 | | PrecisionNode 526 | | PreprocessorNode 527 | | QualifierDeclaratorNode 528 | | QuantifiedIdentifierNode 529 | | QuantifierNode 530 | | ReturnStatementNode 531 | | StructNode 532 | | StructDeclarationNode 533 | | StructDeclaratorNode 534 | | SubroutineQualifierNode 535 | | SwitchCaseNode 536 | | SwitchStatementNode 537 | | TernaryNode 538 | | TypeSpecifierNode 539 | | UintConstantNode 540 | | UnaryNode 541 | | WhileStatementNode; 542 | -------------------------------------------------------------------------------- /src/preprocessor/preprocessor.test.ts: -------------------------------------------------------------------------------- 1 | import util from 'util'; 2 | import { 3 | preprocessComments, 4 | preprocessAst, 5 | PreprocessorProgram, 6 | visitPreprocessedAst, 7 | } from './preprocessor.js'; 8 | import generate from './generator.js'; 9 | import { PreprocessorOptions } from './preprocessor.js'; 10 | import { GlslSyntaxError } from '../error.js'; 11 | 12 | import { buildPreprocessorParser } from '../parser/test-helpers.js'; 13 | 14 | let c!: ReturnType; 15 | let parse: (src: string, options?: PreprocessorOptions) => PreprocessorProgram; 16 | beforeAll(() => { 17 | c = buildPreprocessorParser(); 18 | parse = c.parse; 19 | }); 20 | 21 | const debugProgram = (program: string): void => { 22 | debugAst(c.parse(program)); 23 | }; 24 | 25 | const debugAst = (ast: any) => { 26 | console.log(util.inspect(ast, false, null, true)); 27 | }; 28 | 29 | const expectParsedProgram = ( 30 | sourceGlsl: string, 31 | options?: PreprocessorOptions 32 | ) => { 33 | const ast = parse(sourceGlsl, options); 34 | const glsl = generate(ast); 35 | if (glsl !== sourceGlsl) { 36 | debugAst(ast); 37 | expect(glsl).toBe(sourceGlsl); 38 | } 39 | }; 40 | 41 | // test('pre test file', () => { 42 | // expectParsedProgram(fileContents('./preprocess-test-grammar.glsl')); 43 | // }); 44 | 45 | test('#preprocessComments', () => { 46 | // Should strip comments and replace single-line comments with a single space 47 | expect( 48 | preprocessComments(`// ccc 49 | /* cc */aaa/* cc */ 50 | /** 51 | * cccc 52 | */ 53 | bbb 54 | `) 55 | ).toBe(` 56 | aaa 57 | 58 | 59 | 60 | bbb 61 | `); 62 | }); 63 | 64 | test('preprocessor error', () => { 65 | let error: GlslSyntaxError | undefined; 66 | try { 67 | parse(`#if defined(#)`); 68 | } catch (e) { 69 | error = e as GlslSyntaxError; 70 | } 71 | 72 | expect(error).toBeInstanceOf(GlslSyntaxError); 73 | expect(error!.location.start.line).toBe(1); 74 | expect(error!.location.end.line).toBe(1); 75 | }); 76 | 77 | test('preprocessor ast', () => { 78 | expectParsedProgram(` 79 | #line 0 80 | #version 100 "hi" 81 | #define GL_es_profile 1 82 | #extension all : disable 83 | #error whoopsie 84 | #define A 1 85 | before if 86 | #if A == 1 || B == 2 87 | inside if 88 | #define A 89 | #elif A == 1 || defined B && C == 2 90 | float a; 91 | #elif A == 1 || defined(B) && C == 2 92 | float a; 93 | #define B 94 | #endif 95 | outside endif 96 | #pragma mypragma: something(else) 97 | final line after program 98 | `); 99 | }); 100 | 101 | test('directive whitespace', () => { 102 | const program = `# define X Y 103 | X`; 104 | 105 | const ast = parse(program); 106 | preprocessAst(ast); 107 | expect(generate(ast)).toBe(`Y`); 108 | }); 109 | 110 | test('nested expand macro', () => { 111 | const program = `#define X Y 112 | #define Y Z 113 | X`; 114 | 115 | const ast = parse(program); 116 | preprocessAst(ast); 117 | expect(generate(ast)).toBe(`Z`); 118 | }); 119 | 120 | test('binary evaluation', () => { 121 | const program = ` 122 | #if 1 + 1 > 0 123 | true 124 | #endif 125 | `; 126 | 127 | const ast = parse(program); 128 | preprocessAst(ast); 129 | expect(generate(ast)).toBe(` 130 | true 131 | `); 132 | }); 133 | 134 | test('define inside if/else is properly expanded when the if branch is chosen', () => { 135 | const program = ` 136 | #define MACRO 137 | #ifdef MACRO 138 | #define BRANCH a 139 | #else 140 | #define BRANCH b 141 | #endif 142 | BRANCH 143 | `; 144 | const ast = parse(program); 145 | preprocessAst(ast); 146 | expect(generate(ast)).toBe(` 147 | a 148 | `); 149 | }); 150 | 151 | test('define inside if/else is properly expanded when the else branch is chosen', () => { 152 | const program = ` 153 | #ifdef MACRO 154 | #define BRANCH a 155 | #else 156 | #define BRANCH b 157 | #endif 158 | BRANCH 159 | `; 160 | const ast = parse(program); 161 | preprocessAst(ast); 162 | expect(generate(ast)).toBe(` 163 | b 164 | `); 165 | }); 166 | 167 | test('ifdef inside else is properly expanded', () => { 168 | // Regression: Make sure #ifdef MACRO inside #else isn't expanded 169 | const program = ` 170 | #define MACRO 171 | #ifdef NOT_DEFINED 172 | false 173 | #else 174 | #ifdef MACRO 175 | ____true 176 | #endif 177 | #endif 178 | `; 179 | 180 | const ast = parse(program); 181 | preprocessAst(ast); 182 | expect(generate(ast)).toBe(` 183 | ____true 184 | `); 185 | }); 186 | 187 | test('macro without body becoms empty string', () => { 188 | // There is intentionally whitespace after MACRO to make sure it doesn't apply 189 | // to the expansion-to-nothing 190 | const program = ` 191 | #define MACRO 192 | fn(MACRO); 193 | `; 194 | 195 | const ast = parse(program); 196 | preprocessAst(ast); 197 | expect(generate(ast)).toBe(` 198 | fn(); 199 | `); 200 | }); 201 | 202 | test('if expression', () => { 203 | const program = ` 204 | #define A 205 | before if 206 | #if !defined(A) && (defined(B) && C == 2) 207 | inside first if 208 | #endif 209 | #if ((defined B && C == 2) || defined(A)) 210 | inside second if 211 | #endif 212 | after if 213 | `; 214 | 215 | const ast = parse(program); 216 | preprocessAst(ast); 217 | expect(generate(ast)).toBe(` 218 | before if 219 | inside second if 220 | after if 221 | `); 222 | }); 223 | 224 | test('evaluate if branch', () => { 225 | const program = ` 226 | #define A 227 | before if 228 | #if defined(A) 229 | inside if 230 | #endif 231 | after if 232 | `; 233 | 234 | const ast = parse(program); 235 | preprocessAst(ast); 236 | expect(generate(ast)).toBe(` 237 | before if 238 | inside if 239 | after if 240 | `); 241 | }); 242 | 243 | test('evaluate elseif branch', () => { 244 | const program = ` 245 | #define A 246 | before if 247 | #if defined(B) 248 | inside if 249 | #elif defined(A) 250 | inside elif 251 | #else 252 | else body 253 | #endif 254 | after if`; 255 | 256 | const ast = parse(program); 257 | preprocessAst(ast); 258 | expect(generate(ast)).toBe(` 259 | before if 260 | inside elif 261 | after if`); 262 | }); 263 | 264 | test('empty branch', () => { 265 | const program = `before if 266 | #ifdef GL_ES 267 | precision mediump float; 268 | #endif 269 | after if`; 270 | 271 | const ast = parse(program); 272 | 273 | preprocessAst(ast); 274 | expect(generate(ast)).toBe(`before if 275 | after if`); 276 | }); 277 | 278 | test('evaluate else branch', () => { 279 | const program = ` 280 | #define A 281 | before if 282 | #if defined(D) 283 | inside if 284 | #elif defined(E) 285 | inside elif 286 | #else 287 | else body 288 | #endif 289 | after if`; 290 | 291 | const ast = parse(program); 292 | preprocessAst(ast); 293 | expect(generate(ast)).toBe(` 294 | before if 295 | else body 296 | after if`); 297 | }); 298 | 299 | test('self referential object macro', () => { 300 | const program = ` 301 | #define first first second 302 | #define second first 303 | second`; 304 | 305 | // If this has an infinte loop, the test will never finish 306 | const ast = parse(program); 307 | preprocessAst(ast); 308 | expect(generate(ast)).toBe(` 309 | first second`); 310 | }); 311 | 312 | test('self referential function macro', () => { 313 | const program = ` 314 | #define foo() foo() 315 | foo()`; 316 | 317 | // If this has an infinte loop, the test will never finish 318 | const ast = parse(program); 319 | preprocessAst(ast); 320 | expect(generate(ast)).toBe(` 321 | foo()`); 322 | }); 323 | 324 | test('self referential macro combinations', () => { 325 | const program = ` 326 | #define b c 327 | #define first(a,b) a + b 328 | #define second first(1,b) 329 | second`; 330 | 331 | // If this has an infinte loop, the test will never finish 332 | const ast = parse(program); 333 | preprocessAst(ast); 334 | expect(generate(ast)).toBe(` 335 | 1 + c`); 336 | }); 337 | 338 | test("function call macro isn't expanded", () => { 339 | const program = ` 340 | #define foo() no expand 341 | foo`; 342 | 343 | const ast = parse(program); 344 | // debugAst(ast); 345 | preprocessAst(ast); 346 | expect(generate(ast)).toBe(` 347 | foo`); 348 | }); 349 | 350 | test(`function macro where source variable is same as macro argument`, () => { 351 | const program = ` 352 | #define FN(x, y) x + y 353 | FN(y, x); 354 | FN(y.y, x.x); 355 | FN(yy, xx); 356 | `; 357 | 358 | const ast = parse(program); 359 | preprocessAst(ast); 360 | 361 | // Ensure that if the argument passed to the fn FN(X) has the 362 | // same name as the macro definition #define FN(X), it doesn't get expanded 363 | // https://github.com/ShaderFrog/glsl-parser/issues/31 364 | expect(generate(ast)).toBe(` 365 | y + x; 366 | y.y + x.x; 367 | yy + xx; 368 | `); 369 | }); 370 | 371 | test("macro that isn't macro function call call is expanded", () => { 372 | const program = ` 373 | #define foo () yes expand 374 | foo`; 375 | 376 | const ast = parse(program); 377 | // debugAst(ast); 378 | preprocessAst(ast); 379 | expect(generate(ast)).toBe(` 380 | () yes expand`); 381 | }); 382 | 383 | test('unterminated macro function call', () => { 384 | const program = ` 385 | #define foo() yes expand 386 | foo( 387 | foo()`; 388 | 389 | const ast = parse(program); 390 | expect(() => preprocessAst(ast)).toThrow( 391 | 'foo( unterminated macro invocation' 392 | ); 393 | }); 394 | 395 | test('macro function calls with no arguments', () => { 396 | const program = ` 397 | #define foo() yes expand 398 | foo() 399 | foo 400 | ()`; 401 | 402 | const ast = parse(program); 403 | preprocessAst(ast); 404 | expect(generate(ast)).toBe(` 405 | yes expand 406 | yes expand`); 407 | }); 408 | 409 | test('macro function calls with bad arguments', () => { 410 | expect(() => { 411 | preprocessAst( 412 | parse(` 413 | #define foo( a, b ) a + b 414 | foo(1,2,3)`) 415 | ); 416 | }).toThrow("'foo': Too many arguments for macro"); 417 | 418 | expect(() => { 419 | preprocessAst( 420 | parse(` 421 | #define foo( a ) a + b 422 | foo(,)`) 423 | ); 424 | }).toThrow("'foo': Too many arguments for macro"); 425 | 426 | expect(() => { 427 | preprocessAst( 428 | parse(` 429 | #define foo( a, b ) a + b 430 | foo(1)`) 431 | ); 432 | }).toThrow("'foo': Not enough arguments for macro"); 433 | }); 434 | 435 | test('macro function calls with arguments', () => { 436 | const program = ` 437 | #define foo( a, b ) a + b 438 | foo(x + y, (z-t + vec3(0.0, 1.0))) 439 | foo 440 | (q, 441 | r) 442 | foo(,)`; 443 | 444 | const ast = parse(program); 445 | preprocessAst(ast); 446 | expect(generate(ast)).toBe(` 447 | x + y + (z-t + vec3(0.0, 1.0)) 448 | q + r 449 | + `); 450 | }); 451 | 452 | test('nested function macro expansion', () => { 453 | const program = ` 454 | #define foo(x, y) x + y 455 | foo (foo (a, b), c) 456 | foo(foo (foo (x, y), z) , w)`; 457 | 458 | const ast = parse(program); 459 | preprocessAst(ast); 460 | expect(generate(ast)).toBe(` 461 | a + b + c 462 | x + y + z + w`); 463 | }); 464 | 465 | test('nested function macro expansion referencing other macros', () => { 466 | const program = ` 467 | #define foo(x, y) bar(x, y) + bar(y, x) 468 | #define bar(x, y) (x * y) 469 | foo(a, b)`; 470 | 471 | const ast = parse(program); 472 | preprocessAst(ast); 473 | expect(generate(ast)).toBe(` 474 | (a * b) + (b * a)`); 475 | }); 476 | 477 | test('macros that reference each other', () => { 478 | const program = ` 479 | #define foo() bar() 480 | #define bar() foo() 481 | bar()`; 482 | 483 | const ast = parse(program); 484 | preprocessAst(ast); 485 | expect(generate(ast)).toBe(` 486 | bar()`); 487 | }); 488 | 489 | // The preprocessor does the wrong thing here so I'm commenting out this test 490 | // for now. The current preprocessor results in bar(bar(1.0)). To achieve the 491 | // correct result here I likely need to redo the expansion system to use "blue 492 | // painting" https://en.wikipedia.org/wiki/Painted_blue 493 | xtest('nested function macros that reference each other', () => { 494 | const program = ` 495 | #define foo(x) bar(x) 496 | #define bar(x) foo(x) 497 | bar(foo(1.0))`; 498 | 499 | const ast = parse(program); 500 | preprocessAst(ast); 501 | expect(generate(ast)).toBe(` 502 | bar(foo(1.0))`); 503 | }); 504 | 505 | test('multi-pass cross-referencing object macros', () => { 506 | const program = ` 507 | #define INNER x 508 | #define OUTER (1.0/INNER) 509 | OUTER 510 | INNER`; 511 | 512 | const ast = parse(program); 513 | preprocessAst(ast); 514 | expect(generate(ast)).toBe(` 515 | (1.0/x) 516 | x`); 517 | }); 518 | 519 | test('token pasting', () => { 520 | const program = ` 521 | #define COMMAND(NAME) { NAME, NAME ## _command ## x ## y } 522 | COMMAND(x)`; 523 | 524 | const ast = parse(program); 525 | preprocessAst(ast); 526 | expect(generate(ast)).toBe(` 527 | { x, x_commandxy }`); 528 | }); 529 | 530 | test('preservation', () => { 531 | const program = ` 532 | #line 0 533 | #version 100 "hi" 534 | #define GL_es_profile 1 535 | #extension all : disable 536 | #error whoopsie 537 | #define A 1 538 | before if 539 | #if A == 1 || B == 2 540 | inside if 541 | #define A 542 | #elif A == 1 || defined(B) && C == 2 543 | float a; 544 | #define B 545 | #endif 546 | outside endif 547 | #pragma mypragma: something(else) 548 | function_call line after program`; 549 | 550 | const ast = parse(program); 551 | 552 | preprocessAst(ast, { 553 | preserve: { 554 | conditional: () => false, 555 | line: () => true, 556 | error: () => true, 557 | extension: () => true, 558 | pragma: () => true, 559 | version: () => true, 560 | }, 561 | }); 562 | expect(generate(ast)).toBe(` 563 | #line 0 564 | #version 100 "hi" 565 | #extension all : disable 566 | #error whoopsie 567 | before if 568 | inside if 569 | outside endif 570 | #pragma mypragma: something(else) 571 | function_call line after program`); 572 | }); 573 | 574 | test('different line breaks character', () => { 575 | const program = '#ifndef x\rfloat a = 1.0;\r\n#endif'; 576 | 577 | const ast = parse(program); 578 | preprocessAst(ast); 579 | expect(generate(ast)).toBe('float a = 1.0;\r\n'); 580 | }); 581 | 582 | test('generate #ifdef & #ifndef & #else', () => { 583 | expectParsedProgram(` 584 | #ifdef AA 585 | float a; 586 | #else 587 | float b; 588 | #endif 589 | 590 | #ifndef CC 591 | float c; 592 | #endif 593 | 594 | #if AA == 2 595 | float d; 596 | #endif 597 | `); 598 | }); 599 | 600 | test('test macro with "defined" at start of name', () => { 601 | const program = ` 602 | #define definedX 1 603 | #if defined(definedX) && defined definedX && definedX 604 | true 605 | #endif 606 | `; 607 | expectParsedProgram(program); 608 | const ast = parse(program); 609 | preprocessAst(ast); 610 | expect(generate(ast)).toBe(` 611 | true 612 | `); 613 | }); 614 | 615 | test('inline comments in if statement expression', () => { 616 | const program = ` 617 | #define AAA 618 | #define BBB 619 | #if defined/**/AAA && defined/**/ BBB 620 | true 621 | #endif 622 | `; 623 | expectParsedProgram(program, { preserveComments: true }); 624 | const ast = parse(program, { preserveComments: true }); 625 | preprocessAst(ast); 626 | expect(generate(ast)).toBe(` 627 | true 628 | `); 629 | }); 630 | 631 | test('multiline macros', () => { 632 | const program = ` 633 | #define X a\\ 634 | b 635 | #define Y \\ 636 | c\\ 637 | d\\ 638 | 639 | #define Z \\ 640 | e \\ 641 | f 642 | #define W\\ 643 | 644 | vec3 x() { 645 | X 646 | Y 647 | Z 648 | W 649 | }`; 650 | const ast = parse(program); 651 | preprocessAst(ast); 652 | expect(generate(ast)).toBe(` 653 | vec3 x() { 654 | a b 655 | cd 656 | e f 657 | 658 | }`); 659 | }); 660 | 661 | test('visitPreprocessedAst test', () => { 662 | const program = ` 663 | #ifdef AA // test 664 | true 665 | #endif`; 666 | const ast = parse(program, { 667 | preserveComments: true, 668 | }); 669 | visitPreprocessedAst(ast, {}); 670 | expect(generate(ast)).toBe(program); 671 | }); 672 | -------------------------------------------------------------------------------- /src/parser/scope.test.ts: -------------------------------------------------------------------------------- 1 | import generate from './generator.js'; 2 | import { renameBindings, renameFunctions, renameTypes } from './utils.js'; 3 | import { UNKNOWN_TYPE } from './grammar.js'; 4 | import { buildParser, nextWarn } from './test-helpers.js'; 5 | 6 | let c!: ReturnType; 7 | beforeAll(() => (c = buildParser())); 8 | 9 | test('scope bindings and type names', () => { 10 | const ast = c.parseSrc(` 11 | float selfref, b = 1.0, c = selfref; 12 | vec2 texcoord1, texcoord2; 13 | vec3 position; 14 | vec4 myRGBA; 15 | ivec2 textureLookup; 16 | bvec3 less; 17 | float arr1[5] = float[5](3.4, 4.2, 5.0, 5.2, 1.1); 18 | vec4[2] arr2[3]; 19 | vec4[3][2] arr3; 20 | vec3 fnName() {} 21 | struct light { 22 | float intensity; 23 | vec3 position; 24 | }; 25 | coherent buffer Block { 26 | readonly vec4 member1; 27 | vec4 member2; 28 | };`); 29 | // debugAst(ast); 30 | expect(Object.keys(ast.scopes[0].bindings)).toEqual([ 31 | 'selfref', 32 | 'b', 33 | 'c', 34 | 'texcoord1', 35 | 'texcoord2', 36 | 'position', 37 | 'myRGBA', 38 | 'textureLookup', 39 | 'less', 40 | 'arr1', 41 | 'arr2', 42 | 'arr3', 43 | 'Block', 44 | ]); 45 | expect(Object.keys(ast.scopes[0].functions)).toEqual(['fnName']); 46 | expect(Object.keys(ast.scopes[0].types)).toEqual(['light']); 47 | }); 48 | 49 | test('scope references', () => { 50 | const ast = c.parseSrc( 51 | ` 52 | float selfref, b = 1.0, c = selfref; 53 | mat2x2 myMat = mat2( vec2( 1.0, 0.0 ), vec2( 0.0, 1.0 ) ); 54 | struct { 55 | float s; 56 | float t; 57 | } structArr[]; 58 | struct structType { 59 | float s; 60 | float t; 61 | }; 62 | structType z; 63 | 64 | float protoFn(float x); 65 | 66 | float shadowed; 67 | float reused; 68 | float unused; 69 | void useMe() {} 70 | vec3 fnName(float arg1, vec3 arg2) { 71 | float shadowed = arg1; 72 | structArr[0].x++; 73 | 74 | if(true) { 75 | float x = shadowed + 1 + reused; 76 | } 77 | 78 | { 79 | float compound; 80 | compound = shadowed + reused; 81 | } 82 | 83 | { 84 | float compound; 85 | compound = shadowed + reused + compound; 86 | } 87 | unknown(); 88 | 89 | MyStruct dataArray[1] = { 90 | {1.0} 91 | }; 92 | 93 | protoFn(1.0); 94 | useMe(); 95 | }`, 96 | { quiet: true } 97 | ); 98 | expect(ast.scopes[0].bindings.selfref.references).toHaveLength(2); 99 | expect(ast.scopes[0].bindings.b.references).toHaveLength(1); 100 | expect(ast.scopes[0].bindings.c.references).toHaveLength(1); 101 | expect(ast.scopes[0].bindings.myMat.references).toHaveLength(1); 102 | expect(ast.scopes[0].bindings.structArr.references).toHaveLength(2); 103 | expect(ast.scopes[0].bindings.shadowed.references).toHaveLength(1); 104 | expect(ast.scopes[0].types.structType.references).toHaveLength(2); 105 | expect(ast.scopes[0].functions.useMe['void: void'].references).toHaveLength( 106 | 2 107 | ); 108 | expect(ast.scopes[2].bindings.arg1.references).toHaveLength(2); 109 | expect(ast.scopes[2].bindings.arg2.references).toHaveLength(1); 110 | expect(ast.scopes[2].bindings.shadowed.references).toHaveLength(4); 111 | // reused - used in inner scope 112 | expect(ast.scopes[0].bindings.reused.references).toHaveLength(4); 113 | // compound - used in first innermost scope only 114 | expect(ast.scopes[4].bindings.compound.references).toHaveLength(2); 115 | // compound - used in last innermost scope only 116 | expect(ast.scopes[5].bindings.compound.references).toHaveLength(3); 117 | 118 | expect( 119 | ast.scopes[0].functions.unknown['UNKNOWN TYPE: void'].references 120 | ).toHaveLength(1); 121 | expect( 122 | ast.scopes[0].functions.unknown['UNKNOWN TYPE: void'].declaration 123 | ).toBe(undefined); 124 | }); 125 | 126 | test('scope binding declarations', () => { 127 | const ast = c.parseSrc( 128 | ` 129 | float selfref, b = 1.0, c = selfref; 130 | void main() { 131 | selfref += d; 132 | }`, 133 | { quiet: true } 134 | ); 135 | expect(ast.scopes[0].bindings.selfref.references).toHaveLength(3); 136 | expect(ast.scopes[0].bindings.selfref.declaration).toBeTruthy(); 137 | expect(ast.scopes[0].bindings.b.references).toHaveLength(1); 138 | expect(ast.scopes[0].bindings.b.declaration).toBeTruthy(); 139 | expect(ast.scopes[0].bindings.c.references).toHaveLength(1); 140 | expect(ast.scopes[0].bindings.c.declaration).toBeTruthy(); 141 | 142 | expect(ast.scopes[1].bindings.d.references).toHaveLength(1); 143 | expect(ast.scopes[1].bindings.d.declaration).toBeFalsy(); 144 | }); 145 | 146 | test('struct constructor identified in scope', () => { 147 | const ast = c.parseSrc(` 148 | struct light { 149 | float intensity; 150 | vec3 position; 151 | }; 152 | light lightVar = light(3.0, vec3(1.0, 2.0, 3.0)); 153 | `); 154 | expect(ast.scopes[0].types.light.references).toHaveLength(3); 155 | }); 156 | 157 | test('function overloaded scope', () => { 158 | const ast = c.parseSrc(` 159 | vec4 overloaded(vec4 x) { 160 | return x; 161 | } 162 | float overloaded(float x) { 163 | return x; 164 | }`); 165 | expect(Object.entries(ast.scopes[0].functions.overloaded)).toHaveLength(2); 166 | }); 167 | 168 | test('overriding glsl builtin function', () => { 169 | // "noise" is a built-in GLSL function that should be identified and renamed 170 | const ast = c.parseSrc(` 171 | float noise() {} 172 | float fn() { 173 | vec2 uv; 174 | uv += noise(); 175 | } 176 | `); 177 | 178 | expect(ast.scopes[0].functions.noise); 179 | ast.scopes[0].functions = renameFunctions( 180 | ast.scopes[0].functions, 181 | (name) => `${name}_FUNCTION` 182 | ); 183 | expect(generate(ast)).toBe(` 184 | float noise_FUNCTION() {} 185 | float fn_FUNCTION() { 186 | vec2 uv; 187 | uv += noise_FUNCTION(); 188 | } 189 | `); 190 | }); 191 | 192 | test('rename bindings and functions', () => { 193 | const ast = c.parseSrc( 194 | ` 195 | float selfref, b = 1.0, c = selfref; 196 | mat2x2 myMat = mat2( vec2( 1.0, 0.0 ), vec2( 0.0, 1.0 ) ); 197 | struct { 198 | float s; 199 | float t; 200 | } structArr[]; 201 | struct structType { 202 | float s; 203 | float t; 204 | }; 205 | structType z; 206 | 207 | float shadowed; 208 | float reused; 209 | float unused; 210 | void x() {} 211 | vec3 fnName(float arg1, vec3 arg2) { 212 | float shadowed = arg1; 213 | float y = x().length(); 214 | structArr[0].x++; 215 | 216 | if(true) { 217 | float x = shadowed + 1 + reused; 218 | } 219 | 220 | { 221 | float compound; 222 | compound = shadowed + reused; 223 | } 224 | 225 | { 226 | float compound; 227 | compound = shadowed + reused + compound; 228 | } 229 | } 230 | vec4 LinearToLinear( in vec4 value ) { 231 | return value; 232 | } 233 | vec4 mapTexelToLinear( vec4 value ) { return LinearToLinear( value ); } 234 | vec4 linearToOutputTexel( vec4 value ) { return LinearToLinear( value ); } 235 | `, 236 | { quiet: true } 237 | ); 238 | 239 | ast.scopes[0].bindings = renameBindings( 240 | ast.scopes[0].bindings, 241 | (name) => `${name}_VARIABLE` 242 | ); 243 | ast.scopes[0].functions = renameFunctions( 244 | ast.scopes[0].functions, 245 | (name) => `${name}_FUNCTION` 246 | ); 247 | 248 | expect(generate(ast)).toBe(` 249 | float selfref_VARIABLE, b_VARIABLE = 1.0, c_VARIABLE = selfref_VARIABLE; 250 | mat2x2 myMat_VARIABLE = mat2( vec2( 1.0, 0.0 ), vec2( 0.0, 1.0 ) ); 251 | struct { 252 | float s; 253 | float t; 254 | } structArr_VARIABLE[]; 255 | struct structType { 256 | float s; 257 | float t; 258 | }; 259 | structType z_VARIABLE; 260 | 261 | float shadowed_VARIABLE; 262 | float reused_VARIABLE; 263 | float unused_VARIABLE; 264 | void x_FUNCTION() {} 265 | vec3 fnName_FUNCTION(float arg1, vec3 arg2) { 266 | float shadowed = arg1; 267 | float y = x_FUNCTION().length(); 268 | structArr_VARIABLE[0].x++; 269 | 270 | if(true) { 271 | float x = shadowed + 1 + reused_VARIABLE; 272 | } 273 | 274 | { 275 | float compound; 276 | compound = shadowed + reused_VARIABLE; 277 | } 278 | 279 | { 280 | float compound; 281 | compound = shadowed + reused_VARIABLE + compound; 282 | } 283 | } 284 | vec4 LinearToLinear_FUNCTION( in vec4 value ) { 285 | return value; 286 | } 287 | vec4 mapTexelToLinear_FUNCTION( vec4 value ) { return LinearToLinear_FUNCTION( value ); } 288 | vec4 linearToOutputTexel_FUNCTION( vec4 value ) { return LinearToLinear_FUNCTION( value ); } 289 | `); 290 | }); 291 | 292 | test('detecting struct scope and usage', () => { 293 | const ast = c.parseSrc(` 294 | struct StructName { 295 | vec3 color; 296 | }; 297 | struct OtherStruct { 298 | StructName inner; 299 | }; 300 | StructName proto(StructName x, StructName[3]); 301 | 302 | subroutine StructName colorRedBlue(); 303 | subroutine (colorRedBlue) StructName redColor() { 304 | return StructName(1.0, 0.0, 0.0, 1.0); 305 | } 306 | 307 | StructName reflectedLight = StructName(vec3(0.0)); 308 | StructName main(in StructName x, StructName[3] y) { 309 | StructName ref = StructName(); 310 | float a = 1.0 + StructName(1.0).color.x; 311 | struct StructName { 312 | vec3 color; 313 | }; 314 | StructName ref2 = StructName(); 315 | float a2 = 1.0 + StructName(1.0).color.x; 316 | } 317 | `); 318 | ast.scopes[0].types = renameTypes(ast.scopes[0].types, (name) => `${name}_x`); 319 | 320 | expect(generate(ast)).toBe(` 321 | struct StructName_x { 322 | vec3 color; 323 | }; 324 | struct OtherStruct_x { 325 | StructName_x inner; 326 | }; 327 | StructName_x proto(StructName_x x, StructName_x[3]); 328 | 329 | subroutine StructName_x colorRedBlue(); 330 | subroutine (colorRedBlue) StructName_x redColor() { 331 | return StructName_x(1.0, 0.0, 0.0, 1.0); 332 | } 333 | 334 | StructName_x reflectedLight = StructName_x(vec3(0.0)); 335 | StructName_x main(in StructName_x x, StructName_x[3] y) { 336 | StructName_x ref = StructName_x(); 337 | float a = 1.0 + StructName_x(1.0).color.x; 338 | struct StructName { 339 | vec3 color; 340 | }; 341 | StructName ref2 = StructName(); 342 | float a2 = 1.0 + StructName(1.0).color.x; 343 | } 344 | `); 345 | // Ensure structs aren't added to global function scope since they should be 346 | // identified as types 347 | expect(Object.keys(ast.scopes[0].functions)).toEqual([ 348 | 'proto', 349 | 'colorRedBlue', 350 | 'redColor', 351 | 'main', 352 | ]); 353 | expect(Object.keys(ast.scopes[0].bindings)).toEqual(['reflectedLight']); 354 | expect(Object.keys(ast.scopes[0].types)).toEqual([ 355 | 'StructName_x', 356 | 'OtherStruct_x', 357 | ]); 358 | expect(ast.scopes[0].types.StructName_x.references).toHaveLength(16); 359 | 360 | // Inner struct definition should be found in inner fn scope 361 | expect(Object.keys(ast.scopes[2].types)).toEqual(['StructName']); 362 | }); 363 | 364 | test('fn args shadowing global scope identified as separate bindings', () => { 365 | const ast = c.parseSrc(` 366 | attribute vec3 position; 367 | vec3 func(vec3 position) { 368 | return position; 369 | }`); 370 | ast.scopes[0].bindings = renameBindings(ast.scopes[0].bindings, (name) => 371 | name === 'position' ? 'renamed' : name 372 | ); 373 | // The func arg "position" shadows the global binding, it should be untouched 374 | expect(generate(ast)).toBe(` 375 | attribute vec3 renamed; 376 | vec3 func(vec3 position) { 377 | return position; 378 | }`); 379 | }); 380 | 381 | test('I do not yet know what to do with layout()', () => { 382 | const ast = c.parseSrc(` 383 | layout(std140,column_major) uniform; 384 | float a; 385 | uniform Material 386 | { 387 | uniform vec2 vProp; 388 | };`); 389 | 390 | // This shouldn't crash - see the comment block in renameBindings() 391 | ast.scopes[0].bindings = renameBindings( 392 | ast.scopes[0].bindings, 393 | (name) => `${name}_x` 394 | ); 395 | expect(generate(ast)).toBe(` 396 | layout(std140,column_major) uniform; 397 | float a_x; 398 | uniform Material 399 | { 400 | uniform vec2 vProp; 401 | };`); 402 | }); 403 | 404 | test(`(regression) ensure self-referenced variables don't appear as types`, () => { 405 | const ast = c.parseSrc(` 406 | float a = 1.0, c = a; 407 | `); 408 | expect(Object.keys(ast.scopes[0].types)).toEqual([]); 409 | }); 410 | 411 | test('identifies a declared function with references', () => { 412 | const ast = c.parseSrc(` 413 | vec4[3] main(float a, vec3 b) {} 414 | void x() { 415 | float a = 1.0; 416 | float b = 1.0; 417 | main(a, b); 418 | } 419 | `); 420 | const signature = 'vec4[3]: float, vec3'; 421 | // Should have found no types 422 | expect(ast.scopes[0].types).toMatchObject({}); 423 | // Should have found one overload signature 424 | expect(ast.scopes[0].functions).toHaveProperty('main'); 425 | expect(ast.scopes[0].functions.main).toHaveProperty([signature]); 426 | expect(Object.keys(ast.scopes[0].functions.main)).toHaveLength(1); 427 | // Should be declared with references 428 | expect(ast.scopes[0].functions.main[signature].declaration).toBeTruthy(); 429 | expect(ast.scopes[0].functions.main[signature].references).toHaveLength(2); 430 | }); 431 | 432 | test('does not match function overload with different argument length', () => { 433 | const ast = c.parseSrc( 434 | ` 435 | float main(float a, float b) {} 436 | void x() { 437 | main(a, b, c); 438 | } 439 | `, 440 | { quiet: true } 441 | ); 442 | 443 | const unknownSig = `${UNKNOWN_TYPE}: ${UNKNOWN_TYPE}, ${UNKNOWN_TYPE}, ${UNKNOWN_TYPE}`; 444 | const knownSig = `float: float, float`; 445 | // Should have found no types 446 | expect(ast.scopes[0].types).toMatchObject({}); 447 | // Should have found one overload signature 448 | expect(ast.scopes[0].functions).toHaveProperty('main'); 449 | expect(ast.scopes[0].functions.main).toHaveProperty(knownSig); 450 | expect(ast.scopes[0].functions.main).toHaveProperty(unknownSig); 451 | expect(Object.keys(ast.scopes[0].functions.main)).toHaveLength(2); 452 | // Declaration should not match bad overload 453 | expect(ast.scopes[0].functions.main[knownSig].declaration).toBeTruthy(); 454 | expect(ast.scopes[0].functions.main[knownSig].references).toHaveLength(1); 455 | // Bad call should not match definition 456 | expect(ast.scopes[0].functions.main[unknownSig].declaration).toBeFalsy(); 457 | expect(ast.scopes[0].functions.main[unknownSig].references).toHaveLength(1); 458 | }); 459 | 460 | test('handles declared, undeclared, and unknown function cases', () => { 461 | const ast = c.parseSrc( 462 | ` 463 | // Prototype for undeclared function 464 | float main(float, float, float[3]); 465 | 466 | // Prototype and definition for declared function 467 | float main(float a, float b); 468 | float main(float a, float b) {} 469 | 470 | void x() { 471 | main(a, b); 472 | main(a, b, c); 473 | main(a, b, c, d); 474 | } 475 | `, 476 | { quiet: true } 477 | ); 478 | 479 | const defSig = `float: float, float`; 480 | const undefSig = `float: float, float, float[3]`; 481 | const unknownSig = `${UNKNOWN_TYPE}: ${UNKNOWN_TYPE}, ${UNKNOWN_TYPE}, ${UNKNOWN_TYPE}, ${UNKNOWN_TYPE}`; 482 | 483 | // Should have found no types 484 | expect(ast.scopes[0].types).toMatchObject({}); 485 | 486 | // Should have found 3 overload signatures. One overload for defined, one for 487 | // undefined, and one for the unknown call 488 | expect(ast.scopes[0].functions).toHaveProperty('main'); 489 | expect(Object.keys(ast.scopes[0].functions.main)).toHaveLength(3); 490 | expect(ast.scopes[0].functions.main).toHaveProperty([defSig]); 491 | expect(ast.scopes[0].functions.main).toHaveProperty([undefSig]); 492 | expect(ast.scopes[0].functions.main).toHaveProperty([unknownSig]); 493 | 494 | // Defined function has prototype, definition 495 | expect(ast.scopes[0].functions.main[defSig].declaration).toBeTruthy(); 496 | expect(ast.scopes[0].functions.main[defSig].references).toHaveLength(3); 497 | 498 | // Undeclared call has prototype and call, but no declaration 499 | expect(ast.scopes[0].functions.main[undefSig].declaration).toBeFalsy(); 500 | expect(ast.scopes[0].functions.main[undefSig].references).toHaveLength(2); 501 | 502 | // Unknown function is hanging out by itself 503 | expect(ast.scopes[0].functions.main[unknownSig].declaration).toBeFalsy(); 504 | expect(ast.scopes[0].functions.main[unknownSig].references).toHaveLength(1); 505 | }); 506 | 507 | test('warns on undeclared functions and structs', () => { 508 | const next = nextWarn(); 509 | 510 | c.parseSrc(` 511 | MyStruct x = MyStruct(); 512 | void main() { 513 | a(); 514 | a(1); 515 | z += 1; 516 | } 517 | struct MyStruct { float y; }; 518 | `); 519 | 520 | expect(next()).toContain('undeclared function: "MyStruct"'); 521 | expect(next()).toContain('undeclared type: "MyStruct"'); 522 | expect(next()).toContain('undeclared function: "a"'); 523 | expect(next()).toContain('No matching overload for function: "a"'); 524 | expect(next()).toContain('Encountered undefined variable: "z"'); 525 | expect(next()).toContain('Type "MyStruct" was used before it was declared'); 526 | }); 527 | 528 | test('warns on duplicate declarations', () => { 529 | const next = nextWarn(); 530 | 531 | c.parseSrc(` 532 | struct MyStruct { float y; }; 533 | struct MyStruct { float y; }; 534 | float dupefloat = 1.0; 535 | float dupefloat = 1.0; 536 | float dupefn(float b); 537 | float dupefn(float); 538 | void dupefn() {} 539 | void dupefn() {} 540 | `); 541 | 542 | expect(next()).toContain('duplicate type declaration: "MyStruct"'); 543 | expect(next()).toContain('duplicate variable declaration: "dupefloat"'); 544 | expect(next()).toContain('duplicate function prototype: "dupefn"'); 545 | expect(next()).toContain('duplicate function definition: "dupefn"'); 546 | }); 547 | 548 | test('undeclared variables are added to the expected scope', () => { 549 | const ast = c.parseSrc( 550 | ` 551 | void a() { 552 | MyStruct x; 553 | a(); 554 | } 555 | `, 556 | { quiet: true } 557 | ); 558 | // Function should get added to global scope 559 | expect(ast.scopes[0].types).toMatchObject({}); 560 | expect(ast.scopes[0].functions).toHaveProperty('a'); 561 | // Struct should get added to inner scope 562 | expect(ast.scopes[1].types).toHaveProperty('MyStruct'); 563 | }); 564 | 565 | test('postfix is added to scope', () => { 566 | const ast = c.parseSrc(` 567 | void a() {} 568 | void main() { 569 | float y = a().xyz; 570 | float z = a().length(); 571 | }`); 572 | const a = Object.values(ast.scopes[0].functions.a)[0]; 573 | expect(a.references).toHaveLength(3); 574 | }); 575 | 576 | test('rename function prototypes', () => { 577 | const ast = c.parseSrc( 578 | `vec3 hash3(vec3 p3); 579 | vec3 hash3(vec3 p3) {}` 580 | ); 581 | 582 | ast.scopes[0].functions = renameFunctions( 583 | ast.scopes[0].functions, 584 | (name) => `${name}_FUNCTION` 585 | ); 586 | 587 | expect(generate(ast)).toBe(`vec3 hash3_FUNCTION(vec3 p3); 588 | vec3 hash3_FUNCTION(vec3 p3) {}`); 589 | }); 590 | -------------------------------------------------------------------------------- /src/parser/grammar.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Helper functions used by preprocessor-grammar.pegjs. Also re-exports 3 | * functions from other files used in the grammar. 4 | */ 5 | 6 | import { 7 | AstNode, 8 | CompoundStatementNode, 9 | LocationInfo, 10 | LocationObject, 11 | BinaryNode, 12 | FunctionPrototypeNode, 13 | LiteralNode, 14 | FunctionNode, 15 | FunctionCallNode, 16 | TypeNameNode, 17 | FullySpecifiedTypeNode, 18 | TypeSpecifierNode, 19 | } from '../ast/index.js'; 20 | import { ParserOptions } from './parser.js'; 21 | import { 22 | Scope, 23 | findGlobalScope, 24 | findOverloadDefinition, 25 | findTypeScope, 26 | functionDeclarationSignature, 27 | functionUseSignature, 28 | newOverloadIndex, 29 | isDeclaredFunction, 30 | isDeclaredType, 31 | makeScopeIndex, 32 | findBindingScope, 33 | } from './scope.js'; 34 | 35 | export { 36 | Scope, 37 | findGlobalScope, 38 | findOverloadDefinition, 39 | findTypeScope, 40 | functionDeclarationSignature, 41 | functionUseSignature, 42 | newOverloadIndex, 43 | isDeclaredFunction, 44 | isDeclaredType, 45 | }; 46 | 47 | export const UNKNOWN_TYPE = 'UNKNOWN TYPE'; 48 | 49 | // Peggyjs globals 50 | type Text = () => string; 51 | type Location = () => LocationObject; 52 | 53 | // Context passed to makeLocals 54 | type Context = { 55 | text: Text; 56 | location: Location; 57 | options: ParserOptions; 58 | scope: Scope; 59 | scopes: Scope[]; 60 | }; 61 | 62 | // A "partial" is data that's computed as part of definition production, but is then 63 | // merged into some higher rule, and doesn't itself become definition node. 64 | export type PartialNode = { partial: any }; 65 | export const partial = (typeNameOrAttrs: string | object, attrs: object) => ({ 66 | partial: 67 | attrs === undefined 68 | ? typeNameOrAttrs 69 | : { 70 | type: typeNameOrAttrs, 71 | ...attrs, 72 | }, 73 | }); 74 | 75 | // Filter out "empty" elements from an array 76 | export const xnil = (...args: any[]) => 77 | args 78 | .flat() 79 | .filter((e) => e !== undefined && e !== null && e !== '' && e.length !== 0); 80 | 81 | // Given an array of nodes with potential null empty values, convert to text. 82 | // Kind of like $(rule) but filters out empty rules 83 | export const toText = (...args: any[]) => xnil(args).join(''); 84 | 85 | export const ifOnly = (arr: any[]) => (arr.length > 1 ? arr : arr[0]); 86 | 87 | // Remove empty elements and return value if only 1 element remains 88 | export const collapse = (...args: any[]) => ifOnly(xnil(args)); 89 | 90 | // Create definition left associative tree of nodes 91 | export const leftAssociate = ( 92 | head: AstNode, 93 | ...tail: [[LiteralNode, AstNode]][] 94 | ) => 95 | tail.flat().reduce( 96 | (left, [operator, right]) => ({ 97 | type: 'binary', 98 | operator, 99 | left, 100 | right, 101 | }), 102 | head 103 | ); 104 | 105 | // From https://www.khronos.org/opengl/wiki/Built-in_Variable_(GLSL) 106 | export const BUILT_INS = { 107 | vertex: new Set([ 108 | 'gl_VertexID', 109 | 'gl_InstanceID', 110 | 'gl_DrawID', 111 | 'gl_BaseVertex', 112 | 'gl_BaseInstance', 113 | 'gl_Position', 114 | 'gl_PointSize', 115 | 'gl_ClipDistance', 116 | ]), 117 | fragment: new Set([ 118 | 'gl_FragColor', 119 | 'gl_FragData', 120 | 'gl_FragCoord', 121 | 'gl_FrontFacing', 122 | 'gl_PointCoord', 123 | 'gl_SampleID', 124 | 'gl_SamplePosition', 125 | 'gl_SampleMaskIn', 126 | 'gl_ClipDistance', 127 | 'gl_PrimitiveID', 128 | 'gl_Layer', 129 | 'gl_ViewportIndex', 130 | 'gl_FragDepth', 131 | 'gl_SampleMask', 132 | ]), 133 | }; 134 | 135 | // From https://www.khronos.org/registry/OpenGL-Refpages/gl4/index.php 136 | // excluding gl_ prefixed builtins, which don't appear to be functions 137 | export const FN_BUILT_INS = new Set([ 138 | 'abs', 139 | 'acos', 140 | 'acosh', 141 | 'all', 142 | 'any', 143 | 'asin', 144 | 'asinh', 145 | 'atan', 146 | 'atanh', 147 | 'atomicAdd', 148 | 'atomicAnd', 149 | 'atomicCompSwap', 150 | 'atomicCounter', 151 | 'atomicCounterDecrement', 152 | 'atomicCounterIncrement', 153 | 'atomicExchange', 154 | 'atomicMax', 155 | 'atomicMin', 156 | 'atomicOr', 157 | 'atomicXor', 158 | 'barrier', 159 | 'bitCount', 160 | 'bitfieldExtract', 161 | 'bitfieldInsert', 162 | 'bitfieldReverse', 163 | 'ceil', 164 | 'clamp', 165 | 'cos', 166 | 'cosh', 167 | 'cross', 168 | 'degrees', 169 | 'determinant', 170 | 'dFdx', 171 | 'dFdxCoarse', 172 | 'dFdxFine', 173 | 'dFdy', 174 | 'dFdyCoarse', 175 | 'dFdyFine', 176 | 'distance', 177 | 'dot', 178 | 'EmitStreamVertex', 179 | 'EmitVertex', 180 | 'EndPrimitive', 181 | 'EndStreamPrimitive', 182 | 'equal', 183 | 'exp', 184 | 'exp2', 185 | 'faceforward', 186 | 'findLSB', 187 | 'findMSB', 188 | 'floatBitsToInt', 189 | 'floatBitsToUint', 190 | 'floor', 191 | 'fma', 192 | 'fract', 193 | 'frexp', 194 | 'fwidth', 195 | 'fwidthCoarse', 196 | 'fwidthFine', 197 | 'greaterThan', 198 | 'greaterThanEqual', 199 | 'groupMemoryBarrier', 200 | 'imageAtomicAdd', 201 | 'imageAtomicAnd', 202 | 'imageAtomicCompSwap', 203 | 'imageAtomicExchange', 204 | 'imageAtomicMax', 205 | 'imageAtomicMin', 206 | 'imageAtomicOr', 207 | 'imageAtomicXor', 208 | 'imageLoad', 209 | 'imageSamples', 210 | 'imageSize', 211 | 'imageStore', 212 | 'imulExtended', 213 | 'intBitsToFloat', 214 | 'interpolateAtCentroid', 215 | 'interpolateAtOffset', 216 | 'interpolateAtSample', 217 | 'inverse', 218 | 'inversesqrt', 219 | 'isinf', 220 | 'isnan', 221 | 'ldexp', 222 | 'length', 223 | 'lessThan', 224 | 'lessThanEqual', 225 | 'log', 226 | 'log2', 227 | 'matrixCompMult', 228 | 'max', 229 | 'memoryBarrier', 230 | 'memoryBarrierAtomicCounter', 231 | 'memoryBarrierBuffer', 232 | 'memoryBarrierImage', 233 | 'memoryBarrierShared', 234 | 'min', 235 | 'mix', 236 | 'mod', 237 | 'modf', 238 | 'noise', 239 | 'noise1', 240 | 'noise2', 241 | 'noise3', 242 | 'noise4', 243 | 'normalize', 244 | 'not', 245 | 'notEqual', 246 | 'outerProduct', 247 | 'packDouble2x32', 248 | 'packHalf2x16', 249 | 'packSnorm2x16', 250 | 'packSnorm4x8', 251 | 'packUnorm', 252 | 'packUnorm2x16', 253 | 'packUnorm4x8', 254 | 'pow', 255 | 'radians', 256 | 'reflect', 257 | 'refract', 258 | 'round', 259 | 'roundEven', 260 | 'sign', 261 | 'sin', 262 | 'sinh', 263 | 'smoothstep', 264 | 'sqrt', 265 | 'step', 266 | 'tan', 267 | 'tanh', 268 | 'texelFetch', 269 | 'texelFetchOffset', 270 | 'texture', 271 | 'textureGather', 272 | 'textureGatherOffset', 273 | 'textureGatherOffsets', 274 | 'textureGrad', 275 | 'textureGradOffset', 276 | 'textureLod', 277 | 'textureLodOffset', 278 | 'textureOffset', 279 | 'textureProj', 280 | 'textureProjGrad', 281 | 'textureProjGradOffset', 282 | 'textureProjLod', 283 | 'textureProjLodOffset', 284 | 'textureProjOffset', 285 | 'textureQueryLevels', 286 | 'textureQueryLod', 287 | 'textureSamples', 288 | 'textureSize', 289 | 'transpose', 290 | 'trunc', 291 | 'uaddCarry', 292 | 'uintBitsToFloat', 293 | 'umulExtended', 294 | 'unpackDouble2x32', 295 | 'unpackHalf2x16', 296 | 'unpackSnorm2x16', 297 | 'unpackSnorm4x8', 298 | 'unpackUnorm', 299 | 'unpackUnorm2x16', 300 | 'unpackUnorm4x8', 301 | 'usubBorrow', 302 | // GLSL ES 1.00 303 | 'texture2D', 304 | 'textureCube', 305 | ]); 306 | 307 | /** 308 | * Uses a closure to provide Peggyjs-parser-execution-aware context 309 | */ 310 | export const makeLocals = (context: Context) => { 311 | const getLocation = (loc?: LocationObject) => { 312 | // Try to avoid calling getLocation() more than neccessary 313 | if (!context.options.includeLocation) { 314 | return; 315 | } 316 | // Intentionally drop the "source" and "offset" keys from the location object 317 | const { start, end } = loc || context.location(); 318 | return { start, end }; 319 | }; 320 | 321 | // getLocation() (and etc. functions) are not available in global scope, 322 | // so node() is moved to per-parse scope 323 | const node = (type: AstNode['type'], attrs: any): AstNode => { 324 | const n: AstNode = { 325 | type, 326 | ...attrs, 327 | }; 328 | if (context.options.includeLocation) { 329 | n.location = getLocation(); 330 | } 331 | return n; 332 | }; 333 | 334 | const makeScope = ( 335 | name: string, 336 | parent?: Scope, 337 | startLocation?: LocationObject 338 | ): Scope => { 339 | let newLocation = getLocation(startLocation); 340 | 341 | return { 342 | name, 343 | parent, 344 | ...(newLocation ? { location: newLocation } : false), 345 | bindings: {}, 346 | types: {}, 347 | functions: {}, 348 | }; 349 | }; 350 | 351 | const warn = (message: string): void => { 352 | if (context.options.failOnWarn) { 353 | throw new Error(message); 354 | } 355 | if (!context.options.quiet) { 356 | console.warn(message); 357 | } 358 | }; 359 | 360 | const pushScope = (scope: Scope) => { 361 | context.scopes.push(scope); 362 | return scope; 363 | }; 364 | const popScope = (scope: Scope) => { 365 | if (!scope.parent) { 366 | throw new Error(`Popped bad scope ${scope} at ${context.text()}`); 367 | } 368 | return scope.parent; 369 | }; 370 | 371 | const setScopeEnd = (scope: Scope, end: LocationInfo) => { 372 | if (context.options.includeLocation) { 373 | if (!scope.location) { 374 | console.error(`No end location at ${context.text()}`); 375 | } else { 376 | scope.location.end = end; 377 | } 378 | } 379 | }; 380 | 381 | /** 382 | * Use this when you encounter a function call. warns() if the function is 383 | * not defined or doesn't have a known overload. See the "Caution" note in the 384 | * README for the false positive in findOverloadDefinition() 385 | */ 386 | const addFunctionCallReference = ( 387 | scope: Scope, 388 | name: string, 389 | fnRef: FunctionCallNode 390 | ) => { 391 | const global = findGlobalScope(scope); 392 | 393 | const signature = functionUseSignature(fnRef); 394 | if (!global.functions[name]) { 395 | warn( 396 | `Encountered undeclared function: "${name}" with signature "${signature[2]}"` 397 | ); 398 | global.functions[name] = { 399 | [signature[2]]: newOverloadIndex(signature[0], signature[1], fnRef), 400 | }; 401 | } else { 402 | const existingOverload = findOverloadDefinition( 403 | signature, 404 | global.functions[name] 405 | ); 406 | if (!existingOverload) { 407 | warn( 408 | `No matching overload for function: "${name}" with signature "${signature[2]}"` 409 | ); 410 | global.functions[name][signature[2]] = newOverloadIndex( 411 | signature[0], 412 | signature[1], 413 | fnRef 414 | ); 415 | } else { 416 | existingOverload.references.push(fnRef); 417 | } 418 | } 419 | }; 420 | 421 | /** 422 | * Create a definition for a function in the global scope. Use this when you 423 | * encounter a function definition. 424 | */ 425 | const createFunctionDefinition = ( 426 | scope: Scope, 427 | name: string, 428 | fnRef: FunctionNode 429 | ) => { 430 | const global = findGlobalScope(scope); 431 | 432 | const signature = functionDeclarationSignature(fnRef); 433 | if (!global.functions[name]) { 434 | global.functions[name] = {}; 435 | } 436 | const existing = global.functions[name][signature[2]]; 437 | if (existing) { 438 | if (existing.declaration) { 439 | warn( 440 | `Encountered duplicate function definition: "${name}" with signature "${signature[2]}"` 441 | ); 442 | } else { 443 | existing.declaration = fnRef; 444 | } 445 | existing.references.push(fnRef); 446 | } else { 447 | global.functions[name][signature[2]] = newOverloadIndex( 448 | signature[0], 449 | signature[1], 450 | fnRef 451 | ); 452 | global.functions[name][signature[2]].declaration = fnRef; 453 | } 454 | }; 455 | 456 | /** 457 | * Create a definition for a function prototype. This is *not* the function 458 | * declaration in scope. 459 | */ 460 | const createFunctionPrototype = ( 461 | scope: Scope, 462 | name: string, 463 | fnRef: FunctionPrototypeNode 464 | ) => { 465 | const global = findGlobalScope(scope); 466 | 467 | const signature = functionDeclarationSignature(fnRef); 468 | if (!global.functions[name]) { 469 | global.functions[name] = {}; 470 | } 471 | const existing = global.functions[name][signature[2]]; 472 | if (existing) { 473 | warn( 474 | `Encountered duplicate function prototype: "${name}" with signature "${signature[2]}"` 475 | ); 476 | existing.references.push(fnRef); 477 | } else { 478 | global.functions[name][signature[2]] = newOverloadIndex( 479 | signature[0], 480 | signature[1], 481 | fnRef 482 | ); 483 | } 484 | }; 485 | 486 | /** 487 | * Add the use of a struct TYPE_NAME to the scope. Use this when you know 488 | * you've encountered a struct name. 489 | */ 490 | const addTypeReference = ( 491 | scope: Scope, 492 | name: string, 493 | reference: TypeNameNode 494 | ) => { 495 | const declaredScope = findTypeScope(scope, name); 496 | if (declaredScope) { 497 | declaredScope.types[name].references.push(reference); 498 | } else { 499 | warn(`Encountered undeclared type: "${name}"`); 500 | scope.types[name] = { 501 | references: [reference], 502 | }; 503 | } 504 | }; 505 | 506 | /** 507 | * Create a new user defined type (struct) scope entry. Use this only when you 508 | * know this is a valid struct definition. If the struct name is already 509 | * defined, warn() 510 | */ 511 | const createType = ( 512 | scope: Scope, 513 | name: string, 514 | declaration: TypeNameNode 515 | ) => { 516 | if (name in scope.types) { 517 | if (scope.types[name].declaration) { 518 | warn(`Encountered duplicate type declaration: "${name}"`); 519 | } else { 520 | warn(`Type "${name}" was used before it was declared`); 521 | scope.types[name].declaration = declaration; 522 | } 523 | scope.types[name].references.push(declaration); 524 | } else { 525 | scope.types[name] = { 526 | declaration, 527 | references: [declaration], 528 | }; 529 | } 530 | }; 531 | 532 | /** 533 | * Given a TypeSpecifier, check if it includes a TYPE_NAME node, and if so, 534 | * track it in scope. Use this on any TypeSpecifier. 535 | */ 536 | const addTypeIfFound = ( 537 | scope: Scope, 538 | node: FullySpecifiedTypeNode | TypeSpecifierNode 539 | ) => { 540 | const specifier = 541 | node.type === 'fully_specified_type' 542 | ? node?.specifier?.specifier 543 | : node?.specifier; 544 | 545 | if (specifier.type === 'type_name') { 546 | const name = specifier.identifier; 547 | addTypeReference(scope, name, specifier); 548 | // If type is 'struct', then it was declared in struct_specifier. If 549 | } else if (specifier.type !== 'struct' && specifier.type !== 'keyword') { 550 | console.warn('Unknown specifier', specifier); 551 | throw new Error( 552 | `Unknown declarator specifier ${specifier?.type}. Please file a bug against @shaderfrog/glsl-parser and incldue your source grammar.` 553 | ); 554 | } 555 | }; 556 | 557 | /** 558 | * Create new variable declarations in the scope. Only use this when you know 559 | * the variable is being defined by the AstNode in question. 560 | */ 561 | const createBindings = (scope: Scope, ...bindings: [string, AstNode][]) => { 562 | bindings.forEach(([identifier, binding]) => { 563 | const existing = scope.bindings[identifier]; 564 | if (existing) { 565 | warn(`Encountered duplicate variable declaration: "${identifier}"`); 566 | existing.references.unshift(binding); 567 | } else { 568 | scope.bindings[identifier] = makeScopeIndex(binding, binding); 569 | } 570 | }); 571 | }; 572 | 573 | /** 574 | * When a variable name is encountered in the AST, either add it to the scope 575 | * it's defined in, or if it's not defined, warn(), and add a scope entry 576 | * without a declaraiton. 577 | * Used in the parse tree when you don't know if a variable should be defined 578 | * yet or not, like encountering an IDENTIFIER in an expression. 579 | */ 580 | const addOrCreateBindingReference = ( 581 | scope: Scope, 582 | name: string, 583 | reference: AstNode 584 | ) => { 585 | // In the case of "float definition = 1, b = definition;" we parse the final "definition" before the 586 | // parent declarator list is parsed. So we might need to add the final "definition" 587 | // to the scope first. 588 | const foundScope = findBindingScope(scope, name); 589 | if (foundScope) { 590 | foundScope.bindings[name].references.push(reference); 591 | } else { 592 | if ( 593 | !context.options.stage || 594 | (context.options.stage === 'vertex' && !BUILT_INS.vertex.has(name)) || 595 | (context.options.stage === 'fragment' && 596 | !BUILT_INS.fragment.has(name)) || 597 | (context.options.stage === 'either' && 598 | !BUILT_INS.vertex.has(name) && 599 | !BUILT_INS.fragment) 600 | ) { 601 | warn(`Encountered undefined variable: "${name}"`); 602 | } 603 | // This intentionally does not provide a declaration 604 | scope.bindings[name] = makeScopeIndex(reference); 605 | } 606 | }; 607 | 608 | // Group the statements in a switch statement into cases / default arrays 609 | const groupCases = (statements: (AstNode | PartialNode)[]) => 610 | statements.reduce((cases, stmt) => { 611 | const partial = 'partial' in stmt ? stmt.partial : {}; 612 | if (partial.type === 'case_label') { 613 | return [ 614 | ...cases, 615 | node('switch_case', { 616 | statements: [], 617 | case: partial.case, 618 | test: partial.test, 619 | colon: partial.colon, 620 | }), 621 | ]; 622 | } else if (partial.type === 'default_label') { 623 | return [ 624 | ...cases, 625 | node('default_case', { 626 | statements: [], 627 | default: partial.default, 628 | colon: partial.colon, 629 | }), 630 | ]; 631 | // It would be nice to encode this in the grammar instead of a manual check 632 | } else if (!cases.length) { 633 | throw new Error( 634 | 'A switch statement body must start with a case or default label' 635 | ); 636 | } else { 637 | // While converting this file to Typescript, I don't remember what this 638 | // else case is covering 639 | const tail = cases.slice(-1)[0]; 640 | return [ 641 | ...cases.slice(0, -1), 642 | { 643 | ...tail, 644 | statements: [...(tail as CompoundStatementNode).statements, stmt], 645 | } as AstNode, 646 | ]; 647 | } 648 | }, []); 649 | 650 | context.scope = makeScope('global'); 651 | context.scopes = [context.scope]; 652 | 653 | return { 654 | getLocation, 655 | node, 656 | makeScope, 657 | warn, 658 | pushScope, 659 | popScope, 660 | setScopeEnd, 661 | createFunctionDefinition, 662 | addFunctionCallReference, 663 | createFunctionPrototype, 664 | groupCases, 665 | addTypeReference, 666 | addTypeIfFound, 667 | createType, 668 | createBindings, 669 | addOrCreateBindingReference, 670 | }; 671 | }; 672 | -------------------------------------------------------------------------------- /src/preprocessor/preprocessor.ts: -------------------------------------------------------------------------------- 1 | import { NodeVisitor, Path, visit } from '../ast/visit.js'; 2 | import { 3 | PreprocessorAstNode, 4 | PreprocessorElseIfNode, 5 | PreprocessorIdentifierNode, 6 | PreprocessorIfNode, 7 | PreprocessorLiteralNode, 8 | PreprocessorSegmentNode, 9 | } from './preprocessor-node.js'; 10 | 11 | export type PreprocessorProgram = { 12 | type: string; 13 | program: PreprocessorSegmentNode[]; 14 | wsEnd?: string; 15 | }; 16 | 17 | const without = (obj: object, ...keys: string[]) => 18 | Object.entries(obj).reduce( 19 | (acc, [key, value]) => ({ 20 | ...acc, 21 | ...(!keys.includes(key) && { [key]: value }), 22 | }), 23 | {} 24 | ); 25 | 26 | // Scan for the use of a function-like macro, balancing parentheses until 27 | // encountering a final closing ")" marking the end of the macro use 28 | const scanFunctionArgs = ( 29 | src: string 30 | ): { args: string[]; length: number } | null => { 31 | let char: string; 32 | let parens: number = 0; 33 | let args: string[] = []; 34 | let arg: string = ''; 35 | 36 | for (let i = 0; i < src.length; i++) { 37 | char = src.charAt(i); 38 | 39 | if (char === '(') { 40 | parens++; 41 | } 42 | 43 | if (char === ')') { 44 | parens--; 45 | } 46 | 47 | if (parens === -1) { 48 | // In the case of "()", we don't want to add the argument of empty string, 49 | // but we do in case of "(,)" and "(asdf)". When we hit the closing paren, 50 | // only capture the arg of empty string if there was a previous comma, 51 | // which we can infer from there being a previous arg 52 | if (arg !== '' || args.length) { 53 | args.push(arg); 54 | } 55 | return { args, length: i }; 56 | } 57 | 58 | if (char === ',' && parens === 0) { 59 | args.push(arg); 60 | arg = ''; 61 | } else { 62 | arg += char; 63 | } 64 | } 65 | 66 | return null; 67 | }; 68 | 69 | // From glsl2s https://github.com/cimaron/glsl2js/blob/4046611ac4f129a9985d74704159c41a402564d0/preprocessor/comments.js 70 | const preprocessComments = (src: string): string => { 71 | let i; 72 | let chr; 73 | let la; 74 | let out = ''; 75 | let line = 1; 76 | let in_single = 0; 77 | let in_multi = 0; 78 | 79 | for (i = 0; i < src.length; i++) { 80 | chr = src.substring(i, i + 1); 81 | la = src.substring(i + 1, i + 2); 82 | 83 | // Enter single line comment 84 | if (chr == '/' && la == '/' && !in_single && !in_multi) { 85 | in_single = line; 86 | i++; 87 | continue; 88 | } 89 | 90 | // Exit single line comment 91 | if (chr == '\n' && in_single) { 92 | in_single = 0; 93 | } 94 | 95 | // Enter multi line comment 96 | if (chr == '/' && la == '*' && !in_multi && !in_single) { 97 | in_multi = line; 98 | i++; 99 | continue; 100 | } 101 | 102 | // Exit multi line comment 103 | if (chr == '*' && la == '/' && in_multi) { 104 | // Treat single line multi-comment as space 105 | if (in_multi == line) { 106 | out += ' '; 107 | } 108 | 109 | in_multi = 0; 110 | i++; 111 | continue; 112 | } 113 | 114 | // Newlines are preserved 115 | if ((!in_multi && !in_single) || chr == '\n') { 116 | out += chr; 117 | line++; 118 | } 119 | } 120 | 121 | return out; 122 | }; 123 | 124 | const tokenPaste = (str: string): string => str.replace(/\s+##\s+/g, ''); 125 | 126 | type NodeEvaluator = ( 127 | node: NodeType, 128 | visit: (node: PreprocessorAstNode) => any 129 | ) => any; 130 | 131 | export type NodeEvaluators = Partial< 132 | { 133 | [NodeType in PreprocessorAstNode['type']]: NodeEvaluator< 134 | Extract 135 | >; 136 | } 137 | >; 138 | 139 | const evaluate = (ast: PreprocessorAstNode, evaluators: NodeEvaluators) => { 140 | const visit = (node: PreprocessorAstNode) => { 141 | const evaluator = evaluators[node.type]; 142 | if (!evaluator) { 143 | throw new Error(`No evaluate() evaluator for ${node.type}`); 144 | } 145 | 146 | // I can't figure out why evalutor has node type never here 147 | // @ts-ignore 148 | return evaluator(node, visit); 149 | }; 150 | return visit(ast); 151 | }; 152 | 153 | export type Macro = { 154 | args?: PreprocessorAstNode[]; 155 | body: string; 156 | }; 157 | 158 | export type Macros = { 159 | [name: string]: Macro; 160 | }; 161 | 162 | const expandFunctionMacro = ( 163 | macros: Macros, 164 | macroName: string, 165 | macro: Macro, 166 | text: string 167 | ) => { 168 | const pattern = `\\b${macroName}\\s*\\(`; 169 | const startRegex = new RegExp(pattern, 'm'); 170 | 171 | let expanded = ''; 172 | let current = text; 173 | let startMatch; 174 | 175 | while ((startMatch = startRegex.exec(current))) { 176 | const result = scanFunctionArgs( 177 | current.substring(startMatch.index + startMatch[0].length) 178 | ); 179 | if (result === null) { 180 | throw new Error( 181 | `${current.match(startRegex)} unterminated macro invocation` 182 | ); 183 | } 184 | const macroArgs = (macro.args || []).filter( 185 | (arg) => (arg as PreprocessorLiteralNode).literal !== ',' 186 | ); 187 | const { args, length: argLength } = result; 188 | 189 | // The total length of the raw text to replace is the macro name in the 190 | // text (startMatch), plus the length of the arguments, plus one to 191 | // encompass the closing paren that the scan fn skips 192 | const matchLength = startMatch[0].length + argLength + 1; 193 | 194 | if (args.length > macroArgs.length) { 195 | throw new Error(`'${macroName}': Too many arguments for macro`); 196 | } 197 | if (args.length < macroArgs.length) { 198 | throw new Error(`'${macroName}': Not enough arguments for macro`); 199 | } 200 | 201 | // Collect the macro identifiers and build a replacement map from those to 202 | // the user defined replacements 203 | const argIdentifiers = macroArgs.map( 204 | (a) => (a as PreprocessorIdentifierNode).identifier 205 | ); 206 | 207 | const argKeys = argIdentifiers.reduce>( 208 | (acc, identifier, index) => ({ 209 | ...acc, 210 | // We are scanning from the outside in - so fn(args) - we need to 211 | // recursivey test if args itself has macros to expand. We don't need 212 | // to remove the current macroName from macros because we are scanning 213 | // outside in, so self-referencing macros won't get into an infinite 214 | // loop here 215 | [identifier]: expandMacros(args[index].trim(), macros), 216 | }), 217 | {} 218 | ); 219 | 220 | const replacedBody = tokenPaste( 221 | macro.body.replace( 222 | // Replace all instances of macro arguments in the macro definition 223 | // (the arg separated by word boundaries) with its user defined 224 | // replacement. This one-pass strategy ensures that we won't clobber 225 | // previous replacements when the user supplied args have the same names 226 | // as the macro arguments 227 | new RegExp( 228 | '(' + argIdentifiers.map((a) => `\\b${a}\\b`).join(`|`) + ')', 229 | 'g' 230 | ), 231 | (match) => (match in argKeys ? argKeys[match] : match) 232 | ) 233 | ); 234 | 235 | // Any text expanded is then scanned again for more replacements. The 236 | // self-reference rule means that a macro that references itself won't be 237 | // expanded again, so remove it from the search. WARNING! There is a known 238 | // bug here! See the xtest in preprocessor.test.ts. 239 | const expandedReplace = expandMacros( 240 | replacedBody, 241 | without(macros, macroName) 242 | ); 243 | 244 | // We want to break this string at where we finished expanding the macro 245 | const endOfReplace = startMatch.index + expandedReplace.length; 246 | 247 | // Replace the use of the macro with the expansion 248 | const processed = current.replace( 249 | current.substring(startMatch.index, startMatch.index + matchLength), 250 | expandedReplace 251 | ); 252 | 253 | // Add text up to the end of the expanded macro to what we've procssed 254 | expanded += processed.substring(0, endOfReplace); 255 | 256 | // Only work on the rest of the text, not what we already expanded. This is 257 | // to avoid a nested macro #define foo() foo() where we'll try to expand foo 258 | // forever. With this strategy, we expand foo() to foo() and move on 259 | current = processed.substring(endOfReplace); 260 | } 261 | 262 | return expanded + current; 263 | }; 264 | 265 | const expandObjectMacro = ( 266 | macros: Macros, 267 | macroName: string, 268 | macro: Macro, 269 | text: string 270 | ) => { 271 | const regex = new RegExp(`\\b${macroName}\\b`, 'g'); 272 | let expanded = text; 273 | if (regex.test(text)) { 274 | // Macro definitions like 275 | // #define MACRO 276 | // Have null for the body. Make it empty string if null to avoid 'null' expanded 277 | const replacement = macro.body || ''; 278 | 279 | // Recursively scan this macro body for more replacements, ignoring our own 280 | // macro to avoid the self-reference rule. 281 | const scanned = expandMacros(replacement, without(macros, macroName)); 282 | 283 | expanded = tokenPaste( 284 | text.replace(new RegExp(`\\b${macroName}\\b`, 'g'), scanned) 285 | ); 286 | } 287 | return expanded; 288 | }; 289 | 290 | const expandMacros = (text: string, macros: Macros) => 291 | Object.entries(macros).reduce( 292 | (result, [macroName, macro]) => 293 | macro.args 294 | ? expandFunctionMacro(macros, macroName, macro, result) 295 | : expandObjectMacro(macros, macroName, macro, result), 296 | text 297 | ); 298 | 299 | const isTruthy = (x: any): boolean => !!x; 300 | 301 | // Given an expression AST node, visit it to expand the macro macros to in the 302 | // right places 303 | const expandInExpressions = ( 304 | macros: Macros, 305 | ...expressions: PreprocessorAstNode[] 306 | ) => { 307 | expressions.forEach((expression) => { 308 | visitPreprocessedAst(expression, { 309 | unary_defined: { 310 | enter: (path) => { 311 | path.skip(); 312 | }, 313 | }, 314 | identifier: { 315 | enter: (path) => { 316 | path.node.identifier = expandMacros(path.node.identifier, macros); 317 | }, 318 | }, 319 | }); 320 | }); 321 | }; 322 | 323 | const evaluateIfPart = (macros: Macros, ifPart: PreprocessorAstNode) => { 324 | if (ifPart.type === 'if') { 325 | return evaluteExpression(ifPart.expression, macros); 326 | } else if (ifPart.type === 'ifdef') { 327 | return ifPart.identifier.identifier in macros; 328 | } else if (ifPart.type === 'ifndef') { 329 | return !(ifPart.identifier.identifier in macros); 330 | } 331 | }; 332 | 333 | // TODO: Are all of these operators equivalent between javascript and GLSL? 334 | const evaluteExpression = (node: PreprocessorAstNode, macros: Macros) => 335 | evaluate(node, { 336 | // TODO: Handle non-base-10 numbers. Should these be parsed in the peg grammar? 337 | int_constant: (node) => parseInt(node.token, 10), 338 | unary_defined: (node) => node.identifier.identifier in macros, 339 | identifier: (node) => node.identifier, 340 | group: (node, visit) => visit(node.expression), 341 | binary: ({ left, right, operator: { literal } }, visit) => { 342 | switch (literal) { 343 | // multiplicative 344 | case '*': { 345 | return visit(left) * visit(right); 346 | } 347 | // division 348 | case '/': { 349 | return visit(left) / visit(right); 350 | } 351 | // modulo 352 | case '%': { 353 | return visit(left) % visit(right); 354 | } 355 | // addition 356 | case '+': { 357 | return visit(left) + visit(right); 358 | } 359 | // subtraction 360 | case '-': { 361 | return visit(left) - visit(right); 362 | } 363 | // bit-wise shift 364 | case '<<': { 365 | return visit(left) << visit(right); 366 | } 367 | // bit-wise shift 368 | case '>>': { 369 | return visit(left) >> visit(right); 370 | } 371 | case '<': { 372 | return visit(left) < visit(right); 373 | } 374 | case '>': { 375 | return visit(left) > visit(right); 376 | } 377 | case '<=': { 378 | return visit(left) <= visit(right); 379 | } 380 | case '>=': { 381 | return visit(left) >= visit(right); 382 | } 383 | case '==': { 384 | return visit(left) == visit(right); 385 | } 386 | case '!=': { 387 | return visit(left) != visit(right); 388 | } 389 | // bit-wise and 390 | case '&': { 391 | return visit(left) & visit(right); 392 | } 393 | // bit-wise exclusive or 394 | case '^': { 395 | return visit(left) ^ visit(right); 396 | } 397 | // bit-wise inclusive or 398 | case '|': { 399 | return visit(left) | visit(right); 400 | } 401 | case '&&': { 402 | return visit(left) && visit(right); 403 | } 404 | case '||': { 405 | return visit(left) || visit(right); 406 | } 407 | default: { 408 | throw new Error( 409 | `Preprocessing error: Unknown binary operator ${literal}` 410 | ); 411 | } 412 | } 413 | }, 414 | unary: (node, visit) => { 415 | switch (node.operator.literal) { 416 | case '+': { 417 | return visit(node.expression); 418 | } 419 | case '-': { 420 | return -1 * visit(node.expression); 421 | } 422 | case '!': { 423 | return !visit(node.expression); 424 | } 425 | case '~': { 426 | return ~visit(node.expression); 427 | } 428 | default: { 429 | throw new Error( 430 | `Preprocessing error: Unknown unary operator ${node.operator.literal}` 431 | ); 432 | } 433 | } 434 | }, 435 | }); 436 | 437 | const shouldPreserve = (preserve: NodePreservers = {}) => ( 438 | path: PathOverride 439 | ) => { 440 | const test = preserve?.[path.node.type]; 441 | return typeof test === 'function' ? test(path) : test; 442 | }; 443 | 444 | // HACK: The AST visitors are hard coded to the GLSL AST (not preprocessor AST) 445 | // types. I'm not clever enough to make the core AST type geneeric so that both 446 | // GLSL AST (in ast.ts) and the preprocessed AST can use the same 447 | // visitor/evaluator/path pattern. I took a stab at it but it become tricky to 448 | // track all the nested generics. Instead, I hack re-cast the visit function 449 | // here, which at least gives some minor type safety. 450 | type VisitorOverride = ( 451 | ast: PreprocessorAstNode | PreprocessorProgram, 452 | visitors: { 453 | [NodeType in PreprocessorAstNode['type']]?: NodeVisitor< 454 | Extract 455 | >; 456 | } 457 | ) => void; 458 | 459 | // @ts-ignore 460 | export const visitPreprocessedAst = visit as VisitorOverride; 461 | 462 | type PathOverride = { 463 | node: NodeType; 464 | parent: PreprocessorAstNode | undefined; 465 | parentPath: Path | undefined; 466 | key: string | undefined; 467 | index: number | undefined; 468 | skip: () => void; 469 | remove: () => void; 470 | replaceWith: (replacer: PreprocessorAstNode) => void; 471 | findParent: (test: (p: Path) => boolean) => Path | undefined; 472 | 473 | skipped?: boolean; 474 | removed?: boolean; 475 | replaced?: any; 476 | }; 477 | const convertPath = (p: Path) => 478 | (p as unknown) as PathOverride; 479 | 480 | /** 481 | * Perform the preprocessing logic, aka the "preprocessing" phase of the compiler. 482 | * Expand macros, evaluate conditionals, etc 483 | * TODO: Define the strategy for conditionally removing certain macro types 484 | * and conditionally expanding certain expressions. And take in optiona list 485 | * of pre defined thigns? 486 | * TODO: Handle __LINE__ and other constants. 487 | */ 488 | 489 | export type NodePreservers = { 490 | [nodeType: string]: (path: PathOverride) => boolean; 491 | }; 492 | 493 | export type PreprocessorOptions = { 494 | defines?: { [definitionName: string]: Object }; 495 | preserve?: NodePreservers; 496 | preserveComments?: boolean; 497 | stopOnError?: boolean; 498 | grammarSource?: string; 499 | }; 500 | 501 | // Remove escaped newlines, rather than try to handle them in the grammar 502 | const unescapeSrc = (src: string, options: PreprocessorOptions = {}) => 503 | src.replace(/\\[\n\r]/g, ''); 504 | 505 | const preprocessAst = ( 506 | program: PreprocessorProgram, 507 | options: PreprocessorOptions = {} 508 | ) => { 509 | const macros: Macros = Object.entries(options.defines || {}).reduce( 510 | (defines, [name, body]) => ({ ...defines, [name]: { body } }), 511 | {} 512 | ); 513 | 514 | const { preserve } = options; 515 | const preserveNode = shouldPreserve(preserve); 516 | 517 | visitPreprocessedAst(program, { 518 | conditional: { 519 | enter: (initialPath) => { 520 | const path = convertPath(initialPath); 521 | const { node } = path; 522 | // TODO: Determining if we need to handle edge case conditionals here 523 | if (preserveNode(path)) { 524 | return; 525 | } 526 | 527 | // Expand macros in if/else *expressions* only. Macros are expanded in: 528 | // #if X + 1 529 | // #elif Y + 2 530 | // But *not* in 531 | // # ifdef X 532 | // Because X should not be expanded in the ifdef. Note that 533 | // # if defined(X) 534 | // does have an expression, but the skip() in unary_defined prevents 535 | // macro expansion in there. Checking for .expression and filtering out 536 | // any conditionals without expressions is how ifdef is avoided. 537 | // It's not great that ifdef is skipped differentaly than defined(). 538 | expandInExpressions( 539 | macros, 540 | ...[ 541 | (node.ifPart as PreprocessorIfNode).expression, 542 | ...node.elseIfParts.map( 543 | (elif: PreprocessorElseIfNode) => elif.expression 544 | ), 545 | ].filter(isTruthy) 546 | ); 547 | 548 | if (evaluateIfPart(macros, node.ifPart)) { 549 | path.replaceWith(node.ifPart.body); 550 | } else { 551 | const elseBranchHit = node.elseIfParts.reduce( 552 | (res: boolean, elif: PreprocessorElseIfNode) => 553 | res || 554 | (evaluteExpression(elif.expression, macros) && 555 | // path/visit hack to remove type error 556 | (path.replaceWith(elif.body as PreprocessorAstNode), true)), 557 | false 558 | ); 559 | if (!elseBranchHit) { 560 | if (node.elsePart) { 561 | path.replaceWith(node.elsePart.body as PreprocessorAstNode); 562 | } else { 563 | path.remove(); 564 | } 565 | } 566 | } 567 | }, 568 | }, 569 | text: { 570 | enter: (initialPath) => { 571 | const path = convertPath(initialPath); 572 | path.node.text = expandMacros(path.node.text, macros); 573 | }, 574 | }, 575 | define_arguments: { 576 | enter: (initialPath) => { 577 | const path = convertPath(initialPath); 578 | const { 579 | identifier: { identifier }, 580 | body, 581 | args, 582 | } = path.node; 583 | 584 | macros[identifier] = { args, body }; 585 | !preserveNode(path) && path.remove(); 586 | }, 587 | }, 588 | define: { 589 | enter: (initialPath) => { 590 | const path = convertPath(initialPath); 591 | const { 592 | identifier: { identifier }, 593 | body, 594 | } = path.node; 595 | 596 | macros[identifier] = { body }; 597 | !preserveNode(path) && path.remove(); 598 | }, 599 | }, 600 | undef: { 601 | enter: (initialPath) => { 602 | const path = convertPath(initialPath); 603 | delete macros[path.node.identifier.identifier]; 604 | !preserveNode(path) && path.remove(); 605 | }, 606 | }, 607 | error: { 608 | enter: (initialPath) => { 609 | const path = convertPath(initialPath); 610 | if (options.stopOnError) { 611 | throw new Error(path.node.message); 612 | } 613 | !preserveNode(path) && path.remove(); 614 | }, 615 | }, 616 | pragma: { 617 | enter: (initialPath) => { 618 | const path = convertPath(initialPath); 619 | !preserveNode(path) && path.remove(); 620 | }, 621 | }, 622 | version: { 623 | enter: (initialPath) => { 624 | const path = convertPath(initialPath); 625 | !preserveNode(path) && path.remove(); 626 | }, 627 | }, 628 | extension: { 629 | enter: (initialPath) => { 630 | const path = convertPath(initialPath); 631 | !preserveNode(path) && path.remove(); 632 | }, 633 | }, 634 | // TODO: Causes a failure 635 | line: { 636 | enter: (initialPath) => { 637 | const path = convertPath(initialPath); 638 | !preserveNode(path) && path.remove(); 639 | }, 640 | }, 641 | }); 642 | 643 | // Even though it mutates, useful for passing around functions 644 | return program; 645 | }; 646 | 647 | export { preprocessAst, preprocessComments, unescapeSrc }; 648 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shaderfrog GLSL Compiler 2 | 3 | The [Shaderfrog](https://shaderfrog.com/app) GLSL compiler is an open source 4 | GLSL 1.00 and 3.00 parser and preprocessor that compiles [back to 5 | GLSL](src/parser/generator.ts). Both the parser and preprocessor can preserve 6 | comments and whitespace. 7 | 8 | The parser is built with a PEG grammar, via the [Peggy](https://peggyjs.org) 9 | Javascript library. The PEG grammars for both the preprocessor and parser 10 | are available [on Github](https://github.com/ShaderFrog/glsl-parser). 11 | 12 | See [the state of this library](#state-of-this-library) for limitations and 13 | goals of this compiler. 14 | 15 | # Table of Contents 16 | 17 | - [Installation](#installation) 18 | - [Parsing](#parsing) 19 | - [The program type](#the-program-type) 20 | - [Scope](#scope) 21 | - [Errors](#errors) 22 | - [Preprocessing](#preprocessing) 23 | - [Manipulating and Searching ASTs](#manipulating-and-searching-asts) 24 | - [Visitors](#visitors) 25 | - [Utility Functions](#utility-functions) 26 | - [What are "parsing" and "preprocessing"?](#what-are-parsing-and-preprocessing) 27 | - [State of this library](#state-of-this-library) 28 | - [Limitations of the Parser and Preprocessor](#limitations-of-the-parser-and-preprocessor) 29 | - [Local Development](#local-development) 30 | 31 | # Usage 32 | 33 | ## Installation 34 | 35 | ```bash 36 | npm install --save @shaderfrog/glsl-parser 37 | ``` 38 | 39 | ## Parsing 40 | 41 | ```typescript 42 | import { parse, generate } from '@shaderfrog/glsl-parser'; 43 | 44 | // To parse a GLSL program's source code into an AST: 45 | const program = parse('float a = 1.0;'); 46 | 47 | // To turn a parsed AST back into a source program 48 | const transpiled = generate(program); 49 | ``` 50 | 51 | The parser accepts an optional second `options` argument: 52 | ```js 53 | parse('float a = 1.0;', options); 54 | ``` 55 | 56 | Where `options` is: 57 | 58 | ```typescript 59 | type ParserOptions = { 60 | // The stage of the GLSL source, which defines the built in variables when 61 | // parsing, like gl_Position. If specified, when the compiler encounters an 62 | // expected variable for that stage, it will continue on normally (and add 63 | // that variable, like gl_Position, to the current scope). If it encounters an 64 | // unknown variable, it will log a warning or raise an exception, depending 65 | // on if `failOnWarn` is set. If the stage is set to 'either' - used in the 66 | // case when you don't yet know what stage the GLSL is, the compiler *won't* 67 | // warn on built-in variables for *either* stage. If this option is not set, 68 | // the compiler will warn on *all* built-in variables it encounters as not 69 | // defined. 70 | stage: 'vertex' | 'fragment' | 'either'; 71 | 72 | // Hide warnings. If set to false or not set, then the parser logs warnings 73 | // like undefined functions and variables. If `failOnWarn` is set to true, 74 | // warnings will still cause the parser to raise an error. Defaults to false. 75 | quiet?: boolean; 76 | 77 | // If true, sets location information on each AST node, in the form of 78 | // { column: number, line: number, offset: number }. Defaults to false. 79 | includeLocation?: boolean; 80 | 81 | // If true, causes the parser to raise an error instead of log a warning. 82 | // The parser does limited type checking, and things like undeclared variables 83 | // are treated as warnings. Defaults to false. 84 | failOnWarn?: boolean; 85 | 86 | // An optional string representing the file name of your GLSL For example, 87 | // you can pass "main.glsl" here. If an error is raised, the error message 88 | // will show "main.glsl:row:column". This is only used in the error message. 89 | grammarSource?: string; 90 | } 91 | ``` 92 | 93 | ## The program type 94 | `parse()` returns a `Program`, which is a special AST Node: 95 | 96 | ```typescript 97 | interface Program { 98 | // Hard coded AST node type of "program" 99 | type: 'program'; 100 | // The AST itself is an array of the top level statements 101 | program: AstNode[]; 102 | // All of the scopes found during parsing 103 | scopes: Scope[]; 104 | // Leading whitespace of the source code 105 | wsStart?: string; 106 | // Trailing whitespace of the source code 107 | wsEnd?: string; 108 | } 109 | ``` 110 | 111 | ## Scope 112 | 113 | `parse()` returns a [`Program`](#the-program-type), which has a `scopes` array on it. A scope looks 114 | like: 115 | ```typescript 116 | type Scope = { 117 | name: string; 118 | parent?: Scope; 119 | bindings: ScopeIndex; 120 | types: TypeScopeIndex; 121 | functions: FunctionScopeIndex; 122 | location?: LocationObject; 123 | } 124 | ``` 125 | 126 | The `name` of a scope is either `"global"`, the name of the function that 127 | introduced the scope, or in anonymous blocks, `"{"`. In each scope, `bindings` represents variables, 128 | `types` represents user-created types (structs in GLSL), and `functions` represents 129 | functions. 130 | 131 | For `bindings` and `types`, the scope index looks like: 132 | ```typescript 133 | type ScopeIndex = { 134 | [name: string]: { 135 | declaration?: AstNode; 136 | references: AstNode[]; 137 | } 138 | } 139 | ``` 140 | 141 | Where `name` is the name of the variable or type. `declaration` is the AST node 142 | where the variable was declared. In the case the variable is used without being 143 | declared, `declaration` won't be present. If you set the [`failOnWarn` parser 144 | option](#Parsing) to `true`, the parser will throw an error when encountering 145 | an undeclared variable, rather than allow a scope entry without a declaration. 146 | 147 | For `functions`, the scope index is slightly different: 148 | ```typescript 149 | type FunctionScopeIndex = { 150 | [name: string]: { 151 | [signature: string]: { 152 | returnType: string; 153 | parameterTypes: string[]; 154 | declaration?: FunctionNode; 155 | references: AstNode[]; 156 | } 157 | } 158 | }; 159 | ``` 160 | 161 | Where `name` is the name of the function, and `signature` is a string representing 162 | the function's return and parameter types, in the form of `"returnType: paramType1, paramType2, ..."` 163 | or `"returnType: void"` in the case of no arguments. Each `signature` in this 164 | index represents an "overloaded" function in GLSL, as in: 165 | 166 | ```glsl 167 | void someFunction(int x) {}; 168 | void someFunction(int x, int y) {}; 169 | ``` 170 | 171 | With this source code, there will be two entries under `name`, one for each 172 | overload signature. The `references` are the uses of that specific overloaded 173 | version of the function. `references` also contains the function prototypes 174 | for the overloaded function, if present. 175 | 176 | In the case there is only one declaration for a function, there will still be 177 | a single entry under `name` with the function's `signature`. 178 | 179 | ⚠️ Caution! This parser does very limited type checking. This leads to a known 180 | case where a function call can match to the wrong overload in scope: 181 | 182 | ```glsl 183 | void someFunction(float, float); 184 | void someFunction(bool, bool); 185 | someFunction(true, true); // This will be attributed to the wrong scope entry 186 | ``` 187 | 188 | The parser doesn't know the type of the operands in the function call, so it 189 | matches based on the name and arity of the functions. 190 | 191 | See also [Utility-Functions](#Utility-Functions) for renaming scope references. 192 | 193 | ## Errors 194 | 195 | If you have invalid GLSL, the parser throws a `GlslSyntaxError`. 196 | 197 | ```ts 198 | import { parse, GlslSyntaxError } from '@shaderfrog/glsl-parser'; 199 | 200 | let error: GlslSyntaxError | undefined; 201 | try { 202 | // Line without a semicolon 203 | parse(`float a`); 204 | } catch (e) { 205 | error = e as GlslSyntaxError; 206 | } 207 | ``` 208 | 209 | If you want to check if a caught error is an `instanceof` a `GlslSyntaxError`: 210 | ```ts 211 | console.log(error instanceof GlslSyntaxError) 212 | // true 213 | ``` 214 | 215 | The only error the parser intentionally throws is a `GlslSyntaxError`. You 216 | should be safe to cast it to a `GlslSyntaxError` with `as` in Typescript. 217 | 218 | The error message string is automatically generated by Peggy: 219 | ```ts 220 | console.log(error.message) 221 | /* 222 | Error: Expected ",", ";", "=", or array specifier but "f" found. 223 | --> undefined:2:1 224 | | 225 | 2 | float b 226 | | ^ 227 | */ 228 | ``` 229 | 230 | The error object includes the location of the error. 231 | 232 | ```ts 233 | console.log(error.location) 234 | /* 235 | { 236 | source: undefined, 237 | start: { offset: 7, line: 1, column: 8 }, 238 | end: { offset: 7, line: 1, column: 8 } 239 | } 240 | */ 241 | ``` 242 | Note the `source` field on the error object is the `grammarSource` string 243 | provided to the parser options, which is `undefined` by default. If you pass in 244 | a `grammarSource` to `parse()`, it shows up in the error object. This is 245 | meant to help you track which source file you're parsing, for example you could 246 | enter `"myfile.glsl"` as an argument to `parse()` so that the error 247 | includes that your source GLSL came from your application's `myfile.glsl` file. 248 | 249 | ## Preprocessing 250 | 251 | The parser also ships with a `preprocess()` function. 252 | 253 | The preprocessor takes in a program source code string and produces a 254 | preprocessed program source code string. If you want to access and manipulate 255 | the AST produced by preprocessing, see the next sections. 256 | 257 | ```typescript 258 | import preprocess from '@shaderfrog/glsl-parser/preprocessor'; 259 | 260 | // Preprocess a program 261 | console.log(preprocess(` 262 | #define a 1 263 | float b = a; 264 | `, options)); 265 | ``` 266 | 267 | Where `options` is: 268 | 269 | ```typescript 270 | type PreprocessorOptions = { 271 | // Don't strip comments before preprocessing 272 | preserveComments: boolean, 273 | // Macro definitions to use when preprocessing 274 | defines: { 275 | SOME_MACRO_NAME: 'macro body' 276 | }, 277 | // A list of callbacks evaluated for each node type, and returns whether or not 278 | // this AST node is subject to preprocessing 279 | preserve: { 280 | ast_node_name: (path) => boolean 281 | } 282 | } 283 | ``` 284 | 285 | A preprocessed program string can be handed off to the main GLSL parser. 286 | Preprocessing is optional, but a program string may not be valid until it is 287 | preprocessed. 288 | 289 | If you want more control over preprocessing, the `preprocess` function above is 290 | a convenience method for approximately the following: 291 | 292 | ```typescript 293 | import { 294 | preprocessAst, 295 | preprocessComments, 296 | generate, 297 | parse, 298 | } from '@shaderfrog/glsl-parser/preprocessor'; 299 | 300 | // Remove comments before preprocessing 301 | const commentsRemoved = preprocessComments(`float a = 1.0;`); 302 | 303 | // Parse the source text into an AST 304 | const ast = parse(commentsRemoved); 305 | 306 | // Then preprocess it, expanding #defines, evaluating #ifs, etc 307 | preprocessAst(ast); 308 | 309 | // Then convert it back into a program string, which can be passed to the 310 | // core glsl parser 311 | const preprocessed = generate(ast); 312 | ``` 313 | 314 | ## Accessing the raw Peggy parser 315 | 316 | If you need to access the Peggy compiled parser directly, import the `parser`. 317 | You can manually call `parser.parse()` if you prefer. Note if there are errors, 318 | rather than returning a `GlslSyntaxError`, it will return the underlying 319 | `peg$syntaxError`. 320 | 321 | ```typescript 322 | import { parser } from '@shaderfrog/glsl-parser'; 323 | import { parser as preprocessorParser } from '@shaderfrog/glsl-parser/preprocessor'; 324 | 325 | try { 326 | const ast = parser.parse('float f = 1.0;'); 327 | } catch (e) { 328 | console.log(e instanceof parser.SyntaxError); // true 329 | } 330 | ``` 331 | 332 | ## Manipulating and Searching ASTs 333 | 334 | ### Visitors 335 | 336 | The Shaderfrog parser provides a AST visitor function for manipulating and 337 | searching an AST. The visitor API loosely follows the [Babel visitor API](https://github.com/jamiebuilds/babel-handbook/blob/master/translations/en/plugin-handbook.md#toc-visitors). A visitor object looks 338 | like: 339 | 340 | ```typescript 341 | const visitors = { 342 | function_call: { 343 | enter: (path) => {}, 344 | exit: (path) => {}, 345 | } 346 | } 347 | ``` 348 | 349 | Where every key in the object is a node type, and every value is an object 350 | with optional `enter` and `exit` functions. What's passed to each function 351 | is **not** the AST node itself, instead it's a "path" object, which gives you 352 | information about the node's parents, methods to manipulate the node, and the 353 | node itself. The path object: 354 | 355 | ```typescript 356 | { 357 | // Properties: 358 | 359 | // The node itself 360 | node: AstNode; 361 | // The parent of this node 362 | parent: AstNode | null; 363 | // The parent path of this path 364 | parentPath: Path | null; 365 | // The key of this node in the parent object, if node parent is an object 366 | key: string | null; 367 | // The index of this node in the parent array, if node parent is an array 368 | index: number | null; 369 | 370 | // Methods: 371 | 372 | // Don't visit any children of this node 373 | skip: () => void; 374 | // Stop traversal entirely 375 | stop: () => void; 376 | // Remove this node from the AST 377 | remove: () => void; 378 | // Replace this node with another AST node. See replaceWith() documentation. 379 | replaceWith: (replacer: any) => void; 380 | // Search for parents of this node's parent using a test function 381 | findParent: (test: (p: Path) => boolean) => Path | null; 382 | } 383 | ``` 384 | 385 | Visit an AST by calling the visit method with an AST and visitors: 386 | 387 | ```typescript 388 | import { visit } from '@shaderfrog/glsl-parser/ast'; 389 | 390 | visit(ast, visitors); 391 | ``` 392 | 393 | The visit function doesn't return a value. If you want to collect data from the 394 | AST, use a variable in the outer scope to collect data. For example: 395 | 396 | ```typescript 397 | let numberOfFunctionCalls = 0; 398 | visit(ast, { 399 | function_call: { 400 | enter: (path) => { 401 | numberOfFunctionCalls += 1; 402 | }, 403 | } 404 | }); 405 | console.log('There are ', numberOfFunctionCalls, 'function calls'); 406 | ``` 407 | 408 | You can also visit the preprocessed AST with `visitPreprocessedAst`. Visitors 409 | follow the same convention outlined above. 410 | 411 | ```typescript 412 | import { 413 | parse, 414 | visitPreprocessedAst, 415 | } from '@shaderfrog/glsl-parser/preprocessor'; 416 | 417 | // Parse the source text into an AST 418 | const ast = parse(`float a = 1.0;`); 419 | visitPreprocessedAst(ast, visitors); 420 | ``` 421 | 422 | ### Stopping traversal 423 | 424 | To skip all children of a node, call `path.skip()`. 425 | 426 | To stop traversal entirely, call `path.stop()` in either `enter()` or `exit()`. 427 | No future `enter()` nor `exit()` callbacks will fire. 428 | 429 | ### Visitor `.replaceWith()` Behavior 430 | 431 | When you visit a node and call `path.replaceWith(otherNode)` inside the visitor's `enter()` method: 432 | 1. `otherNode` and its children **are** visited by the same visitors. 433 | 2. The `exit()` function of the visitor **is not called.** 434 | 435 | Notes: 436 | - Calling `.replaceWith()` in a visitor's `exit()` method is undefined behavior. 437 | - Replacing a node with the same type can cause infinite recursion, as the 438 | visitor will continue to visit the replaced node of the same type. You must 439 | handle this case manually. 440 | 441 | These rules apply to all visitors, both GLSL AST visitors and preprocessor AST 442 | visitors. 443 | 444 | ### Utility Functions 445 | 446 | #### Rename variables / identifiers in a program 447 | 448 | You can rename bindings (aka variables), functions, and types (aka structs) with `renameBindings`, `renameFunctions`, and `renameTypes` respectively. 449 | 450 | The signature for these methods: 451 | 452 | ```ts 453 | const renameBindings = ( 454 | // The scope to rename the bindings in. ast.scopes[0] is the global scope. 455 | // Passing this ast.scopes[0] renames all global variables 456 | bindings: ScopeIndex, 457 | 458 | // The rename function. This is called once per scope entry with the original 459 | // name in the scope, to generate the renamed variable. 460 | mangle: (name: string) => string 461 | ): ScopeIndex 462 | ``` 463 | 464 | These scope renaming functions, `renameBindings`, `renameFunctions`, and `renameTypes`, do two things: 465 | 1. Each function *mutates* the AST to rename identifiers in place. 466 | 2. They *return* an *immutable* new ScopeIndex where the scope references 467 | themselves are renamed. 468 | 469 | If you want your ast.scopes array to stay in sync with your AST, you need to 470 | re-assign it to the output of the functions! Examples: 471 | 472 | ```typescript 473 | import { renameBindings, renameFunctions, renameTypes } from '@shaderfrog/glsl-parser/parser/utils'; 474 | 475 | // Suffix top level variables with _x, and update the scope 476 | ast.scopes[0].bindings = renameBindings(ast.scopes[0].bindings, (name) => `${name}_x`); 477 | 478 | // Suffix function names with _x 479 | ast.scopes[0].functions = renameFunctions(ast.scopes[0].functions, (name) => `${name}_x`); 480 | 481 | // Suffix struct names and usages (including constructors) with _x 482 | ast.scopes[0].types = renameTypes(ast.scopes[0].types, (name) => `${name}_x`); 483 | ``` 484 | 485 | There are also functions to rename only one variable/identifier in a given 486 | scope. Use these if you know specifically which variable you want to rename. 487 | 488 | ```typescript 489 | import { renameBinding, renameFunction, renameType } from '@shaderfrog/glsl-parser/parser/utils'; 490 | 491 | // Replace all instances of "oldVar" with "newVar" in the global scope, and 492 | // creates a new global scope entry named "newVar" 493 | ast.scopes[0].bindings.newVar = renameBinding( 494 | ast.scopes[0].bindings.oldVar, 495 | 'newVar', 496 | ); 497 | // You need to manually delete the old scope entry if you want the scope to stay 498 | // in sync with your program AST 499 | delete ast.scopes[0].bindings.oldVar; 500 | 501 | // Rename a specific function 502 | ast.scopes[0].functions.newFn = renameFunction( 503 | ast.scopes[0].functions.oldFn, 504 | 'newFn', 505 | ); 506 | delete ast.scopes[0].functions.oldFn; 507 | 508 | // Rename a specific type/struct 509 | ast.scopes[0].functions.newType = renametype( 510 | ast.scopes[0].functions.oldType, 511 | 'newType', 512 | ); 513 | delete ast.scopes[0].functions.oldType; 514 | ``` 515 | 516 | #### Debugging utility functions 517 | 518 | The parser also exports a debugging function, useful for logging information 519 | about the AST. 520 | 521 | ```ts 522 | import { debugScopes } from '@shaderfrog/glsl-parser/parser/utils'; 523 | 524 | // Print a condensed representation of the AST scopes to the console 525 | debugScopes(ast); 526 | ``` 527 | 528 | ## What are "parsing" and "preprocessing"? 529 | 530 | In general, a parser is a computer program that analyzes source code and turn it 531 | into a data structure called an "abstract syntax tree" (AST). The AST is a tree 532 | representation of the source program, which can be analyzed or manipulated. A 533 | use of this GLSL parser could be to parse a program into an AST, find all 534 | variable names in the AST, rename them, and generate new GLSL source code with 535 | renamed variables. 536 | 537 | Preprocessing is a text manipulation step supported in shader source code. One 538 | way to think about preprocessing is it's a glorified find and replace language 539 | that's also part of your program's source code. Special lines starting with `#` 540 | tell the preprocessor how to manipulate other text in the program source code. 541 | Some programs are not parsable until they are preprocessed, because the source 542 | code may be invalid until the text find and replacements are done. During 543 | preprocessing, the source code is treated purely as text. Said another way, 544 | there is no consideration for the GLSL source code structure, like `float`, 545 | `vec2`, etc. Preprocessing only handles special lines and rules starting with 546 | `#`. 547 | 548 | See the [GLSL Langauge Spec](https://www.khronos.org/registry/OpenGL/specs/gl/GLSLangSpec.4.60.pdf) to learn more about GLSL preprocessing. Some notable 549 | differences from the C++ parser are no "stringize" operator (`#`), no `#include` 550 | operator, and `#if` expressions can only operate on integer constants, not other 551 | types of data. The Shaderfrog GLSL preprocessor can't be used as a C/C++ 552 | preprocessor without modification. 553 | 554 | Parsing, preprocessing, and code generation, are all phases of a compiler. This 555 | library is technically a source code > source code compiler, also known as a 556 | "transpiler." The input and output source code are both GLSL. 557 | 558 | # State of this library 559 | 560 | The Shaderfrog compiler [has tests](parser/parse.test.ts) for the more complex 561 | parts of the GLSL ES 3.00 grammar. This library is definitively the most 562 | complete GLSL compiler written in **Javascript.** 563 | 564 | This library is used by the experimental [Shaderfrog 2.0 shader 565 | composer](https://twitter.com/andrewray/status/1558307538063437826). The 566 | compiler has wide expoure to different GLSL programs. 567 | 568 | This library also exposed: 569 | - [A typo](https://github.com/KhronosGroup/GLSL/issues/161) in the official GLSL grammar specification. 570 | - [A bug](https://bugs.chromium.org/p/angleproject/issues/detail?id=6338#c1) in Chrome's ANGLE compiler. 571 | 572 | This library doesn't support full "semantic analysis" required by the Khronos 573 | GLSL specification. For example, some tokens are only valid in GLSL 1.00 vs 574 | 3.00, like `texture()` vs `texture2D()`. This parser considers both valid as 575 | they're both part of the grammar. However if you send compiled source code off 576 | to a native compiler like ANGLE with the wrong `texture` function, it will fail 577 | to compile. 578 | 579 | This library is mainly for manipulating ASTs before handing off a generated 580 | program to a downstream compilers like as ANGLE. 581 | 582 | The preprocessor supports full macro evaluations and expansions, with the 583 | exceptions of `__LINE__`. Additional control lines like `#error` and `#pragma` 584 | and `#extension` have no effect, and can be fully preserved as part of parsing. 585 | 586 | # Limitations of the Parser and Preprocessor 587 | 588 | ## Known Issues 589 | 590 | - There's probably some bugs in the preprocessor logic. I haven't yet verified 591 | all of the evaluations of "binary" expressions in `preprocessor.ts` 592 | - `preprocessor.ts` has lots of yucky typecasting 593 | 594 | ## Known missing semantic analysis compared to the specification 595 | 596 | - Compilers are supposed to raise an error if a switch body ends in a case or 597 | default label. 598 | - Currently no semantic analysis of vertex vs fragment shaders 599 | 600 | ## Deviations from the Khronos Grammar 601 | 602 | - The parser allows for programs to contain top level preprocessor statements, 603 | like `#define X`, as a convenience to avoid preprocessing some programs, and 604 | for simple programs lets you preserve preprocessor lines in the AST if you 605 | want to do something specific with them. However, preprocessor statements at 606 | any other part of the program are not allowed. The Khronos grammar does not 607 | include preprocessor statements. 608 | - `selection_statement` is renamed to `if_statement` 609 | - The grammar specifies `declaration` itself ends with a semicolon. I moved the 610 | semicolon into the `declaration_statement` rule. 611 | - The grammar has a left paren "(" in the function_call production. Due to how 612 | I de-left-recursed the function_call -> postfix_expression loop, I moved the 613 | left paren into the function_identifier production. 614 | - Function calls in the grammar are TYPE_NAME LEFT_PAREN, in my grammar they're 615 | IDENTIFIER LEFT_PAREN, because TYPE_NAME is only used for structs, and 616 | function names are stored in their own separate place in the scope. 617 | 618 | # Local Development 619 | 620 | To work on the tests, run `npx jest --watch`. 621 | 622 | The GLSL grammar definition lives in `src/parser/glsl-grammar.pegjs`. Peggy 623 | supports inlining Javascript code in the `.pegjs` file to define utility 624 | functions, but that means you have to write in vanilla Javascript, which is 625 | terrible. Instead, I've pulled out utility functions into the `grammar.ts` 626 | entrypoint. Some functions need access to Peggy's local variables, like 627 | `location(s)`, so the `makeLocals()` function uses a closure to provide that 628 | access. 629 | 630 | To submit a change, please open a pull request. Tests are appreciated! 631 | 632 | See [the Github workflow](.github/workflows/main.yml) for the checks run against 633 | each PR. 634 | --------------------------------------------------------------------------------