├── .gitignore ├── README.md ├── bin └── lz.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /common/ 3 | /cpp/ 4 | /css/ 5 | /generator/ 6 | /highlight/ 7 | /html/ 8 | /import-tree-sitter/ 9 | /java/ 10 | /javascript/ 11 | /json/ 12 | /lezer/ 13 | /lr/ 14 | /markdown/ 15 | /php/ 16 | /python/ 17 | /rust/ 18 | /website/ 19 | /xml/ 20 | /sass/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lezer 2 | 3 | This repository holds the bug tracker for the [Lezer parser 4 | system](https://lezer.codemirror.net/). The actual code is in other 5 | repositories: 6 | 7 | - [common](https://github.com/lezer-parser/common) for the shared 8 | data structures. 9 | - [lr](https://github.com/lezer-parser/lr) for the LR parser runtime. 10 | - [generator](https://github.com/lezer-parser/generator) for the LR 11 | parser generator. 12 | - Individual 13 | [repositories](https://github.com/orgs/lezer-parser/repositories) 14 | for the various grammars and languages. 15 | -------------------------------------------------------------------------------- /bin/lz.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const child = require("child_process"), fs = require("fs"), fsp = fs.promises, path = require("path") 4 | 5 | let root = path.join(__dirname, "..") 6 | 7 | class Pkg { 8 | constructor(name, options = {}) { 9 | this.name = name 10 | this.entry = options.entry || "index" 11 | this.dir = path.join(root, name) 12 | this.options = options 13 | this._dependencies = null 14 | } 15 | 16 | get sources() { 17 | let src = path.join(this.dir, "src") 18 | return fs.readdirSync(src).filter(file => /\.ts$/.test(file)).map(file => path.join(src, file)) 19 | } 20 | 21 | get declarations() { 22 | let dist = path.join(this.dir, "dist") 23 | return !fs.existsSync(dist) ? [] : 24 | fs.readdirSync(dist).filter(file => /\.d\.ts$/.test(file)).map(file => path.join(dist, file)) 25 | } 26 | 27 | get entrySource() { 28 | return path.join(this.dir, "src", this.entry + ".ts") 29 | } 30 | 31 | get esmFile() { 32 | return path.join(this.dir, "dist", "index.es.js") 33 | } 34 | 35 | get cjsFile() { 36 | return path.join(this.dir, "dist", "index.js") 37 | } 38 | 39 | get dependencies() { 40 | if (!this._dependencies) { 41 | this._dependencies = [] 42 | for (let file of this.sources) { 43 | let text = fs.readFileSync(file, "utf8") 44 | let imp = /(?:^|\n)\s*import.* from "\.\.\/\.\.\/([\w-]+)"/g, m 45 | while (m = imp.exec(text)) 46 | if (!this._dependencies.includes(m[1]) && packageNames[m[1]]) 47 | this._dependencies.push(packageNames[m[1]]) 48 | } 49 | } 50 | return this._dependencies 51 | } 52 | 53 | get inputFiles() { 54 | return this.sources.concat(this.dependencies.reduce((arr, dep) => arr.concat(dep.declarations), [])) 55 | } 56 | 57 | rollupConfig(options) { 58 | return this._rollup || (this._rollup = { 59 | input: this.entrySource, 60 | external(id) { return id != "tslib" && !/^\.?\//.test(id) }, 61 | output: [...options.esm ? [{ 62 | format: "es", 63 | file: this.esmFile, 64 | externalLiveBindings: false 65 | }] : [], { 66 | format: "cjs", 67 | file: this.cjsFile, 68 | externalLiveBindings: false 69 | }], 70 | plugins: [tsPlugin(this.dir, { 71 | lib: ["es6", "scripthost"], 72 | target: "es6", 73 | declaration: true 74 | })], 75 | }) 76 | } 77 | } 78 | 79 | const baseCompilerOptions = { 80 | noImplicitReturns: false, 81 | noUnusedLocals: false, 82 | } 83 | 84 | function tsPlugin(cwd, options) { 85 | return require("rollup-plugin-typescript2")({ 86 | clean: true, 87 | tsconfig: path.join(cwd, "tsconfig.json"), 88 | tsconfigOverride: { 89 | references: [], 90 | compilerOptions: {...baseCompilerOptions, ...options}, 91 | include: [] 92 | } 93 | }) 94 | } 95 | 96 | function loadPackages() { 97 | const packages = [ 98 | new Pkg("common"), 99 | new Pkg("highlight", {entry: "highlight"}), 100 | new Pkg("lr"), 101 | new Pkg("generator", {node: true}), 102 | new Pkg("javascript", {grammar: true}), 103 | new Pkg("css", {grammar: true}), 104 | new Pkg("sass", {grammar: true}), 105 | new Pkg("html", {grammar: true}), 106 | new Pkg("xml", {grammar: true}), 107 | new Pkg("cpp", {grammar: true}), 108 | new Pkg("java", {grammar: true}), 109 | new Pkg("python", {grammar: true}), 110 | new Pkg("json", {grammar: true}), 111 | new Pkg("rust", {grammar: true}), 112 | new Pkg("lezer", {grammar: true}), 113 | new Pkg("php", {grammar: true}), 114 | new Pkg("markdown"), 115 | ] 116 | const packageNames = Object.create(null) 117 | for (let pkg of packages) packageNames[pkg.name] = pkg 118 | return {packages, packageNames} 119 | } 120 | 121 | let {packages, packageNames} = loadPackages() 122 | 123 | function start() { 124 | let command = process.argv[2] 125 | let args = process.argv.slice(3) 126 | let cmdFn = { 127 | install, 128 | packages: listPackages, 129 | release, 130 | status, 131 | "bump-deps": bumpDeps, 132 | run: runCmd, 133 | "--help": () => help(0), 134 | notes 135 | }[command] 136 | if (!cmdFn || cmdFn.length > args.length) help(1) 137 | new Promise(r => r(cmdFn.apply(null, args))).catch(e => error(e)) 138 | } 139 | 140 | function help(status) { 141 | console.log(`Usage: 142 | lz packages Emit a list of all pkg names 143 | lz install [--ssh] Clone the packages and install deps 144 | lz release Tag a release 145 | lz run [--cont] Run the given command in all packages 146 | lz notes Emit pending release notes 147 | lz status Display the git status of packages 148 | lz --help`) 149 | process.exit(status) 150 | } 151 | 152 | function error(err) { 153 | console.error(err) 154 | process.exit(1) 155 | } 156 | 157 | function run(cmd, args, wd = root, out = "pipe") { 158 | return child.execFileSync(cmd, args, {cwd: wd, encoding: "utf8", stdio: ["ignore", out, process.stderr]}) 159 | } 160 | 161 | function install(arg = null) { 162 | let base = arg == "--ssh" ? "git@github.com:lezer-parser/" : "https://github.com/lezer-parser/" 163 | if (arg && arg != "--ssh") help(1) 164 | 165 | for (let pkg of packages) { 166 | if (fs.existsSync(pkg.dir)) { 167 | console.warn(`Skipping cloning of ${pkg.name} (directory exists)`) 168 | } else { 169 | let origin = base + (pkg.name == "lezer" ? "lezer-grammar" : pkg.name) + ".git" 170 | run("git", ["clone", origin, pkg.dir]) 171 | } 172 | } 173 | 174 | // Horrible hack to work around npm trying to build a workspace's 175 | // packages in some arbitrary order (see https://github.com/npm/rfcs/issues/548) 176 | updatePackageFiles(json => json.replace(/"prepare"/, '"prepareDISABLED"')) 177 | console.log("Running npm install") 178 | try { 179 | run("npm", ["install", "--ignore-scripts"]) 180 | } finally { 181 | updatePackageFiles(json => json.replace(/"prepareDISABLED"/, '"prepare"')) 182 | } 183 | ;({packages, packageNames} = loadPackages()) 184 | console.log("Building packages") 185 | for (let pkg of packages) 186 | run("npm", ["run", "prepare"], pkg.dir) 187 | } 188 | 189 | function updatePackageFiles(f) { 190 | for (let pkg of packages) { 191 | let file = path.join(pkg.dir, "package.json") 192 | fs.writeFileSync(file, f(fs.readFileSync(file, "utf8"))) 193 | } 194 | } 195 | 196 | function listPackages() { 197 | console.log(packages.map(p => p.name).join("\n")) 198 | } 199 | 200 | function status() { 201 | for (let pkg of packages) { 202 | let output = run("git", ["status", "-sb"], pkg.dir) 203 | if (output != "## main...origin/main\n") 204 | console.log(`${pkg.name}:\n${output}`) 205 | } 206 | } 207 | 208 | function changelog(pkg, since, extra) { 209 | let commits = run("git", ["log", "--format=%B", "--reverse", since + "..main"], pkg.dir) 210 | if (extra) commits = "\n\n" + extra + "\n\n" + commits 211 | let result = {fix: [], feature: [], breaking: []} 212 | let re = /\n\r?\n(BREAKING|FIX|FEATURE):\s*([^]*?)(?=\r?\n\r?\n|\r?\n?$)/g, match 213 | while (match = re.exec(commits)) result[match[1].toLowerCase()].push(match[2].replace(/\r?\n/g, " ")) 214 | return result 215 | } 216 | 217 | function bumpVersion(version, changes) { 218 | let [major, minor, patch] = version.split(".") 219 | if (changes.breaking.length && major != "0") return `${Number(major) + 1}.0.0` 220 | if (changes.feature.length && major != "0" || changes.breaking.length) return `${major}.${Number(minor) + 1}.0` 221 | if (changes.fix.length || changes.feature.length) return `${major}.${minor}.${Number(patch) + 1}` 222 | throw new Error("No new release notes!") 223 | } 224 | 225 | function releaseNotes(changes, version) { 226 | let pad = n => n < 10 ? "0" + n : n 227 | let d = new Date, date = d.getFullYear() + "-" + pad(d.getMonth() + 1) + "-" + pad(d.getDate()) 228 | 229 | let types = {breaking: "Breaking changes", fix: "Bug fixes", feature: "New features"} 230 | 231 | let refTarget = "https://lezer.codemirror.net/docs/ref/" 232 | let head = `## ${version} (${date})\n\n`, body = "" 233 | for (let type in types) { 234 | let messages = changes[type] 235 | if (messages.length) body += `### ${types[type]}\n\n` 236 | messages.forEach(message => body += message.replace(/\]\(##/g, "](" + refTarget + "#") + "\n\n") 237 | } 238 | return {head, body} 239 | } 240 | 241 | function setModuleVersion(pkg, version) { 242 | let file = path.join(pkg.dir, "package.json") 243 | fs.writeFileSync(file, fs.readFileSync(file, "utf8").replace(/"version":\s*".*?"/, `"version": "${version}"`)) 244 | } 245 | 246 | function version(pkg) { 247 | return require(path.join(pkg.dir, "package.json")).version 248 | } 249 | 250 | function doRelease(pkg, changes, newVersion) { 251 | setModuleVersion(pkg, newVersion) 252 | let notes = releaseNotes(changes, newVersion) 253 | let log = path.join(pkg.dir, "CHANGELOG.md") 254 | fs.writeFileSync(log, notes.head + notes.body + fs.readFileSync(log, "utf8")) 255 | run("git", ["add", "package.json"], pkg.dir) 256 | run("git", ["add", "CHANGELOG.md"], pkg.dir) 257 | run("git", ["commit", "-m", `Mark version ${newVersion}`], pkg.dir) 258 | run("git", ["tag", newVersion, "-m", `Version ${newVersion}\n\n${notes.body}`, "--cleanup=verbatim"], pkg.dir) 259 | } 260 | 261 | function release(pkgName, ...args) { 262 | let pkg = packageNames[pkgName] 263 | if (!pkg) error(`No package ${pkgName} known`) 264 | let newVersion = null, message = "" 265 | for (let i = 0; i < args.length; i++) { 266 | if (args[i] == "--version") newVersion = args[++i] 267 | else if (args[i] == "-m") message += args[++i] + "\n\n" 268 | else error("Invalid arguments to release " + args.join()) 269 | } 270 | let currentVersion = version(pkg) 271 | let changes = changelog(pkg, currentVersion, message) 272 | if (!newVersion) newVersion = bumpVersion(currentVersion, changes) 273 | console.log(`Creating ${pkgName} ${newVersion}`) 274 | doRelease(pkg, changes, newVersion) 275 | } 276 | 277 | function bumpDeps(version) { 278 | for (let pkg of packages) { 279 | let file = path.join(pkg.dir, "package.json") 280 | fs.writeFileSync(file, fs.readFileSync(file, "utf8").replace(/"@lezer\/([\w-]+)":\s*"\^?\d+\.\d+\.\d+"/g, `"@lezer\/$1": "^${version}"`)) 281 | } 282 | } 283 | 284 | function runCmd(...args) { 285 | let cont = args[0] == "--cont" 286 | if (cont) args.shift() 287 | for (let pkg of packages) { 288 | try { run(args[0], args.slice(1), pkg.dir, process.stdout) } 289 | catch(e) { 290 | console.error(e + "") 291 | if (!cont) process.exit(1) 292 | } 293 | } 294 | } 295 | 296 | function notes(name) { 297 | let pkg = packageNames[name] 298 | if (!pkg) error(`No package ${name} known`) 299 | let notes = releaseNotes(changelog(pkg, version(pkg)), "XXX") 300 | console.log(notes.head + notes.body) 301 | } 302 | 303 | start() 304 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "workspaces": [ 4 | "common", "highlight", "lr", "generator", 5 | "css", "javascript", "html", "xml", "json", 6 | "cpp", "rust", "java", "php", "python", 7 | "lezer", "markdown", "sass", 8 | "website" 9 | ] 10 | } 11 | --------------------------------------------------------------------------------