├── .editorconfig ├── .gitattributes ├── .github └── workflows │ └── main.yml ├── .gitignore ├── .npmrc ├── .prettierignore ├── .remarkignore ├── build.js ├── example ├── index.css ├── index.html └── index.jsx ├── grammar.yml ├── license ├── package.json ├── readme.md ├── screenshot-md-dark.png ├── screenshot-mdx-light.png ├── source.mdx.js ├── source.mdx.tmLanguage ├── test ├── attention-asterisk.md ├── attention-asterisk.md.snap ├── attention-tilde.md ├── attention-tilde.md.snap ├── attention_underscore.md ├── attention_underscore.md.snap ├── character-escape.md ├── character-escape.md.snap ├── character-reference.md ├── character-reference.md.snap ├── code-text.md ├── code-text.md.snap ├── frontmatter-toml.md ├── frontmatter-toml.md.snap ├── frontmatter-yaml.md ├── frontmatter-yaml.md.snap ├── headings.md ├── headings.md.snap ├── hello-world.md ├── hello-world.md.snap ├── math-text.md ├── math-text.md.snap ├── source.toml.json ├── source.yaml.json ├── thematic-break.md └── thematic-break.md.snap ├── text.md.js ├── text.md.tmLanguage └── tsconfig.json /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 2 7 | indent_style = space 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # https://github.com/github/linguist/blob/HEAD/docs/overrides.md 2 | test/* linguist-vendored 3 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | main: 3 | runs-on: ubuntu-latest 4 | steps: 5 | - uses: actions/checkout@v4 6 | - uses: actions/setup-node@v4 7 | with: 8 | node-version: node 9 | - run: npm install 10 | - run: npm test 11 | - uses: JamesIves/github-pages-deploy-action@releases/v4 12 | with: 13 | branch: gh-pages 14 | commit-message: . 15 | folder: example 16 | git-config-email: tituswormer@gmail.com 17 | git-config-name: Titus Wormer 18 | single-commit: true 19 | name: main 20 | on: 21 | push: 22 | branches: 23 | - main 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.d.ts 2 | *.log 3 | *.map 4 | *.tsbuildinfo 5 | .DS_Store 6 | /example/index.module.js 7 | /example/index.nomodule.js 8 | /typescript-react.xml 9 | coverage/ 10 | node_modules/ 11 | yarn.lock 12 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | ignore-scripts=true 2 | package-lock=false 3 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.md 3 | coverage/ 4 | -------------------------------------------------------------------------------- /.remarkignore: -------------------------------------------------------------------------------- 1 | test/ 2 | -------------------------------------------------------------------------------- /build.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @import {Grammar} from '@wooorm/starry-night' 3 | */ 4 | 5 | /** 6 | * @typedef {Grammar['patterns'][number]} Rule 7 | * 8 | * @typedef {'autolink' | 'code-indented' | 'html'} ConditionCommonmark 9 | * @typedef {'directive' | 'frontmatter' | 'gfm' | 'github' | 'math' | 'mdx'} ConditionExtension 10 | * @typedef {ConditionCommonmark | ConditionExtension} Condition 11 | * @typedef {`!${Condition}`} NegatedCondition 12 | * 13 | * @typedef LanguageInfo 14 | * Configuration for a language. 15 | * @property {Array} conditions 16 | * Conditions found in `grammar.yml` to choose. 17 | * @property {boolean | undefined} [embedTsx] 18 | * Whether to embed a copy of the TypeScript grammar; 19 | * the TypeScript grammar is required for MDX to work; 20 | * this is normally assumed to be used by the end user, 21 | * but if that can’t be guaranteed, 22 | * enable this flag. 23 | * @property {Array} extensions 24 | * List of file extensions, with dots; 25 | * used in `starry-night` and `tmLanguage` file. 26 | * @property {string | undefined} [filename] 27 | * Name of file, such as `text.md`; 28 | * defaults to `scopeName`. 29 | * @property {Array} names 30 | * Names of language, used in the `starry-night` grammar. 31 | * @property {string} name 32 | * Name of language; 33 | * used in the `tmLanguage` file. 34 | * @property {string} scopeName 35 | * Name of scope, such as `text.md`; 36 | * when `source.mdx`, the suffix of all rules will be `.mdx`. 37 | * @property {string} uuid 38 | * UUID to use for language; 39 | * used in the `tmLanguage` file. 40 | */ 41 | 42 | /* eslint-disable complexity */ 43 | 44 | import assert from 'node:assert/strict' 45 | import fs from 'node:fs/promises' 46 | import {common} from '@wooorm/starry-night' 47 | import sourceClojure from '@wooorm/starry-night/source.clojure' 48 | import sourceCoffee from '@wooorm/starry-night/source.coffee' 49 | import sourceCssLess from '@wooorm/starry-night/source.css.less' 50 | import sourceDockerfile from '@wooorm/starry-night/source.dockerfile' 51 | import sourceElixir from '@wooorm/starry-night/source.elixir' 52 | import sourceElm from '@wooorm/starry-night/source.elm' 53 | import sourceErlang from '@wooorm/starry-night/source.erlang' 54 | import sourceGitconfig from '@wooorm/starry-night/source.gitconfig' 55 | import sourceHaskell from '@wooorm/starry-night/source.haskell' 56 | import sourceJulia from '@wooorm/starry-night/source.julia' 57 | import sourceRaku from '@wooorm/starry-night/source.raku' 58 | import sourceScala from '@wooorm/starry-night/source.scala' 59 | import sourceTsx from '@wooorm/starry-night/source.tsx' 60 | import sourceToml from '@wooorm/starry-night/source.toml' 61 | import textHtmlAsciidoc from '@wooorm/starry-night/text.html.asciidoc' 62 | import textHtmlMarkdownSourceGfmApib from '@wooorm/starry-night/text.html.markdown.source.gfm.apib' 63 | import textHtmlPhp from '@wooorm/starry-night/text.html.php' 64 | import textPythonConsole from '@wooorm/starry-night/text.python.console' 65 | import textShellSession from '@wooorm/starry-night/text.shell-session' 66 | import {characterEntities} from 'character-entities' 67 | import {characterEntitiesLegacy} from 'character-entities-legacy' 68 | import escapeStringRegexp from 'escape-string-regexp' 69 | import {nameToEmoji} from 'gemoji' 70 | import {gzipSize} from 'gzip-size' 71 | import {htmlBlockNames, htmlRawNames} from 'micromark-util-html-tag-name' 72 | import plist from 'plist' 73 | import prettyBytes from 'pretty-bytes' 74 | import regexgen from 'regexgen' 75 | import {fetch} from 'undici' 76 | import {parse} from 'yaml' 77 | 78 | /** @type {string | undefined} */ 79 | let file 80 | 81 | // Crawl TypeScript grammar. 82 | try { 83 | file = String(await fs.readFile('typescript-react.xml')) 84 | } catch { 85 | const result = await fetch( 86 | 'https://raw.githubusercontent.com/microsoft/TypeScript-TmLanguage/master/TypeScriptReact.tmLanguage' 87 | ) 88 | file = await result.text() 89 | await fs.writeFile('typescript-react.xml', file) 90 | } 91 | 92 | /* eslint-disable camelcase */ 93 | /** @type {Record} */ 94 | const dynamicVariables = { 95 | character_reference_name_terminated: regexgen(Object.keys(characterEntities)) 96 | .source, 97 | character_reference_name_unterminated: regexgen(characterEntitiesLegacy) 98 | .source, 99 | html_basic_name: regexgen(htmlBlockNames).source, 100 | html_raw_name: regexgen(htmlRawNames).source, 101 | github_gemoji_name: regexgen(Object.keys(nameToEmoji)).source 102 | } 103 | /* eslint-enable camelcase */ 104 | 105 | const document = String(await fs.readFile('grammar.yml')) 106 | 107 | /** @type {Grammar} */ 108 | const grammar = parse(document) 109 | 110 | // Rule injection 111 | // Figure out embedded grammars. 112 | const embeddedGrammars = [ 113 | ...common, 114 | sourceClojure, 115 | sourceCoffee, 116 | sourceCssLess, 117 | sourceDockerfile, 118 | sourceElixir, 119 | sourceElm, 120 | sourceErlang, 121 | sourceGitconfig, 122 | sourceHaskell, 123 | sourceJulia, 124 | sourceRaku, 125 | sourceScala, 126 | sourceTsx, 127 | sourceToml, 128 | textHtmlAsciidoc, 129 | textHtmlMarkdownSourceGfmApib, 130 | textHtmlPhp, 131 | textPythonConsole, 132 | textShellSession 133 | ] 134 | .map((d) => { 135 | let id = d.scopeName.split('.').pop() 136 | assert(id, 'expected `id`') 137 | id = 138 | id === 'basic' ? 'html' : id === 'c++' ? 'cpp' : id === 'gfm' ? 'md' : id 139 | 140 | const grammar = { 141 | scopeNames: [d.scopeName], 142 | extensions: d.extensions, 143 | extensionsWithDot: d.extensionsWithDot || [], 144 | names: d.names, 145 | id 146 | } 147 | 148 | // Remove `.tsx`, that’s weird! 149 | if (id === 'xml') { 150 | grammar.extensions = grammar.extensions.filter((d) => d !== '.tsx') 151 | } 152 | 153 | if (id === 'cpp') { 154 | grammar.scopeNames.push('source.cpp') 155 | } 156 | 157 | if (id === 'svg') { 158 | grammar.scopeNames.push('text.xml') 159 | } 160 | 161 | if (id === 'md') { 162 | grammar.scopeNames = ['text.md', 'source.gfm', 'text.html.markdown'] 163 | // Remove `.mdx`. 164 | grammar.extensions = grammar.extensions.filter((d) => d !== '.mdx') 165 | } 166 | 167 | return grammar 168 | }) 169 | .filter( 170 | (d) => 171 | d.names.length > 0 || 172 | d.extensions.length > 0 || 173 | (d.extensionsWithDot && d.extensionsWithDot.length > 0) 174 | ) 175 | 176 | embeddedGrammars.push({ 177 | scopeNames: ['source.mdx'], 178 | extensions: ['.mdx'], 179 | extensionsWithDot: [], 180 | names: ['mdx'], 181 | id: 'mdx' 182 | }) 183 | 184 | embeddedGrammars.sort((a, b) => a.id.localeCompare(b.id)) 185 | 186 | /** 187 | * The grammars included in: 188 | * 189 | */ 190 | const gfm = [ 191 | 'source.clojure', 192 | 'source.coffee', 193 | 'source.cpp', 194 | 'source.cs', 195 | 'source.css.less', 196 | 'source.css', 197 | 'source.c', 198 | 'source.diff', 199 | 'source.dockerfile', 200 | 'source.elixir', 201 | 'source.elm', 202 | 'source.erlang', 203 | 'source.gfm', 204 | 'source.gitconfig', 205 | 'source.go', 206 | 'source.graphql', 207 | 'source.haskell', 208 | 'source.java', 209 | 'source.json', 210 | 'source.js', 211 | 'source.julia', 212 | 'source.kotlin', 213 | 'source.makefile', 214 | 'source.objc', 215 | 'source.perl', 216 | 'source.python', 217 | 'source.raku', 218 | 'source.r', 219 | 'source.ruby', 220 | 'source.rust', 221 | 'source.scala', 222 | 'source.shell', 223 | 'source.sql', 224 | 'source.swift', 225 | 'source.toml', 226 | 'source.ts', 227 | 'source.yaml', 228 | 'text.html.asciidoc', 229 | 'text.html.basic', 230 | 'text.html.markdown.source.gfm.apib', 231 | // Turned off: this could be embedded in `language-gfm`, but not actually used in linguist. 232 | // See: . 233 | // 'text.html.markdown.source.gfm.mson', 234 | 'text.html.php', 235 | 'text.python.console', 236 | 'text.shell-session', 237 | 'text.xml' 238 | ] 239 | 240 | // Make sure we embed everything that `atom/language-gfm` did. 241 | for (const scopeName of gfm) { 242 | const found = embeddedGrammars.find((d) => d.scopeNames.includes(scopeName)) 243 | assert(found, scopeName) 244 | } 245 | 246 | // Inject grammars for code blocks with embedded grammars. 247 | assert(grammar.repository, 'expected `repository`') 248 | const codeFencedUnknown = grammar.repository['commonmark-code-fenced-unknown'] 249 | assert(codeFencedUnknown, 'expected `codeFencedUnknown` rule in `repository`') 250 | assert( 251 | 'patterns' in codeFencedUnknown && codeFencedUnknown.patterns, 252 | 'expected `patterns` in `commonmark-code-fenced-unknown` rule' 253 | ) 254 | const backtick = codeFencedUnknown.patterns[0] 255 | const tilde = codeFencedUnknown.patterns[1] 256 | assert( 257 | 'begin' in backtick && 258 | 'end' in backtick && 259 | backtick.begin && 260 | backtick.end && 261 | !('include' in backtick), 262 | 'expected `begin`, `end` in backtick rule' 263 | ) 264 | assert(/`/.test(backtick.begin), 'expected `` ` `` in `backtick` rule') 265 | assert( 266 | 'begin' in tilde && 267 | 'end' in tilde && 268 | tilde.begin && 269 | tilde.end && 270 | !('include' in tilde), 271 | 'expected `begin`, `end` in tilde rule' 272 | ) 273 | assert(/~/.test(tilde.begin), 'expected `~` in `tilde` rule') 274 | 275 | const codeFenced = grammar.repository['commonmark-code-fenced'] 276 | assert(codeFenced, 'expected `codeFenced` rule in `repository`') 277 | assert( 278 | 'patterns' in codeFenced && codeFenced.patterns, 279 | 'expected `patterns` rule in `codeFenced`' 280 | ) 281 | 282 | /** @type {Array} */ 283 | const includes = [] 284 | 285 | for (const embedded of embeddedGrammars) { 286 | const id = 'commonmark-code-fenced-' + embedded.id 287 | 288 | const extensions = embedded.extensions 289 | .map((d) => d.slice(1)) 290 | .sort() 291 | .map((d) => escapeStringRegexp(d)) 292 | const extensionsWithDot = embedded.extensionsWithDot 293 | .map((d) => d.slice(1)) 294 | .sort() 295 | .map((d) => escapeStringRegexp(d)) 296 | const uniqueNames = embedded.names 297 | .filter((d) => !extensions.includes(d)) 298 | .sort() 299 | .map((d) => escapeStringRegexp(d)) 300 | 301 | // Dot is optional for extensions. 302 | // . const extensionsSource = '\\.?(?:' + regexgen(extensions).source + ')' 303 | const extensionsSource = 304 | extensions.length === 0 305 | ? '' 306 | : '(?:.*\\.)?' + 307 | (extensions.length === 1 308 | ? extensions[0] 309 | : '(?:' + extensions.join('|') + ')') 310 | const extensionsWithDotSource = 311 | extensionsWithDot.length === 0 312 | ? '' 313 | : '.*\\.' + 314 | (extensionsWithDot.length === 1 315 | ? extensionsWithDot[0] 316 | : '(?:' + extensionsWithDot.join('|') + ')') 317 | const regex = 318 | '(?i:' + 319 | [...uniqueNames, extensionsSource, extensionsWithDotSource] 320 | .filter(Boolean) 321 | .join('|') + 322 | ')' 323 | 324 | const backtickCopy = structuredClone(backtick) 325 | const tildeCopy = structuredClone(tilde) 326 | assert(backtickCopy.begin, 'expected begin') 327 | backtickCopy.begin = backtickCopy.begin 328 | .replace(/var\(char_code_info_tick\)\+/, regex) 329 | .replace(/\)\?\)\?/, ')?)') 330 | delete backtickCopy.contentName 331 | backtickCopy.name = 'markup.code.' + embedded.id + '.var(suffix)' 332 | backtickCopy.patterns = structuredClone([ 333 | { 334 | begin: '(^|\\G)(\\s*)(.*)', 335 | while: '(^|\\G)(?![\\t ]*([`~]{3,})[\\t ]*$)', 336 | contentName: 'meta.embedded.' + embedded.id, 337 | patterns: embedded.scopeNames.map((d) => ({include: d})) 338 | } 339 | ]) 340 | 341 | assert(tildeCopy.begin, 'expected begin') 342 | tildeCopy.begin = tildeCopy.begin 343 | .replace(/var\(char_code_info_tilde\)\+/, regex) 344 | .replace(/\)\?\)\?/, ')?)') 345 | delete tildeCopy.contentName 346 | tildeCopy.name = structuredClone(backtickCopy.name) 347 | tildeCopy.patterns = structuredClone(backtickCopy.patterns) 348 | 349 | grammar.repository[id] = {patterns: [backtickCopy, tildeCopy]} 350 | includes.push({include: '#' + id}) 351 | } 352 | 353 | codeFenced.patterns = [...includes, ...codeFenced.patterns] 354 | 355 | // Just use the `source.tsx` scope normally, and only optionally embed it. 356 | /** @type {Grammar} */ 357 | // @ts-expect-error: fine. 358 | const tsx = plist.parse(file) 359 | 360 | // Rename all rule names so that they don’t conflict with our grammar and we 361 | // can later optionally rename `source.ts#` to `#source-ts-`. 362 | visit(tsx, '#', (rule) => { 363 | // Rename definitions: 364 | if ('repository' in rule && rule.repository) { 365 | /** @type {Record} */ 366 | const replacement = {} 367 | /** @type {string} */ 368 | let key 369 | 370 | for (key in rule.repository) { 371 | if (Object.hasOwn(rule.repository, key)) { 372 | replacement['source-ts-' + key] = rule.repository[key] 373 | } 374 | } 375 | 376 | rule.repository = replacement 377 | } 378 | 379 | if ('include' in rule && rule.include && rule.include.startsWith('#')) { 380 | rule.include = '#source-ts-' + rule.include.slice(1) 381 | } 382 | 383 | // Fix scopes. 384 | // Extensions of scopes used in the TS grammar are `jsdoc`, `regexp`, and `tsx`. 385 | if ('name' in rule && rule.name) { 386 | rule.name = rule.name.replace(/\.tsx$/, '.jsx') 387 | } 388 | 389 | if ('contentName' in rule && rule.contentName) { 390 | rule.contentName = rule.contentName.replace(/\.tsx$/, '.jsx') 391 | } 392 | }) 393 | 394 | assert(tsx.repository, 'expected repository in `ecmascript` grammar') 395 | 396 | /** @type {Array} */ 397 | const languages = [ 398 | { 399 | // Extensions for markdown (from `github/linguist`). 400 | extensions: [ 401 | '.md', 402 | '.livemd', 403 | '.markdown', 404 | '.mdown', 405 | '.mdwn', 406 | '.mkd', 407 | '.mkdn', 408 | '.mkdown', 409 | '.ronn', 410 | '.scd', 411 | '.workbook' 412 | ], 413 | conditions: [ 414 | // CM defaults: 415 | 'autolink', 416 | 'code-indented', 417 | 'html', 418 | // Extensions: 419 | 'directive', 420 | 'frontmatter', 421 | 'gfm', 422 | 'github', 423 | 'math' 424 | ], 425 | // Names for the language (from `github/linguist`). 426 | names: ['markdown', 'md', 'pandoc'], 427 | name: 'markdown', 428 | // Which scope to use? 429 | // In Atom, GitHub used `source.gfm`, which is often included in the 430 | // grammars from `github/linguist`, and the `source.*` prefix is the 431 | // most common prefix. 432 | // In VS Code, Microsoft uses `text.html.markdown`. 433 | // The latter was also used before Atom. 434 | // But it has a problem: it “inherits” from HTML. 435 | // Which we specifically don’t want. 436 | // Especially, because we also care about MDX. 437 | // 438 | // So, we go with the same mechanism, but don’t force GFM: 439 | scopeName: 'text.md', 440 | // See: 441 | uuid: '0A1D9874-B448-11D9-BD50-000D93B6E43C' 442 | }, 443 | { 444 | extensions: ['.mdx'], 445 | conditions: [ 446 | // Extensions: 447 | 'frontmatter', 448 | 'gfm', 449 | 'github', 450 | 'math', 451 | 'mdx' 452 | ], 453 | names: ['mdx'], 454 | name: 'MDX', 455 | scopeName: 'source.mdx', 456 | // Just a random ID I created just now! 457 | uuid: 'fe65e2cd-7c73-4a27-8b5e-5902893626aa' 458 | } 459 | ] 460 | 461 | for (const language of languages) { 462 | const generatedGrammar = structuredClone(grammar) 463 | 464 | /** @type {Record} */ 465 | const variables = { 466 | // @ts-expect-error: hush 467 | ...generatedGrammar.variables, 468 | ...dynamicVariables 469 | } 470 | 471 | // If indented code is enabled (default), we don’t mark stuff that’s 472 | // indented too far. 473 | variables.before = language.conditions.includes('code-indented') 474 | ? '(?:^|\\G)[ ]{0,3}' 475 | : '(?:^|\\G)[\\t ]*' 476 | 477 | variables.suffix = language.scopeName === 'source.mdx' ? 'mdx' : 'md' 478 | 479 | visit(generatedGrammar, '#', (rule) => { 480 | // Conditional rules. 481 | if ('if' in rule) { 482 | /** @type {Condition | NegatedCondition | Array} */ 483 | // @ts-expect-error: custom. 484 | const condition = rule.if 485 | const conditions = typeof condition === 'string' ? [condition] : condition 486 | /** @type {Array} */ 487 | const apply = [] 488 | /** @type {Array} */ 489 | const negate = [] 490 | 491 | for (const value of conditions) { 492 | if (value.startsWith('!')) { 493 | // @ts-expect-error: fine 494 | negate.push(value.slice(1)) 495 | } else { 496 | // @ts-expect-error: fine 497 | apply.push(value) 498 | } 499 | } 500 | 501 | const include = 502 | (apply.length === 0 || 503 | apply.some((d) => language.conditions.includes(d))) && 504 | (negate.length === 0 || 505 | negate.some((d) => !language.conditions.includes(d))) 506 | 507 | if (!include) { 508 | return false 509 | } 510 | 511 | // Fine, but delete the non-standard field 512 | delete rule.if 513 | } 514 | 515 | // Expand variables 516 | if ('name' in rule && rule.name) { 517 | rule.name = expand(rule.name, variables) 518 | } 519 | 520 | if ('contentName' in rule && rule.contentName) { 521 | rule.contentName = expand(rule.contentName, variables) 522 | } 523 | 524 | if (rule.match) { 525 | rule.match = expand(rule.match, variables) 526 | } 527 | 528 | if (rule.begin) { 529 | rule.begin = expand(rule.begin, variables) 530 | } 531 | 532 | if ('end' in rule && rule.end) { 533 | rule.end = expand(rule.end, variables) 534 | } 535 | 536 | if ('while' in rule && rule.while) { 537 | rule.while = expand(rule.while, variables) 538 | } 539 | 540 | // Use our own embedded big TypeScript/JavaScript grammar: 541 | // 1. it might be better than the other ones, 542 | // 2. so we can highlight JS inside code the same as JS in 543 | // ESM/expressions/etc. 544 | if ( 545 | language.embedTsx && 546 | 'include' in rule && 547 | rule.include && 548 | // Use it for anything that looks like JS/TS to get uniform highlighting. 549 | /^source\.(jsx?|tsx?)(?=#|$)/i.test(rule.include) 550 | ) { 551 | const hash = rule.include.indexOf('#') 552 | rule.include = 553 | '#source-ts-' + (hash === -1 ? 'program' : rule.include.slice(hash + 1)) 554 | } 555 | }) 556 | 557 | // Inject TSX grammar. 558 | if (language.embedTsx) { 559 | // Inject all subrules. 560 | Object.assign(grammar.repository, tsx.repository) 561 | // Inject a rule to get the entire embedded TSX grammar. 562 | grammar.repository['source-ts-program'] = {patterns: tsx.patterns} 563 | } 564 | 565 | const {referenced, defined} = analyze(generatedGrammar) 566 | 567 | for (const key of referenced) { 568 | if (!defined.has(key)) { 569 | console.warn( 570 | '%s: includes undefined `%s`, it’s probably removed by some condition, but still referenced somewhere', 571 | language.scopeName, 572 | key 573 | ) 574 | } 575 | } 576 | 577 | for (const key of defined) { 578 | if (!referenced.has(key)) { 579 | console.warn( 580 | '%s: includes unreferenced `%s`, consider adding a condition to it', 581 | language.scopeName, 582 | key 583 | ) 584 | } 585 | } 586 | 587 | const tmLanguage = { 588 | fileTypes: language.extensions.map((d) => { 589 | assert(d.startsWith('.'), 'expected `.`') 590 | return d.slice(1) 591 | }), 592 | name: language.name, 593 | patterns: generatedGrammar.patterns, 594 | repository: generatedGrammar.repository, 595 | scopeName: language.scopeName, 596 | uuid: language.uuid 597 | } 598 | 599 | /** @typedef {Grammar} */ 600 | const starryNightGrammar = { 601 | extensions: language.extensions, 602 | names: language.names, 603 | patterns: generatedGrammar.patterns, 604 | repository: generatedGrammar.repository, 605 | scopeName: language.scopeName 606 | } 607 | 608 | const filename = language.filename || language.scopeName 609 | const size = prettyBytes(await gzipSize(JSON.stringify(tmLanguage) + '\n')) 610 | 611 | console.log('gzip-size:', filename, size) 612 | 613 | // Write files. 614 | await fs.writeFile( 615 | new URL(filename + '.tmLanguage', import.meta.url), 616 | // @ts-expect-error: fine, it’s serializable. 617 | plist.build(tmLanguage) + '\n' 618 | ) 619 | await fs.writeFile( 620 | new URL(filename + '.js', import.meta.url), 621 | [ 622 | '/* eslint-disable no-template-curly-in-string */', 623 | '/**', 624 | ' * @import {Grammar} from "@wooorm/starry-night"', 625 | ' */', 626 | '', 627 | '/** @type {Grammar} */', 628 | 'const grammar = ' + JSON.stringify(starryNightGrammar, null, 2), 629 | 'export default grammar', 630 | '' 631 | ].join('\n') 632 | ) 633 | } 634 | 635 | /** 636 | * @param {Rule} rule 637 | * @returns {{referenced: Set, defined: Set}} 638 | */ 639 | function analyze(rule) { 640 | /** @type {Set} */ 641 | const defined = new Set() 642 | /** @type {Set} */ 643 | const referenced = new Set() 644 | 645 | visit(rule, '#', (rule) => { 646 | if ('repository' in rule && rule.repository) { 647 | for (const key of Object.keys(rule.repository)) { 648 | defined.add(key) 649 | } 650 | } 651 | 652 | if ('include' in rule && rule.include && rule.include.startsWith('#')) { 653 | referenced.add(rule.include.slice(1)) 654 | } 655 | }) 656 | 657 | return {referenced, defined} 658 | } 659 | 660 | /** 661 | * 662 | * @param {Rule} rule 663 | * @param {string} key 664 | * @param {(rule: Rule) => boolean | undefined | void} callback 665 | * @returns {boolean} 666 | */ 667 | function visit(rule, key, callback) { 668 | const result = callback(rule) 669 | 670 | if (result === false) { 671 | return result 672 | } 673 | 674 | if ('captures' in rule && rule.captures) map(rule.captures, key + '.captures') 675 | if ('beginCaptures' in rule && rule.beginCaptures) 676 | map(rule.beginCaptures, key + '.beginCaptures') 677 | if ('endCaptures' in rule && rule.endCaptures) 678 | map(rule.endCaptures, key + '.endCaptures') 679 | if ('whileCaptures' in rule && rule.whileCaptures) 680 | map(rule.whileCaptures, key + '.whileCaptures') 681 | if ('repository' in rule && rule.repository) 682 | map(rule.repository, key + '.repository') 683 | if ('injections' in rule && rule.injections) 684 | map(rule.injections, key + '.injections') 685 | if ('patterns' in rule && rule.patterns) set(rule.patterns, key + '.patterns') 686 | 687 | // Keep. 688 | return true 689 | 690 | /** 691 | * @param {Array} values 692 | * @param {string} key 693 | */ 694 | function set(values, key) { 695 | let index = -1 696 | while (++index < values.length) { 697 | const result = visit(values[index], key + '.' + index, callback) 698 | if (result === false) { 699 | values.splice(index, 1) 700 | index-- 701 | } 702 | } 703 | } 704 | 705 | /** 706 | * @param {Record} values 707 | * @param {string} parentKey 708 | */ 709 | function map(values, parentKey) { 710 | /** @type {string} */ 711 | let key 712 | 713 | for (key in values) { 714 | if (Object.hasOwn(values, key)) { 715 | const result = visit(values[key], parentKey + '.' + key, callback) 716 | if (result === false) { 717 | delete values[key] 718 | } 719 | } 720 | } 721 | } 722 | } 723 | 724 | /** 725 | * @param {string} value 726 | * @param {Record} variables 727 | * @returns {string} 728 | */ 729 | function expand(value, variables) { 730 | let done = false 731 | 732 | // Support recursion. 733 | while (!done) { 734 | done = true 735 | value = replace(value) 736 | } 737 | 738 | return value 739 | 740 | /** 741 | * @param {string} value 742 | */ 743 | function replace(value) { 744 | return value.replace(/var\(([^)]+)\)/g, replacer) 745 | } 746 | 747 | /** 748 | * @param {string} _ 749 | * @param {string} key 750 | * @returns {string} 751 | */ 752 | function replacer(_, key) { 753 | if (!Object.hasOwn(variables, key)) { 754 | throw new Error('Cannot expand variable `' + key + '`') 755 | } 756 | 757 | done = false 758 | return variables[key] 759 | } 760 | } 761 | -------------------------------------------------------------------------------- /example/index.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | line-height: calc(1 * (1em + 1ex)); 4 | } 5 | 6 | a { 7 | color: var(--color-hl); 8 | text-decoration: none; 9 | transition: 200ms; 10 | 11 | transition-property: color; 12 | } 13 | 14 | a:focus, 15 | a:hover, 16 | a:target { 17 | color: inherit; 18 | } 19 | 20 | body { 21 | margin: 0; 22 | } 23 | 24 | code, 25 | kbd, 26 | .draw, 27 | .write { 28 | font-family: 29 | ui-monospace, 30 | SFMono-Regular, 31 | SF Mono, 32 | Menlo, 33 | Consolas, 34 | Liberation Mono, 35 | monospace; 36 | font-feature-settings: normal; 37 | } 38 | 39 | code, 40 | kbd { 41 | font-size: smaller; 42 | } 43 | 44 | fieldset { 45 | border-width: 0; 46 | margin: calc(0.25 * (1em + 1ex)) 0; 47 | } 48 | 49 | h1, 50 | p { 51 | margin-bottom: calc(1 * (1em + 1ex)); 52 | margin-top: calc(1 * (1em + 1ex)); 53 | } 54 | 55 | h1 { 56 | font-size: 2em; 57 | font-weight: 100; 58 | text-align: center; 59 | } 60 | 61 | html { 62 | --color-border: #d0d7de; 63 | --color-canvas-back: #f6f8fa; 64 | --color-canvas-front: #ffffff; 65 | --color-fg: #0d1117; 66 | --color-hl: #0969da; 67 | 68 | background-color: var(--color-canvas-back); 69 | color-scheme: light dark; 70 | color: var(--color-fg); 71 | font-family: system-ui; 72 | word-break: break-word; 73 | } 74 | 75 | label { 76 | margin: 0 calc(0.25 * (1em + 1ex)); 77 | } 78 | 79 | main { 80 | background-color: var(--color-canvas-back); 81 | margin: 0 auto; 82 | max-width: calc(40 * (1em + 1ex)); 83 | padding: 0 calc(2 * (1em + 1ex)); 84 | position: relative; 85 | } 86 | 87 | main > div { 88 | border-radius: inherit; 89 | } 90 | 91 | section { 92 | border: 0 solid var(--color-border); 93 | margin: calc(2 * (1em + 1ex)) calc(-2 * (1em + 1ex)); 94 | padding: calc(2 * (1em + 1ex)); 95 | 96 | border-top-width: 1px; 97 | border-bottom-width: 1px; 98 | } 99 | 100 | section:first-child { 101 | border-top-left-radius: inherit; 102 | border-top-right-radius: inherit; 103 | border-top-width: 0; 104 | margin-top: 0; 105 | } 106 | 107 | section:last-child { 108 | border-bottom-left-radius: inherit; 109 | border-bottom-right-radius: inherit; 110 | border-bottom-width: 0; 111 | margin-bottom: 0; 112 | } 113 | 114 | section + section { 115 | margin-top: calc(-2 * (1em + 1ex) - 1px); 116 | } 117 | 118 | section > :first-child { 119 | margin-top: 0; 120 | } 121 | 122 | section > :last-child { 123 | margin-bottom: 0; 124 | } 125 | 126 | template { 127 | display: none; 128 | } 129 | 130 | .credits { 131 | text-align: center; 132 | } 133 | 134 | .draw, 135 | .write { 136 | background: transparent; 137 | border: none; 138 | box-sizing: border-box; 139 | font-size: 14px; 140 | height: 100%; 141 | letter-spacing: normal; 142 | line-height: calc(1 * (1em + 1ex)); 143 | margin: 0; 144 | outline: none; 145 | overflow: hidden; 146 | padding: 0; 147 | resize: none; 148 | tab-size: 4; 149 | white-space: pre-wrap; 150 | width: 100%; 151 | word-wrap: break-word; 152 | } 153 | 154 | .editor { 155 | max-width: 100%; 156 | overflow: hidden; 157 | position: relative; 158 | } 159 | 160 | .highlight { 161 | background-color: var(--color-canvas-front); 162 | } 163 | 164 | .write { 165 | -webkit-print-color-adjust: exact; 166 | caret-color: var(--color-hl); 167 | color: transparent; 168 | position: absolute; 169 | print-color-adjust: exact; 170 | top: 0; 171 | } 172 | 173 | .write::selection { 174 | background-color: hsl(42.22 74.31% 57.25% / 66%); 175 | color: var(--color-fg); 176 | } 177 | 178 | @media (prefers-color-scheme: dark) { 179 | html { 180 | --color-border: #30363d; 181 | --color-canvas-back: #0d1117; 182 | --color-canvas-front: #161b22; 183 | --color-fg: #f6f8fa; 184 | --color-hl: #58a6ff; 185 | } 186 | } 187 | 188 | @media (min-width: calc(30 * (1em + 1ex))) and (min-height: calc(30 * (1em + 1ex))) { 189 | main { 190 | /* Go all Tschichold when supported */ 191 | border-radius: 3px; 192 | border: 1px solid var(--color-border); 193 | margin: 11vh 22.2vw 22.2vh 11.1vw; 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /example/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | markdown-tm-language 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 | -------------------------------------------------------------------------------- /example/index.jsx: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | 3 | /** 4 | * @import {Grammar} from '@wooorm/starry-night' 5 | */ 6 | 7 | /// 8 | 9 | import sourceCss from '@wooorm/starry-night/source.css' 10 | import sourceDiff from '@wooorm/starry-night/source.diff' 11 | import sourceJson from '@wooorm/starry-night/source.json' 12 | import sourceJs from '@wooorm/starry-night/source.js' 13 | import sourceToml from '@wooorm/starry-night/source.toml' 14 | import sourceTsx from '@wooorm/starry-night/source.tsx' 15 | import sourceTs from '@wooorm/starry-night/source.ts' 16 | import sourceYaml from '@wooorm/starry-night/source.yaml' 17 | import textHtmlBasic from '@wooorm/starry-night/text.html.basic' 18 | import textXmlSvg from '@wooorm/starry-night/text.xml.svg' 19 | import textXml from '@wooorm/starry-night/text.xml' 20 | import {createStarryNight} from '@wooorm/starry-night' 21 | import {toJsxRuntime} from 'hast-util-to-jsx-runtime' 22 | import ReactDom from 'react-dom/client' 23 | import {Fragment, jsxs, jsx} from 'react/jsx-runtime' 24 | import React from 'react' 25 | import sourceMdx from '../source.mdx.js' 26 | import textMarkdown from '../text.md.js' 27 | 28 | /** @type {Array} */ 29 | const grammars = [ 30 | sourceCss, 31 | sourceDiff, 32 | sourceJs, 33 | // @ts-expect-error: TS is wrong about `.json`, it’s not an extension. 34 | sourceJson, 35 | sourceMdx, 36 | sourceToml, 37 | sourceTsx, 38 | sourceTs, 39 | sourceYaml, 40 | textHtmlBasic, 41 | textMarkdown, 42 | textXmlSvg, 43 | textXml 44 | ] 45 | 46 | const main = document.querySelectorAll('main')[0] 47 | 48 | const root = ReactDom.createRoot(main) 49 | 50 | const sampleMarkdown = `--- 51 | yaml: 1 52 | --- 53 | 54 | ^-- extension: frontmatter 55 | 56 | # Hello, *world*! 57 | 58 | Autolink: , . 59 | Attention (emphasis): *hi* / _hi_ 60 | Attention (strong): **hi** / __hi__ 61 | Attention (strong & emphasis): ***hi*** / ___hi___. 62 | Attention (strikethrough): ~hi~ ~~hi~~. 63 | Character escape: \\-, \\&. 64 | Character reference: & { ģ. 65 | Code (text): \`hi\` and \`\` \` \`\`. 66 | HTML (comment): . 67 | HTML (instruction): . 68 | HTML (declaration): . 69 | HTML (cdata): . 70 | HTML (tag, close): . 71 | HTML (tag, open): . 72 | HTML (tag, open, self-closing): . 73 | HTML (tag, open, boolean attribute): . 74 | HTML (tag, open, unquoted attribute): . 75 | HTML (tag, open, double quoted attribute): . 76 | HTML (tag, open, single quoted attribute): . 77 | HTML (tag, open, style attribute): . 78 | HTML (tag, open, on* attribute): . 79 | Label end (resource): [a](https://example˙com 'title'). 80 | Label end (reference, full): [a][b]. 81 | Label end (reference, collapsed, shortcut): [a][], [a]. 82 | 83 | ## Definitions 84 | 85 | [a]: "b" 86 | [a]: https://example˙com 'b' 87 | [a]: # (b\\&c) 88 | [a&b]: <> 89 | 90 | ## Heading (setext) 91 | 92 | alpha 93 | ===== 94 | 95 | *bravo 96 | charlie* 97 | -------- 98 | 99 | ## Heading (atx) 100 | # 101 | ## A ## 102 | ### B 103 | #### C 104 | ##### D 105 | ###### E 106 | ####### ? 107 | 108 | ## Thematic break 109 | 110 | *** 111 | 112 | ## Code (indented) 113 | 114 | \tconsole.log(1) 115 | 116 | ## Code (fenced) 117 | 118 | \`\`\`\`markdown 119 | \`\`\`css 120 | em { color: red } /* What! */ 121 | \`\`\` 122 | \`\`\`\` 123 | 124 | ~~~js eval 125 | alert(true + 2 + '3') 126 | ~~~ 127 | 128 | ## Block quote 129 | 130 | > # asd 131 | 132 | > **asd 133 | qwe** 134 | 135 | > ~~~js 136 | > console.log(1) 137 | > ~~~ 138 | 139 | ## List 140 | 141 | 1. # asd 142 | 143 | * **asd 144 | qwe** 145 | 146 | * ~~~js 147 | console.log('hi!') 148 | ~~~ 149 | 150 | 123456789. \`\`\`js 151 | asd + 123 152 | 153 | ## HTML (flow) 154 | 155 | html (comment, empty) & & { { ģ ģ 156 | 157 | html (comment, empty 2) I'm ¬it; I tell you, I'm ∉ I tell you. 158 | 159 | html (comment, empty 3) 160 | 161 | html (comment, multiline) 163 | 164 | html (instruction, empty) 165 | 166 | html (instruction, empty 2) 167 | 168 | html (instruction, multiline) 170 | 171 | html (declaration, empty) 172 | 173 | html (declaration, filled) 174 | 175 | html (declaration, multiline) 177 | 178 | (cdata, empty) 179 | 180 | (cdata, filled) 181 | 182 | (cdata, multiline) 184 | 185 | (raw, empty) 186 | 187 | (raw, script data) 188 | 189 | (raw, multiline) 194 | 195 | (raw, rawtext, style) 196 | 197 | (raw, rcdata) 198 | 199 |
200 | Basic (this **is not** emphasis!) 201 | 202 |
203 | 204 | Basic (this **is** emphasis!) 205 | 206 | 207 | a &amp; b 208 | (basic, rawtext) 209 | 210 | 211 | a & b 212 | (basic, rcdata) 213 | 214 | 215 | Complete (closing) (this **is not** emphasis!) 216 | 217 | 218 | 219 | Complete (closing) (this **is** emphasis!) 220 | 221 | 222 | Complete (open) (this **is not** emphasis!) 223 | 224 | 225 | 226 | Complete (open) (this **is** emphasis!) 227 | 228 | ## Extension: math 229 | 230 | Math (text): $$hi$$ and $$ $ $$. 231 | Math (flow): 232 | 233 | $$ 234 | L = \\frac{1}{2} \\rho v^2 S C_L 235 | $$ 236 | 237 | ## Extension: directive 238 | 239 | Text: :cite[smith04] 240 | 241 | Leaf: 242 | 243 | ::youtube[Video of a **cat** in a box]{#readme.red.green.blue a=b a="b" a='b' v=01ab2cd3efg} 244 | 245 | Containers: 246 | 247 | ::::spoiler 248 | He dies. 249 | 250 | :::spoiler 251 | She is born. 252 | ::: 253 | :::: 254 | 255 | ## Extension: GFM autolink literals 256 | 257 | a a@b.com/ ("aasd@example.com") mailto:a+b@c.com xmpp:a+b@c.com. 258 | 259 | a www.example.com/a(b) www.example.com/a(b(c)), wWw.example.com/a(b(c(d))). 260 | 261 | ahttps://example.com https://example.com/a(b) hTTp://example.com/a(b(c(d))). 262 | 263 | ## Extension: GFM footnotes 264 | 265 | a[^b], [^c d]. 266 | 267 | [^b]: *Lorem 268 | dolor*. 269 | 270 | [^b]: 271 | ?? 272 | 273 | [^b]: ~~~js 274 | console.log(1) 275 | ~~~ 276 | 277 | ## Extension: GFM task list 278 | 279 | * [ ] not done 280 | 1. [x] done 281 | 282 | ## Extension: GFM table 283 | 284 | | Stuff? | stuff! | 285 | | - | ----- | 286 | | asdasda | | 287 | what¬ | qweeeeeeeeeee 288 | 289 | ## Extension: GitHub gemoji 290 | 291 | :+1: :100: 292 | 293 | ## Extension: GitHub mention 294 | 295 | @username @org/team. 296 | 297 | ## Extension: GitHub reference 298 | 299 | GH-123, #123, GHSA-123asdzxc, cve-123asdzxc, user#123, user/project#123. 300 | ` 301 | 302 | const sampleMdx = `--- 303 | title: Hello! 304 | --- 305 | 306 | import {Chart} from './chart.js' 307 | import population from './population.js' 308 | import {External} from './some/place.js' 309 | 310 | export const year = 2018 311 | export const pi = 3.14 312 | 313 | export function SomeComponent(props) { 314 | const name = (props || {}).name || 'world' 315 | 316 | return
317 |

Hi, {name}!

318 | 319 |

and some more things

320 |
321 | } 322 | 323 | export function Local(props) { 324 | return 325 | } 326 | 327 | # Last year’s snowfall 328 | 329 | In {year}, the snowfall was above average. 330 | It was followed by a warm spring which caused 331 | flood conditions in many of the nearby rivers. 332 | 333 | 334 | 335 |
336 | > Some notable things in a block quote! 337 |
338 | 339 | # Heading (rank 1) 340 | ## Heading 2 341 | ### 3 342 | #### 4 343 | ##### 5 344 | ###### 6 345 | 346 | > Block quote 347 | 348 | * Unordered 349 | * List 350 | 351 | 1. Ordered 352 | 2. List 353 | 354 | A paragraph, introducing a thematic break: 355 | 356 | --- 357 | 358 | \`\`\`js 359 | // Get an element. 360 | const element = document.querySelectorAll('#hi') 361 | 362 | // Add a class. 363 | element.classList.add('asd') 364 | \`\`\` 365 | 366 | a [link](https://example.com), an ![image](./image.png), some *emphasis*, 367 | something **strong**, and finally a little \`code()\`. 368 | 369 | } 374 | /> 375 | 376 | Two 🍰 is: {Math.PI * 2} 377 | 378 | {(function () { 379 | const guess = Math.random() 380 | 381 | if (guess > 0.66) { 382 | return Look at us. 383 | } 384 | 385 | if (guess > 0.33) { 386 | return Who would have guessed?! 387 | } 388 | 389 | return Not me. 390 | })()} 391 | 392 | {/* A comment! */} 393 | ` 394 | /** @type {Awaited>} */ 395 | let starryNight 396 | 397 | // eslint-disable-next-line unicorn/prefer-top-level-await -- XO is wrong. 398 | createStarryNight(grammars).then(function (x) { 399 | starryNight = x 400 | 401 | const missing = starryNight.missingScopes() 402 | if (missing.length > 0) { 403 | throw new Error('Missing scopes: `' + missing + '`') 404 | } 405 | 406 | root.render(React.createElement(Playground)) 407 | }) 408 | 409 | function Playground() { 410 | const [mdx, setMdx] = React.useState(false) 411 | const [text, setText] = React.useState(mdx ? sampleMdx : sampleMarkdown) 412 | 413 | const scope = mdx ? 'source.mdx' : 'text.md' 414 | 415 | return ( 416 |
417 |
418 |

419 | markdown-tm-language 420 |

421 |
422 | 434 | 446 |
447 |
448 |
449 |
450 | {toJsxRuntime(starryNight.highlight(text, scope), { 451 | Fragment, 452 | jsxs, 453 | jsx 454 | })} 455 | {/* Trailing whitespace in a `textarea` is shown, but not in a `div` 456 | with `white-space: pre-wrap`. 457 | Add a `br` to make the last newline explicit. */} 458 | {/\n[ \t]*$/.test(text) ?
: undefined} 459 |
460 |