├── .npmrc ├── .gitignore ├── src ├── tokenize.iife.ts ├── tokenize.scss.iife.ts ├── types │ ├── css-tree2.d.ts │ ├── postcss.d.ts │ └── global │ │ └── global.d.ts ├── lib │ ├── token-types.ts │ ├── token-types.scss.ts │ ├── is.ts │ ├── code-points.ts │ ├── consume.ts │ └── consume.scss.ts ├── tokenize.ts ├── tokenize.scss.ts ├── tokenize.benchmark.ts ├── tokenize.scss.test.ts └── tokenize.test.ts ├── cmd ├── build.post.mjs ├── build.mjs ├── benchmark.post.mjs ├── benchmark.mjs ├── test.mjs ├── test-parser.mjs ├── publish.mjs └── _.mjs ├── babel.config.cjs ├── .editorconfig ├── .github └── workflows │ └── test.yml ├── jest.config.mjs ├── CHANGELOG.md ├── tsconfig.json ├── dist ├── tokenize.d.ts ├── tokenize.js ├── tokenizeSCSS.js ├── tokenize.mjs ├── tokenize.cjs ├── tokenizeSCSS.mjs ├── tokenizeSCSS.cjs └── tokenize.cjs.map ├── package.json ├── rollup.config.mjs ├── LICENSE.md └── README.md /.npmrc: -------------------------------------------------------------------------------- 1 | tag-version-prefix="" 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | node_modules 3 | yarn.lock 4 | -------------------------------------------------------------------------------- /src/tokenize.iife.ts: -------------------------------------------------------------------------------- 1 | export { tokenize as default } from './tokenize.js' 2 | -------------------------------------------------------------------------------- /src/tokenize.scss.iife.ts: -------------------------------------------------------------------------------- 1 | export { tokenize as default } from './tokenize.scss.js' 2 | -------------------------------------------------------------------------------- /cmd/build.post.mjs: -------------------------------------------------------------------------------- 1 | import * as _ from './_.mjs' 2 | 3 | console.log() 4 | _.rmdir('dist/lib') 5 | _.spawnNode('node_modules/rollup/dist/bin/rollup', ['--config', '--silent']) 6 | -------------------------------------------------------------------------------- /src/types/css-tree2.d.ts: -------------------------------------------------------------------------------- 1 | declare module "css-tree2/tokenizer" { 2 | export function tokenize( 3 | css: string, 4 | onToken: (type: number, start: number, end: number) => void 5 | ): void; 6 | } 7 | -------------------------------------------------------------------------------- /babel.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | ['@babel/preset-env', { 4 | targets: { 5 | node: 'current' 6 | } 7 | }], 8 | ['@babel/preset-typescript'] 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_style = tab 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | [*.md] 11 | trim_trailing_whitespace = false 12 | 13 | [*.{json,md,yml}] 14 | indent_size = 2 15 | indent_style = space 16 | -------------------------------------------------------------------------------- /cmd/build.mjs: -------------------------------------------------------------------------------- 1 | import * as _ from './_.mjs' 2 | 3 | const isWatch = process.argv.includes('--watch') 4 | 5 | console.log(`Starting compilation...`) 6 | 7 | _.rmdir('dist') 8 | _.mkdir('dist') 9 | _.copyFile('src/types/global/global.d.ts', 'dist/tokenize.d.ts') 10 | _.spawnNode('cmd/build.post.mjs') 11 | -------------------------------------------------------------------------------- /cmd/benchmark.post.mjs: -------------------------------------------------------------------------------- 1 | import * as _ from './_.mjs' 2 | 3 | _.rmdir('dist') 4 | 5 | _.spawnTsc(['src/tokenize.benchmark.ts', '--esModuleInterop', '--skipLibCheck', '--outDir', 'dist', '--module', 'ESNext', '--moduleResolution', 'node']) 6 | 7 | _.spawnNode('dist/tokenize.benchmark.js') 8 | 9 | _.rmdir('dist') 10 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: 3 | push: 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - uses: actions/setup-node@v4 11 | with: 12 | node-version: 'lts/*' 13 | 14 | - run: npm ci 15 | - run: npm run build 16 | - run: npm run test 17 | -------------------------------------------------------------------------------- /cmd/benchmark.mjs: -------------------------------------------------------------------------------- 1 | import * as _ from './_.mjs' 2 | 3 | const isWatch = process.argv.includes('--watch') 4 | 5 | console.log(`Starting compilation...`) 6 | 7 | _.rmdir('dist') 8 | 9 | if (isWatch) _.spawnTscWatch([ '--build', '--onSuccess', 'node cmd/benchmark.post.mjs' ]) 10 | else { 11 | _.spawnTsc([ '--build' ]) 12 | _.spawnNode('cmd/benchmark.post.mjs') 13 | } 14 | -------------------------------------------------------------------------------- /cmd/test.mjs: -------------------------------------------------------------------------------- 1 | import * as _ from './_.mjs' 2 | 3 | const isWatch = process.argv.includes('--watch') 4 | 5 | const jestBin = 'node_modules/jest/bin/jest.js' 6 | 7 | console.log(`Starting testing...`) 8 | console.log() 9 | 10 | _.rmdir('coverage') 11 | 12 | _.spawnNode(jestBin, [ 13 | '--colors', 14 | '--passWithNoTests', 15 | ...(isWatch ? ['--watchAll'] : []) 16 | ]) 17 | -------------------------------------------------------------------------------- /src/types/postcss.d.ts: -------------------------------------------------------------------------------- 1 | import { Input } from 'postcss' 2 | 3 | declare module 'postcss/lib/tokenize.ts' { 4 | export default function tokenizer (input: Input, options?: { ignoreErrors?: boolean }): Tokenizer 5 | 6 | export type Tokenizer = { 7 | back(): void 8 | nextToken(): Token 9 | endOfFile(): boolean 10 | position(): number 11 | } 12 | 13 | export type Token = [string, string, number, number] 14 | } 15 | -------------------------------------------------------------------------------- /cmd/test-parser.mjs: -------------------------------------------------------------------------------- 1 | import { parser } from '../dist/parser.mjs' 2 | 3 | const css = `:where(.c-sitenav a) { 4 | align-items: center; 5 | color: inherit; 6 | display: flex; 7 | font-size: 18rx; 8 | line-height: calc(22 / 18); 9 | padding-inline: 40rx; 10 | text-decoration: none; 11 | } 12 | ` 13 | 14 | const ast = parser(css) 15 | 16 | for (const value of ast) { 17 | console.log(value, [ String(value) ]) 18 | } 19 | 20 | void ast 21 | -------------------------------------------------------------------------------- /cmd/publish.mjs: -------------------------------------------------------------------------------- 1 | import * as _ from './_.mjs' 2 | import './test.mjs' 3 | import './build.mjs' 4 | 5 | const isDryRun = Boolean(process?.env?.npm_config_dryRun) 6 | 7 | console.log(), console.log('Starting publishing...') 8 | 9 | _.question.options = { input: process.stdin, output: process.stdout } 10 | 11 | _.question('major/minor/path? ').then(answer => { 12 | answer = answer.trim().toLowerCase() 13 | if (answer === 'major' || answer === 'minor' || answer === 'patch') { 14 | if (!isDryRun) _.spawn('npm', ['version', answer]) 15 | } 16 | }) 17 | -------------------------------------------------------------------------------- /jest.config.mjs: -------------------------------------------------------------------------------- 1 | /** @typedef {import('ts-jest')} */ 2 | 3 | export default /** @type {import('@jest/types').Config.InitialOptions} */ ({ 4 | collectCoverage: true, 5 | collectCoverageFrom: [ 6 | 'src/tokenize.ts', 7 | 'src/lib/*.ts' 8 | ], 9 | coverageDirectory: 'coverage', 10 | moduleFileExtensions: [ 11 | 'js', 12 | 'ts' 13 | ], 14 | moduleNameMapper: { 15 | "^\\./(.*)\\.js$": [ 16 | "./$1.js", 17 | "./$1.ts" 18 | ] 19 | }, 20 | roots: [ 21 | '/src' 22 | ], 23 | testEnvironment: 'node', 24 | testMatch: [ 25 | '**/*.test.ts' 26 | ], 27 | verbose: true, 28 | }) 29 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changes to CSS Tokenizer 2 | 3 | ### 3.1.1 (September 25, 2024) 4 | 5 | - Maintenance release after updating dependencies 6 | 7 | ### 3.1.0 (August 27, 2021) 8 | 9 | - Added SCSS tokenization as `@csstools/tokenizer/tokenizeSCSS`. 10 | 11 | ### 3.0.0 (August 23, 2021) 12 | 13 | - Changes the shape of tokens from an array to an object. 14 | - Changes the named export from `tokenizer` to `tokenize`. 15 | - Changes the global IIFE export from `cssTokenizer` to `tokenizeCSS`. 16 | 17 | ### 2.0.2 (August 22, 2021) 18 | 19 | - Fixes an issue consuming digits after a number that begins with a decimal. 20 | 21 | ### 2.0.1 (May 11, 2021) 22 | 23 | - Fixes an issue where string tokens incorrectly put the lead quotation in the main value. 24 | 25 | ### 2.0.0 (May 11, 2021) 26 | 27 | - Supports function token (previously an identifier token followed by a parenthesis delimiter token). 28 | - Changes from default export to named export (`tokenizer`). 29 | 30 | ### 1.0.0 (September 26, 2020) 31 | 32 | Initial version 33 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/tsconfig", 3 | "compilerOptions": { 4 | "allowJs": true, 5 | "allowSyntheticDefaultImports": true, 6 | "alwaysStrict": true, 7 | "declaration": true, 8 | "emitDeclarationOnly": true, 9 | "esModuleInterop": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "incremental": true, 12 | "lib": [ 13 | "es2020", 14 | "es2020.promise", 15 | "es2020.bigint", 16 | "es2020.string" 17 | ], 18 | "module": "ESNext", 19 | "moduleResolution": "node", 20 | "noFallthroughCasesInSwitch": false, 21 | "noImplicitAny": true, 22 | "noImplicitReturns": true, 23 | "noImplicitThis": true, 24 | "noUnusedLocals": true, 25 | "noUnusedParameters": true, 26 | "outFile": "dist/tokenize.d.ts", 27 | "skipLibCheck": false, 28 | "sourceMap": true, 29 | "strict": true, 30 | "target": "ESNext", 31 | "typeRoots": [ 32 | "node_modules/@types", 33 | "src/types", 34 | "src/types/global" 35 | ], 36 | "types": [ 37 | "jest", 38 | "node" 39 | ] 40 | }, 41 | "files": [ 42 | "src/tokenize.ts" 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /src/lib/token-types.ts: -------------------------------------------------------------------------------- 1 | /** [``](https://drafts.csswg.org/css-syntax/#typedef-delim-token) */ 2 | export const SYMBOL = 0x0001 3 | 4 | /** [``](https://drafts.csswg.org/css-syntax/#comment-diagram) */ 5 | export const COMMENT = 0x0002 6 | 7 | /** [``](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 8 | export const SPACE = 0x0003 9 | 10 | /** [``](https://drafts.csswg.org/css-syntax/#ident-token-diagram) */ 11 | export const WORD = 0x0004 12 | 13 | /** [``](https://drafts.csswg.org/css-syntax/#function-token-diagram) */ 14 | export const FUNCTION = 0x0005 15 | 16 | /** [``](https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram) */ 17 | export const ATWORD = 0x0006 18 | 19 | /** [``](https://drafts.csswg.org/css-syntax/#hash-token-diagram) */ 20 | export const HASH = 0x0007 21 | 22 | /** [``](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 23 | export const STRING = 0x0008 24 | 25 | /** [``](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 26 | export const NUMBER = 0x0009 27 | -------------------------------------------------------------------------------- /dist/tokenize.d.ts: -------------------------------------------------------------------------------- 1 | /** The CSS state represents the condition of the tokenizer at any given moment. */ 2 | export type CSSState = { 3 | /** CSS data. */ 4 | data: string 5 | /** Number of characters in the CSS data. */ 6 | size: number 7 | /** Current position in the CSS data. */ 8 | tick: number 9 | /** Unicode character for the current position in the CSS data. */ 10 | codeAt0: number 11 | /** Unicode character for the 1 ahead position in the CSS data. */ 12 | codeAt1: number 13 | /** Unicode character for the 2 ahead position in the CSS data. */ 14 | codeAt2: number 15 | /** Unicode character for the 3 ahead position in the CSS data. */ 16 | codeAt3: number 17 | /** Advances the unicode characters being read from the CSS data by one position. */ 18 | next(): boolean 19 | } 20 | 21 | /** The CSS iterator produces a sequence of CSS tokens in an iterator pattern. */ 22 | export type CSSIterator = { 23 | (): CSSIteration 24 | [Symbol.iterator](): { 25 | next: CSSIterator 26 | } 27 | } 28 | 29 | /** The CSS iteration represents the most recent state and token yielded from the CSS iterator. */ 30 | export type CSSIteration = { 31 | done: boolean 32 | value: CSSToken 33 | } 34 | 35 | export type CSSToken = { 36 | tick: number 37 | type: number 38 | code: number 39 | lead: string 40 | data: string 41 | tail: string 42 | } 43 | 44 | export type CSSTokenize = (data: string) => CSSIterator 45 | 46 | export declare const tokenize: CSSTokenize 47 | -------------------------------------------------------------------------------- /src/types/global/global.d.ts: -------------------------------------------------------------------------------- 1 | /** The CSS state represents the condition of the tokenizer at any given moment. */ 2 | export type CSSState = { 3 | /** CSS data. */ 4 | data: string 5 | /** Number of characters in the CSS data. */ 6 | size: number 7 | /** Current position in the CSS data. */ 8 | tick: number 9 | /** Unicode character for the current position in the CSS data. */ 10 | codeAt0: number 11 | /** Unicode character for the 1 ahead position in the CSS data. */ 12 | codeAt1: number 13 | /** Unicode character for the 2 ahead position in the CSS data. */ 14 | codeAt2: number 15 | /** Unicode character for the 3 ahead position in the CSS data. */ 16 | codeAt3: number 17 | /** Advances the unicode characters being read from the CSS data by one position. */ 18 | next(): boolean 19 | } 20 | 21 | /** The CSS iterator produces a sequence of CSS tokens in an iterator pattern. */ 22 | export type CSSIterator = { 23 | (): CSSIteration 24 | [Symbol.iterator](): { 25 | next: CSSIterator 26 | } 27 | } 28 | 29 | /** The CSS iteration represents the most recent state and token yielded from the CSS iterator. */ 30 | export type CSSIteration = { 31 | done: boolean 32 | value: CSSToken 33 | } 34 | 35 | export type CSSToken = { 36 | tick: number 37 | type: number 38 | code: number 39 | lead: string 40 | data: string 41 | tail: string 42 | } 43 | 44 | export type CSSTokenize = (data: string) => CSSIterator 45 | 46 | export declare const tokenize: CSSTokenize 47 | -------------------------------------------------------------------------------- /src/lib/token-types.scss.ts: -------------------------------------------------------------------------------- 1 | /** [``](https://drafts.csswg.org/css-syntax/#typedef-delim-token) */ 2 | export const SYMBOL = 0x0001 3 | 4 | /** [``](https://drafts.csswg.org/css-syntax/#comment-diagram) */ 5 | export const COMMENT = 0x0002 6 | 7 | /** [``](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 8 | export const SPACE = 0x0003 9 | 10 | /** [``](https://drafts.csswg.org/css-syntax/#ident-token-diagram) */ 11 | export const WORD = 0x0004 12 | 13 | /** [``](https://drafts.csswg.org/css-syntax/#function-token-diagram) */ 14 | export const FUNCTION = 0x0005 15 | 16 | /** [``](https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram) */ 17 | export const ATWORD = 0x0006 18 | 19 | /** [``](https://drafts.csswg.org/css-syntax/#hash-token-diagram) */ 20 | export const HASH = 0x0007 21 | 22 | /** [``](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 23 | export const STRING = 0x0008 24 | 25 | /** [``](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 26 | export const NUMBER = 0x0009 27 | 28 | /** [``](https://sass-lang.com/documentation/variables) */ 29 | export const VARIABLE = 0x0010 30 | -------------------------------------------------------------------------------- /src/tokenize.ts: -------------------------------------------------------------------------------- 1 | import { CSSState, CSSIterator, CSSIteration } from './types/global/global.js' 2 | import { consume } from './lib/consume.js' 3 | 4 | /** Returns a CSS iterator to yield tokens from the given CSS data. */ 5 | export const tokenize = (/** CSS data. */ data: string) => { 6 | let size = data.length 7 | let tick = 0 8 | 9 | /** Condition of the current tokenizer. */ 10 | let state: CSSState = { 11 | data, 12 | size, 13 | tick, 14 | codeAt0: tick + 0 < size ? data.charCodeAt(tick + 0) : -1, 15 | codeAt1: tick + 1 < size ? data.charCodeAt(tick + 1) : -1, 16 | codeAt2: tick + 2 < size ? data.charCodeAt(tick + 2) : -1, 17 | codeAt3: tick + 3 < size ? data.charCodeAt(tick + 3) : -1, 18 | /** Advances the unicode characters being read from the CSS data by one position. */ 19 | next() { 20 | state.tick = ++tick 21 | state.codeAt0 = state.codeAt1 22 | state.codeAt1 = state.codeAt2 23 | state.codeAt2 = state.codeAt3 24 | state.codeAt3 = tick + 3 < size ? data.charCodeAt(tick + 3) : -1 25 | return tick >= size 26 | } 27 | } 28 | 29 | /** Returns the most recent state and token yielded from the CSS iterator. */ 30 | const iterator: CSSIterator = ((): CSSIteration => ( 31 | state.tick >= state.size 32 | ? { 33 | done: true, 34 | value: { tick: state.tick, type: 0, code: -2, lead: '', data: '', tail: '' } 35 | } 36 | : { 37 | done: false, 38 | value: consume(state), 39 | } 40 | )) as CSSIterator 41 | 42 | iterator[Symbol.iterator] = () => ({ next: iterator }) 43 | 44 | return iterator 45 | } 46 | -------------------------------------------------------------------------------- /cmd/_.mjs: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import * as path from 'path' 3 | import * as process from 'child_process' 4 | import * as readline from 'readline' 5 | import * as url from 'url' 6 | 7 | export const __dirname = path.resolve(url.fileURLToPath(import.meta.url), '..', '..') 8 | export const __bin = path.resolve('node_modules', '.bin') 9 | export const __dist = path.resolve('dist') 10 | export const __modules = path.resolve('node_modules') 11 | 12 | export const resolve = (...paths) => path.resolve(__dirname, ...paths) 13 | 14 | export const spawn = (cmd, args, opts) => process.spawnSync(cmd, args, { cwd: __dirname, stdio: 'inherit', ...Object(opts) }) 15 | export const spawnNode = (cmd, args, opts) => spawn('node', [path.resolve(cmd), ...Array.from(Object(args))], opts) 16 | export const spawnTsc = (args, opts) => spawnNode('node_modules/typescript/lib/tsc.js', args, opts) 17 | export const spawnTscWatch = (args, opts) => spawnNode('node_modules/tsc-watch/lib/tsc-watch.js', args, opts) 18 | 19 | export const copyFile = (src, dest) => fs.copyFileSync(resolve(src), resolve(dest)) 20 | export const mkdir = (...paths) => fs.mkdirSync(resolve(...paths)) 21 | export const rmdir = (...paths) => (fs.rmSync || fs.rmdirSync)(resolve(...paths), { force: true, recursive: true }) 22 | 23 | export const question = query => new Promise(resolve => { 24 | const rl = readline.createInterface(question.options) 25 | rl.question('major / minor / patch? ', answer => { 26 | rl.close() 27 | resolve(answer) 28 | }) 29 | }) 30 | 31 | question.options = {} 32 | -------------------------------------------------------------------------------- /src/tokenize.scss.ts: -------------------------------------------------------------------------------- 1 | import { CSSState, CSSIterator, CSSIteration } from './types/global/global.js' 2 | import { consume } from './lib/consume.scss.js' 3 | 4 | /** Returns a CSS iterator to yield tokens from the given CSS data. */ 5 | export const tokenize = (/** CSS data. */ data: string) => { 6 | let size = data.length 7 | let tick = 0 8 | 9 | /** Condition of the current tokenizer. */ 10 | let state: CSSState = { 11 | data, 12 | size, 13 | tick, 14 | codeAt0: tick + 0 < size ? data.charCodeAt(tick + 0) : -1, 15 | codeAt1: tick + 1 < size ? data.charCodeAt(tick + 1) : -1, 16 | codeAt2: tick + 2 < size ? data.charCodeAt(tick + 2) : -1, 17 | codeAt3: tick + 3 < size ? data.charCodeAt(tick + 3) : -1, 18 | /** Advances the unicode characters being read from the CSS data by one position. */ 19 | next() { 20 | state.tick = ++tick 21 | state.codeAt0 = state.codeAt1 22 | state.codeAt1 = state.codeAt2 23 | state.codeAt2 = state.codeAt3 24 | state.codeAt3 = tick + 3 < size ? data.charCodeAt(tick + 3) : -1 25 | return tick >= size 26 | } 27 | } 28 | 29 | /** Returns the most recent state and token yielded from the CSS iterator. */ 30 | const iterator: CSSIterator = ((): CSSIteration => ( 31 | state.tick >= state.size 32 | ? { 33 | done: true, 34 | value: { tick: state.tick, type: 0, code: -2, lead: '', data: '', tail: '' } 35 | } 36 | : { 37 | done: false, 38 | value: consume(state), 39 | } 40 | )) as CSSIterator 41 | 42 | iterator[Symbol.iterator] = () => ({ next: iterator }) 43 | 44 | return iterator 45 | } 46 | -------------------------------------------------------------------------------- /src/lib/is.ts: -------------------------------------------------------------------------------- 1 | import * as cp from './code-points.js' 2 | 3 | /** Returns whether the unicode value is a digit. [↗](https://drafts.csswg.org/css-syntax/#digit) */ 4 | export const digit = (code: number) => code >= cp.DIGIT_ZERO && code <= cp.DIGIT_NINE 5 | 6 | /** Returns whether the unicode value is an identifier. [↗](https://drafts.csswg.org/css-syntax/#identifier-code-point) */ 7 | export const identifier = (code: number) => ( 8 | identifierStart(code) || 9 | (code >= cp.DIGIT_ZERO && code <= cp.DIGIT_NINE) || 10 | (code === cp.HYPHEN_MINUS) 11 | ) 12 | 13 | /** Returns whether the unicode value is an identifier-start. [↗](https://drafts.csswg.org/css-syntax/#identifier-start-code-point) */ 14 | export const identifierStart = (code: number) => ( 15 | (code === cp.LOW_LINE) || 16 | (code >= cp.NON_ASCII) || 17 | (code >= cp.LATIN_CAPITAL_LETTER_A && code <= cp.LATIN_CAPITAL_LETTER_Z) || 18 | (code >= cp.LATIN_SMALL_LETTER_A && code <= cp.LATIN_SMALL_LETTER_Z) 19 | ) 20 | 21 | /** Returns whether the unicode value is a space. [↗](https://drafts.csswg.org/css-syntax/#whitespace) */ 22 | export const space = (code: number) => ( 23 | code === cp.CHARACTER_TABULATION 24 | || code === cp.LINE_FEED 25 | || code === cp.FORM_FEED 26 | || code === cp.CARRIAGE_RETURN 27 | || code === cp.SPACE 28 | ) 29 | 30 | /** Returns whether the unicode values are a valid escape. [↗](https://drafts.csswg.org/css-syntax/#starts-with-a-valid-escape) */ 31 | export const validEscape = (code1of2: number, code2of2: number) => ( 32 | code1of2 === cp.REVERSE_SOLIDUS 33 | && !space(code2of2) 34 | ) 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/package", 3 | "name": "@csstools/tokenizer", 4 | "description": "Tokenize CSS according to the CSS Syntax", 5 | "version": "3.1.1", 6 | "type": "module", 7 | "main": "dist/tokenize.cjs", 8 | "module": "dist/tokenize.mjs", 9 | "types": "dist/tokenize.d.ts", 10 | "unpkg": "dist/tokenize.js", 11 | "exports": { 12 | ".": { 13 | "require": "./dist/tokenize.cjs", 14 | "import": "./dist/tokenize.mjs", 15 | "types": "./dist/tokenize.d.ts" 16 | }, 17 | "./tokenize": { 18 | "require": "./dist/tokenize.cjs", 19 | "import": "./dist/tokenize.mjs", 20 | "types": "./dist/tokenize.d.ts" 21 | }, 22 | "./tokenizeSCSS": { 23 | "require": "./dist/tokenizeSCSS.cjs", 24 | "import": "./dist/tokenizeSCSS.mjs", 25 | "types": "./dist/tokenize.d.ts" 26 | } 27 | }, 28 | "files": [ 29 | "dist" 30 | ], 31 | "engines": { 32 | "node": ">= 12" 33 | }, 34 | "scripts": { 35 | "build": "node cmd/build.mjs", 36 | "benchmark": "node cmd/benchmark.mjs", 37 | "prepublishOnly": "node cmd/publish.mjs", 38 | "test": "node cmd/test.mjs" 39 | }, 40 | "license": "CC0-1.0", 41 | "author": "Jonathan Neal ", 42 | "bugs": "https://github.com/csstools/tokenizer/issues", 43 | "homepage": "https://github.com/csstools/tokenizer", 44 | "repository": "https://github.com/csstools/postcss-tape.git", 45 | "publishConfig": { 46 | "access": "public" 47 | }, 48 | "devDependencies": { 49 | "@babel/core": "^7.15.0", 50 | "@babel/preset-env": "^7.15.0", 51 | "@babel/preset-typescript": "^7.15.0", 52 | "@rollup/plugin-babel": "^6.0.0", 53 | "@types/benchmark": "^2.1.1", 54 | "@types/jest": "^29.0.0", 55 | "@types/node": "^22.0.0", 56 | "benchmark": "^2.1.4", 57 | "bootstrap": "^5.1.0", 58 | "codecov": "^3.8.3", 59 | "css-tree1": "npm:css-tree@^1.1.3", 60 | "css-tree2": "npm:css-tree@^2.0.1", 61 | "jest": "^29.0.0", 62 | "magic-string": "^0.30.11", 63 | "postcss": "^8.4.31", 64 | "rollup": "^4.0.0", 65 | "@rollup/plugin-terser": "^0.4.4", 66 | "tailwindcss": "^2.2.8", 67 | "tsc-watch": "^6.0.0", 68 | "tslib": "^2.3.1", 69 | "typescript": "^5.0.0" 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/tokenize.benchmark.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import Benchmark from 'benchmark' 3 | import csstree1 from 'css-tree1' 4 | // @ts-expect-error no types 5 | import * as csstree2 from 'css-tree2/tokenizer' 6 | import postcss from 'postcss/lib/tokenize' 7 | 8 | import { tokenize } from './tokenize.js' 9 | 10 | const counter = { value: 0 } 11 | 12 | export declare type Suite = Omit & { 13 | filter(callback: string): Suite[] 14 | hz: number 15 | name: string 16 | tokensCount: number 17 | } 18 | 19 | const createCasePostCSS = (css: string) => () => { 20 | let count = 0 21 | const tokenizer = postcss({ css }) 22 | while (!tokenizer.endOfFile()) { 23 | tokenizer.nextToken({}) 24 | ++count 25 | } 26 | counter.value = count 27 | } 28 | 29 | const createCaseTokenizer = (css: string) => () => { 30 | let count = 1 31 | let tokenizer = tokenize(css) 32 | while (!tokenizer().done) ++count 33 | counter.value = count 34 | } 35 | 36 | const createCaseCssTree1 = (css: string) => () => { 37 | let count = 0 38 | const tokenStream = csstree1.tokenize(css) 39 | while (!tokenStream.eof) { 40 | tokenStream.next(); 41 | count++; 42 | } 43 | counter.value = count 44 | } 45 | 46 | const createCaseCssTree2 = (css: string) => () => { 47 | let count = 0 48 | csstree2.tokenize(css, () => count++); 49 | counter.value = count 50 | } 51 | 52 | const initializeBenchmark = (suite: Suite, css: string) => { 53 | counter.value = 0 54 | 55 | suite.add('CSSTree 1', createCaseCssTree1(css)) 56 | suite.add('CSSTree 2', createCaseCssTree2(css)) 57 | suite.add('PostCSS 8', createCasePostCSS(css)) 58 | suite.add('Tokenizer', createCaseTokenizer(css)) 59 | 60 | suite.on('cycle', (evt: { target: { tokensCount: number }}) => evt.target.tokensCount = counter.value) 61 | suite.on('complete', () => { 62 | const result: { [key: string]: { ms: number, 'ms/50k': number, tokens: number } } = {} 63 | const successful = Array.from(suite.filter('successful')) 64 | 65 | successful.forEach(target => { 66 | const ms = 1 / target.hz * 1000 67 | result[target.toString()] = { 68 | 'ms': Number(ms.toFixed(2)), 69 | 'ms/50k': Number(((50e3 * ms) / target.tokensCount).toFixed(2)), 70 | 'tokens': target.tokensCount, 71 | } 72 | }) 73 | 74 | console.log() 75 | console.group('Benchmark:', suite.name) 76 | console.table(result) 77 | console.groupEnd() 78 | }) 79 | 80 | return suite 81 | } 82 | 83 | const cssBootstrap = fs.readFileSync('./node_modules/bootstrap/dist/css/bootstrap.css', 'utf-8') 84 | const cssTailwind = fs.readFileSync('./node_modules/tailwindcss/dist/tailwind.css', 'utf-8') 85 | 86 | const suiteBootstrap = initializeBenchmark(new Benchmark.Suite('Bootstrap') as unknown as Suite, cssBootstrap) 87 | const suiteTailwind = initializeBenchmark(new Benchmark.Suite('Tailwind CSS') as unknown as Suite, cssTailwind) 88 | 89 | console.log() 90 | console.log('Starting benchmarking...') 91 | 92 | suiteTailwind.run() 93 | suiteBootstrap.run() 94 | 95 | console.log() 96 | console.log('Finished benchmarking.') 97 | -------------------------------------------------------------------------------- /rollup.config.mjs: -------------------------------------------------------------------------------- 1 | import * as path from 'path' 2 | import terser from '@rollup/plugin-terser' 3 | import babel from '@rollup/plugin-babel' 4 | import MagicString from 'magic-string' 5 | 6 | const babelOptions = { 7 | babelHelpers: 'bundled', 8 | extensions: ['.cjs', '.js', '.mjs', '.ts'], 9 | } 10 | 11 | const terserOptions = { 12 | ecma: 2020, 13 | compress: {}, 14 | } 15 | 16 | const resolveJsTsExtension = { 17 | name: 'resolve-js-ts-extension', 18 | resolveId(/** @type {string} */ importee, /** @type {string} */ importer) { 19 | if (importer?.endsWith('.ts') && importee) return path.join(path.dirname(importer), importee).replace(/(\.js)?$/, '.ts') 20 | return null 21 | } 22 | } 23 | 24 | const reduceImpact = { 25 | name: 'reduce-impact', 26 | renderChunk(code) { 27 | const str = new MagicString(code) 28 | str.trim(';').remove(0, 4) 29 | return { 30 | code: str.toString(), 31 | map: str.generateMap({ hires: true }), 32 | } 33 | } 34 | } 35 | 36 | const config = /** @type {import('rollup').NormalizedInputOptions[]} */ ([ 37 | { 38 | input: './src/tokenize.ts', 39 | output: { 40 | esModule: false, 41 | exports: 'named', 42 | file: './dist/tokenize.mjs', 43 | format: 'esm', 44 | strict: true, 45 | sourcemap: true, 46 | }, 47 | plugins: [ 48 | resolveJsTsExtension, 49 | babel({ ...babelOptions }), 50 | ], 51 | }, 52 | { 53 | input: './src/tokenize.ts', 54 | output: { 55 | esModule: false, 56 | exports: 'named', 57 | file: './dist/tokenize.cjs', 58 | format: 'cjs', 59 | strict: true, 60 | sourcemap: true, 61 | }, 62 | plugins: [ 63 | resolveJsTsExtension, 64 | babel({ ...babelOptions }), 65 | ], 66 | }, 67 | { 68 | input: './src/tokenize.iife.ts', 69 | output: { 70 | esModule: false, 71 | exports: 'default', 72 | file: './dist/tokenize.js', 73 | format: 'iife', 74 | name: 'tokenizeCSS', 75 | strict: false, 76 | sourcemap: false, 77 | }, 78 | plugins: [ 79 | resolveJsTsExtension, 80 | babel({ ...babelOptions }), 81 | terser({ ...terserOptions }), 82 | reduceImpact, 83 | ] 84 | }, 85 | { 86 | input: './src/tokenize.scss.ts', 87 | output: { 88 | esModule: false, 89 | exports: 'named', 90 | file: './dist/tokenizeSCSS.mjs', 91 | format: 'esm', 92 | strict: true, 93 | sourcemap: true, 94 | }, 95 | plugins: [ 96 | resolveJsTsExtension, 97 | babel({ ...babelOptions }), 98 | ], 99 | }, 100 | { 101 | input: './src/tokenize.scss.ts', 102 | output: { 103 | esModule: false, 104 | exports: 'named', 105 | file: './dist/tokenizeSCSS.cjs', 106 | format: 'cjs', 107 | strict: true, 108 | sourcemap: true, 109 | }, 110 | plugins: [ 111 | resolveJsTsExtension, 112 | babel({ ...babelOptions }), 113 | ], 114 | }, 115 | { 116 | input: './src/tokenize.scss.iife.ts', 117 | output: { 118 | esModule: false, 119 | exports: 'default', 120 | file: './dist/tokenizeSCSS.js', 121 | format: 'iife', 122 | name: 'tokenizeSCSS', 123 | strict: false, 124 | sourcemap: false, 125 | }, 126 | plugins: [ 127 | resolveJsTsExtension, 128 | babel({ ...babelOptions }), 129 | terser({ ...terserOptions }), 130 | reduceImpact, 131 | ] 132 | }, 133 | ]) 134 | 135 | export default config 136 | -------------------------------------------------------------------------------- /dist/tokenize.js: -------------------------------------------------------------------------------- 1 | tokenizeCSS=function(){const t=45,e=t=>t>=48&&t<=57,c=e=>d(e)||e>=48&&e<=57||e===t,d=t=>95===t||t>=128||t>=65&&t<=90||t>=97&&t<=122,a=t=>9===t||10===t||12===t||13===t||32===t,o=(t,e)=>92===t&&!a(e),{fromCharCode:i}=String,r=i=>{switch(!0){case 47===i.codeAt0:if(42===i.codeAt1)return l(i);break;case a(i.codeAt0):return u(i);case 34===i.codeAt0:case 39===i.codeAt0:return s(i);case 35===i.codeAt0:if(c(i.codeAt1))return{tick:i.tick,type:7,code:-1,lead:A(i),data:A(i)+k(i),tail:""};if(o(i.codeAt1,i.codeAt2))return{tick:i.tick,type:7,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""};break;case 92===i.codeAt0:if(o(i.codeAt0,i.codeAt1))return n(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+A(i)+A(i)+k(i),tail:""});break;case d(i.codeAt0):return n(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+k(i),tail:""});case i.codeAt0===t:if(i.codeAt1===t||d(i.codeAt1))return n(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+A(i)+k(i),tail:""});if(o(i.codeAt1,i.codeAt2))return n(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+A(i)+A(i)+k(i),tail:""});if(e(i.codeAt1))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+f(i),tail:h(i)};if(46===i.codeAt1&&e(i.codeAt2))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+A(i)+y(i),tail:h(i)};case 46===i.codeAt0:if(e(i.codeAt1))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+y(i),tail:h(i)};break;case 43===i.codeAt0:if(e(i.codeAt1))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+f(i),tail:h(i)};if(46===i.codeAt1&&e(i.codeAt2))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+A(i)+y(i),tail:h(i)};break;case e(i.codeAt0):return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+f(i),tail:h(i)};case 64===i.codeAt0:if(i.codeAt1===t){if(i.codeAt2===t)return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""};if(d(i.codeAt2))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""};if(o(i.codeAt2,i.codeAt3))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+A(i)+k(i),tail:""}}if(d(i.codeAt1))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+k(i),tail:""};if(o(i.codeAt1,i.codeAt2))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""}}return{tick:i.tick,type:1,code:i.codeAt0,lead:"",data:A(i),tail:""}},A=t=>{const e=i(t.codeAt0);return t.next(),e},k=t=>{let e="";for(;;){switch(!0){case o(t.codeAt0,t.codeAt1):e+=i(t.codeAt0),t.next();case c(t.codeAt0):e+=i(t.codeAt0),t.next();continue}break}return e},n=(t,e)=>(40===t.codeAt0&&(e.code=40,e.type=5,e.lead=e.data,e.data="(",t.next()),e),l=t=>{const e={tick:t.tick,type:2,code:-1,lead:"/*",data:"",tail:""};for(t.next(),t.next();t.tick{const e={tick:t.tick,type:3,code:-1,lead:"",data:A(t),tail:""};for(;t.tick{const{codeAt0:e}=t,c={tick:t.tick,type:8,code:-1,lead:"",data:A(t),tail:""};for(;t.tick{let c="";return c+=p(t),46===t.codeAt0&&e(t.codeAt1)&&(c+=A(t)+A(t)+p(t)),c+y(t)},y=c=>{let d="";if(d+=p(c),69===c.codeAt0||101===c.codeAt0)switch(!0){case 43===c.codeAt1||c.codeAt1===t:if(!e(c.codeAt2))break;d+=A(c);case e(c.codeAt1):d+=A(c)+A(c)+p(c)}return d},p=t=>{let c="";for(;t.ticke.codeAt0===t?e.codeAt1===t||d(e.codeAt1)?A(e)+A(e)+k(e):o(e.codeAt1,e.codeAt2)?A(e)+A(e)+A(e)+k(e):"":37===e.codeAt0?A(e):d(e.codeAt0)?A(e)+k(e):o(e.codeAt0,e.codeAt1)?A(e)+A(e)+k(e):"";return t=>{let e=t.length,c=0,d={data:t,size:e,tick:c,codeAt0:c+0(d.tick=++c,d.codeAt0=d.codeAt1,d.codeAt1=d.codeAt2,d.codeAt2=d.codeAt3,d.codeAt3=c+3=e)};const a=()=>d.tick>=d.size?{done:!0,value:{tick:d.tick,type:0,code:-2,lead:"",data:"",tail:""}}:{done:!1,value:r(d)};return a[Symbol.iterator]=()=>({next:a}),a}}() 2 | -------------------------------------------------------------------------------- /dist/tokenizeSCSS.js: -------------------------------------------------------------------------------- 1 | tokenizeSCSS=function(){const t=45,e=t=>t>=48&&t<=57,c=e=>d(e)||e>=48&&e<=57||e===t,d=t=>95===t||t>=128||t>=65&&t<=90||t>=97&&t<=122,a=t=>9===t||10===t||12===t||13===t||32===t,o=(t,e)=>92===t&&!a(e),{fromCharCode:i}=String,r=i=>{switch(!0){case 47===i.codeAt0:if(42===i.codeAt1)return n(i);break;case a(i.codeAt0):return u(i);case 34===i.codeAt0:case 39===i.codeAt0:return s(i);case 35===i.codeAt0:if(c(i.codeAt1))return{tick:i.tick,type:7,code:-1,lead:A(i),data:A(i)+k(i),tail:""};if(o(i.codeAt1,i.codeAt2))return{tick:i.tick,type:7,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""};break;case 36===i.codeAt0:if(c(i.codeAt1))return{tick:i.tick,type:16,code:-1,lead:A(i),data:A(i)+k(i),tail:""};if(o(i.codeAt1,i.codeAt2))return{tick:i.tick,type:16,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""};break;case 92===i.codeAt0:if(o(i.codeAt0,i.codeAt1))return l(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+A(i)+A(i)+k(i),tail:""});break;case d(i.codeAt0):return l(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+k(i),tail:""});case i.codeAt0===t:if(i.codeAt1===t||d(i.codeAt1))return l(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+A(i)+k(i),tail:""});if(o(i.codeAt1,i.codeAt2))return l(i,{tick:i.tick,type:4,code:-1,lead:"",data:A(i)+A(i)+A(i)+k(i),tail:""});if(e(i.codeAt1))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+f(i),tail:b(i)};if(46===i.codeAt1&&e(i.codeAt2))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+A(i)+y(i),tail:b(i)};case 46===i.codeAt0:if(e(i.codeAt1))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+y(i),tail:b(i)};break;case 43===i.codeAt0:if(e(i.codeAt1))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+f(i),tail:b(i)};if(46===i.codeAt1&&e(i.codeAt2))return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+A(i)+A(i)+y(i),tail:b(i)};break;case e(i.codeAt0):return{tick:i.tick,type:9,code:-1,lead:"",data:A(i)+f(i),tail:b(i)};case 64===i.codeAt0:if(i.codeAt1===t){if(i.codeAt2===t)return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""};if(d(i.codeAt2))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""};if(o(i.codeAt2,i.codeAt3))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+A(i)+k(i),tail:""}}if(d(i.codeAt1))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+k(i),tail:""};if(o(i.codeAt1,i.codeAt2))return{tick:i.tick,type:6,code:-1,lead:A(i),data:A(i)+A(i)+k(i),tail:""}}return{tick:i.tick,type:1,code:i.codeAt0,lead:"",data:A(i),tail:""}},A=t=>{const e=i(t.codeAt0);return t.next(),e},k=t=>{let e="";for(;;){switch(!0){case o(t.codeAt0,t.codeAt1):e+=i(t.codeAt0),t.next();case c(t.codeAt0):e+=i(t.codeAt0),t.next();continue}break}return e},l=(t,e)=>(40===t.codeAt0&&(e.code=40,e.type=5,e.lead=e.data,e.data="(",t.next()),e),n=t=>{const e={tick:t.tick,type:2,code:-1,lead:"/*",data:"",tail:""};for(t.next(),t.next();t.tick{const e={tick:t.tick,type:3,code:-1,lead:"",data:A(t),tail:""};for(;t.tick{const{codeAt0:e}=t,c={tick:t.tick,type:8,code:-1,lead:"",data:A(t),tail:""};for(;t.tick{let c="";return c+=p(t),46===t.codeAt0&&e(t.codeAt1)&&(c+=A(t)+A(t)+p(t)),c+y(t)},y=c=>{let d="";if(d+=p(c),69===c.codeAt0||101===c.codeAt0)switch(!0){case 43===c.codeAt1||c.codeAt1===t:if(!e(c.codeAt2))break;d+=A(c);case e(c.codeAt1):d+=A(c)+A(c)+p(c)}return d},p=t=>{let c="";for(;t.ticke.codeAt0===t?e.codeAt1===t||d(e.codeAt1)?A(e)+A(e)+k(e):o(e.codeAt1,e.codeAt2)?A(e)+A(e)+A(e)+k(e):"":d(e.codeAt0)?A(e)+k(e):o(e.codeAt0,e.codeAt1)?A(e)+A(e)+k(e):"";return t=>{let e=t.length,c=0,d={data:t,size:e,tick:c,codeAt0:c+0(d.tick=++c,d.codeAt0=d.codeAt1,d.codeAt1=d.codeAt2,d.codeAt2=d.codeAt3,d.codeAt3=c+3=e)};const a=()=>d.tick>=d.size?{done:!0,value:{tick:d.tick,type:0,code:-2,lead:"",data:"",tail:""}}:{done:!1,value:r(d)};return a[Symbol.iterator]=()=>({next:a}),a}}() 2 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # CC0 1.0 Universal 2 | 3 | ## Statement of Purpose 4 | 5 | The laws of most jurisdictions throughout the world automatically confer 6 | exclusive Copyright and Related Rights (defined below) upon the creator and 7 | subsequent owner(s) (each and all, an “owner”) of an original work of 8 | authorship and/or a database (each, a “Work”). 9 | 10 | Certain owners wish to permanently relinquish those rights to a Work for the 11 | purpose of contributing to a commons of creative, cultural and scientific works 12 | (“Commons”) that the public can reliably and without fear of later claims of 13 | infringement build upon, modify, incorporate in other works, reuse and 14 | redistribute as freely as possible in any form whatsoever and for any purposes, 15 | including without limitation commercial purposes. These owners may contribute 16 | to the Commons to promote the ideal of a free culture and the further 17 | production of creative, cultural and scientific works, or to gain reputation or 18 | greater distribution for their Work in part through the use and efforts of 19 | others. 20 | 21 | For these and/or other purposes and motivations, and without any expectation of 22 | additional consideration or compensation, the person associating CC0 with a 23 | Work (the “Affirmer”), to the extent that he or she is an owner of Copyright 24 | and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and 25 | publicly distribute the Work under its terms, with knowledge of his or her 26 | Copyright and Related Rights in the Work and the meaning and intended legal 27 | effect of CC0 on those rights. 28 | 29 | 1. Copyright and Related Rights. A Work made available under CC0 may be 30 | protected by copyright and related or neighboring rights (“Copyright and 31 | Related Rights”). Copyright and Related Rights include, but are not limited 32 | to, the following: 33 | 1. the right to reproduce, adapt, distribute, perform, display, communicate, 34 | and translate a Work; 35 | 2. moral rights retained by the original author(s) and/or performer(s); 36 | 3. publicity and privacy rights pertaining to a person’s image or likeness 37 | depicted in a Work; 38 | 4. rights protecting against unfair competition in regards to a Work, 39 | subject to the limitations in paragraph 4(i), below; 40 | 5. rights protecting the extraction, dissemination, use and reuse of data in 41 | a Work; 42 | 6. database rights (such as those arising under Directive 96/9/EC of the 43 | European Parliament and of the Council of 11 March 1996 on the legal 44 | protection of databases, and under any national implementation thereof, 45 | including any amended or successor version of such directive); and 46 | 7. other similar, equivalent or corresponding rights throughout the world 47 | based on applicable law or treaty, and any national implementations 48 | thereof. 49 | 50 | 2. Waiver. To the greatest extent permitted by, but not in contravention of, 51 | applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 52 | unconditionally waives, abandons, and surrenders all of Affirmer’s Copyright 53 | and Related Rights and associated claims and causes of action, whether now 54 | known or unknown (including existing as well as future claims and causes of 55 | action), in the Work (i) in all territories worldwide, (ii) for the maximum 56 | duration provided by applicable law or treaty (including future time 57 | extensions), (iii) in any current or future medium and for any number of 58 | copies, and (iv) for any purpose whatsoever, including without limitation 59 | commercial, advertising or promotional purposes (the “Waiver”). Affirmer 60 | makes the Waiver for the benefit of each member of the public at large and 61 | to the detriment of Affirmer’s heirs and successors, fully intending that 62 | such Waiver shall not be subject to revocation, rescission, cancellation, 63 | termination, or any other legal or equitable action to disrupt the quiet 64 | enjoyment of the Work by the public as contemplated by Affirmer’s express 65 | Statement of Purpose. 66 | 67 | 3. Public License Fallback. Should any part of the Waiver for any reason be 68 | judged legally invalid or ineffective under applicable law, then the Waiver 69 | shall be preserved to the maximum extent permitted taking into account 70 | Affirmer’s express Statement of Purpose. In addition, to the extent the 71 | Waiver is so judged Affirmer hereby grants to each affected person a 72 | royalty-free, non transferable, non sublicensable, non exclusive, 73 | irrevocable and unconditional license to exercise Affirmer’s Copyright and 74 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 75 | maximum duration provided by applicable law or treaty (including future time 76 | extensions), (iii) in any current or future medium and for any number of 77 | copies, and (iv) for any purpose whatsoever, including without limitation 78 | commercial, advertising or promotional purposes (the “License”). The License 79 | shall be deemed effective as of the date CC0 was applied by Affirmer to the 80 | Work. Should any part of the License for any reason be judged legally 81 | invalid or ineffective under applicable law, such partial invalidity or 82 | ineffectiveness shall not invalidate the remainder of the License, and in 83 | such case Affirmer hereby affirms that he or she will not (i) exercise any 84 | of his or her remaining Copyright and Related Rights in the Work or (ii) 85 | assert any associated claims and causes of action with respect to the Work, 86 | in either case contrary to Affirmer’s express Statement of Purpose. 87 | 88 | 4. Limitations and Disclaimers. 89 | 1. No trademark or patent rights held by Affirmer are waived, abandoned, 90 | surrendered, licensed or otherwise affected by this document. 91 | 2. Affirmer offers the Work as-is and makes no representations or warranties 92 | of any kind concerning the Work, express, implied, statutory or 93 | otherwise, including without limitation warranties of title, 94 | merchantability, fitness for a particular purpose, non infringement, or 95 | the absence of latent or other defects, accuracy, or the present or 96 | absence of errors, whether or not discoverable, all to the greatest 97 | extent permissible under applicable law. 98 | 3. Affirmer disclaims responsibility for clearing rights of other persons 99 | that may apply to the Work or any use thereof, including without 100 | limitation any person’s Copyright and Related Rights in the Work. 101 | Further, Affirmer disclaims responsibility for obtaining any necessary 102 | consents, permissions or other rights required for any use of the Work. 103 | 4. Affirmer understands and acknowledges that Creative Commons is not a 104 | party to this document and has no duty or obligation with respect to this 105 | CC0 or use of the Work. 106 | 107 | For more information, please see 108 | http://creativecommons.org/publicdomain/zero/1.0/. 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CSS Tokenizer 2 | 3 | [npm version](https://www.npmjs.com/package/@csstools/tokenizer) 4 | [build status](https://travis-ci.org/github/csstools/tokenizer) 5 | [code coverage](https://codecov.io/gh/csstools/tokenizer) 6 | [issue tracker](https://github.com/csstools/tokenizer/issues) 7 | [pull requests](https://github.com/csstools/tokenizer/pulls) 8 | [support chat](https://gitter.im/postcss/postcss) 9 | 10 | This tools lets you tokenize CSS according to the [CSS Syntax Specification](https://drafts.csswg.org/css-syntax/). 11 | Tokenizing CSS is separating a string of CSS into its smallest, semantic parts — otherwise known as tokens. 12 | 13 | This tool is intended to be used in other tools on the front and back end. It seeks to maintain: 14 | 15 | - 100% compliance with the CSS syntax specification. ✨ 16 | - 100% code coverage. 🦺 17 | - 100% static typing. 💪 18 | - 1kB maximum contribution size. 📦 19 | - Superior quality over Shark P. 🦈 20 | 21 | ## Usage 22 | 23 | Add the [CSS tokenizer](https://github.com/csstools/tokenizer) to your project: 24 | 25 | ```sh 26 | npm install @csstools/tokenizer 27 | ``` 28 | 29 | Tokenize CSS in JavaScript: 30 | 31 | ```js 32 | import { tokenize } from '@csstools/tokenizer' 33 | 34 | for (const token of tokenize(cssText)) { 35 | console.log(token) // logs an individual CSSToken 36 | } 37 | ``` 38 | 39 | Tokenize CSS in _classical_ NodeJS: 40 | 41 | ```js 42 | const { tokenizer } = require('@csstools/tokenizer') 43 | 44 | let iterator = tokenizer(cssText), iteration 45 | 46 | while (!(iteration = iterator()).done) { 47 | console.log(iteration.value) // logs an individual CSSToken 48 | } 49 | ``` 50 | 51 | Tokenize CSS in client-side scripts: 52 | 53 | ```html 54 | 63 | ``` 64 | 65 | Tokenize CSS in _classical_ client-side scripts: 66 | 67 | ```html 68 | 69 | 74 | ``` 75 | 76 | ### Serialize tokens 77 | 78 | ```js 79 | import { tokenize } from '@csstools/tokenizer' 80 | 81 | let cssOutput = ''; 82 | for (const token of tokenize(cssText)) { 83 | // mutate some tokens 84 | 85 | cssOutput += token.lead + token.data + token.tail 86 | } 87 | 88 | console.log(cssOutput) // logs the CSS string 89 | ``` 90 | 91 | ## How it works 92 | 93 | The CSS tokenizer separates a string of CSS into tokens. 94 | 95 | ```ts 96 | interface CSSToken { 97 | /** Position in the string at which the token was retrieved. */ 98 | tick: number 99 | 100 | /** Number identifying the kind of token. */ 101 | type: 102 | | 1 // Symbol 103 | | 2 // Comment 104 | | 3 // Space 105 | | 4 // Word 106 | | 5 // Function 107 | | 6 // Atword 108 | | 7 // Hash 109 | | 8 // String 110 | | 9 // Number 111 | 112 | /** Code, like the character code of a symbol, or the character code of the opening parenthesis of a function. */ 113 | code: number 114 | 115 | /** Lead, like the opening of a comment, the quotation mark of a string, or the name of a function. */ 116 | lead: string, 117 | 118 | /** Data, like the numbers before a unit, the word after an at-sign, or the opening parenthesis of a Function. */ 119 | data: string, 120 | 121 | /** Tail, like the unit after a number, or the closing of a comment. */ 122 | tail: string, 123 | } 124 | ``` 125 | 126 | As an example, the CSS string `@media` would become a **Atword** token where `@` and `media` are recognized as distinct parts of that token. As another example, the CSS string `5px` would become a **Number** token where `5` and `px` are recognized as distinct parts of that token. As a final example, the string `5px 10px` would become 3 tokens; the **Number** as mentioned before (`5px`), a **Space** token that represents a single space (` `), and then another **Number** token (`10px`). 127 | 128 | ## Benchmarks 129 | 130 | As of August 23, 2021, these benchmarks were averaged from my local machine: 131 | 132 | ``` 133 | Benchmark: Tailwind CSS 134 | ┌────────────────────────────────────────────────────┬───────┬────────┬────────┐ 135 | │ (index) │ ms │ ms/50k │ tokens │ 136 | ├────────────────────────────────────────────────────┼───────┼────────┼────────┤ 137 | │ CSSTree 1 x 35.04 ops/sec ±6.55% (64 runs sampled) │ 28.54 │ 1.51 │ 946205 │ 138 | │ CSSTree 2 x 41.76 ops/sec ±7.57% (58 runs sampled) │ 23.95 │ 1.27 │ 946205 │ 139 | │ PostCSS 8 x 14.18 ops/sec ±3.31% (40 runs sampled) │ 70.54 │ 3.77 │ 935282 │ 140 | │ Tokenizer x 17.40 ops/sec ±0.98% (48 runs sampled) │ 57.48 │ 3.04 │ 946206 │ 141 | └────────────────────────────────────────────────────┴───────┴────────┴────────┘ 142 | 143 | Benchmark: Bootstrap 144 | ┌───────────────────────────────────────────────────┬──────┬────────┬────────┐ 145 | │ (index) │ ms │ ms/50k │ tokens │ 146 | ├───────────────────────────────────────────────────┼──────┼────────┼────────┤ 147 | │ CSSTree 1 x 600 ops/sec ±0.87% (96 runs sampled) │ 1.67 │ 1.41 │ 59236 │ 148 | │ CSSTree 2 x 695 ops/sec ±0.08% (100 runs sampled) │ 1.44 │ 1.21 │ 59236 │ 149 | │ PostCSS 8 x 432 ops/sec ±0.94% (94 runs sampled) │ 2.31 │ 2.26 │ 51170 │ 150 | │ Tokenizer x 288 ops/sec ±0.40% (93 runs sampled) │ 3.48 │ 2.93 │ 59237 │ 151 | └───────────────────────────────────────────────────┴──────┴────────┴────────┘ 152 | ``` 153 | 154 | ## Development 155 | 156 | You wanna take a deeper dive? Awesome! Here are a few useful development commands. 157 | 158 | ### npm run build 159 | 160 | The **build** command creates all the files needed to run this tool in many different JavaScript environments. 161 | 162 | ```sh 163 | npm run build 164 | ``` 165 | 166 | ### npm run benchmark 167 | 168 | The **benchmark** command builds the project and then tests its performance as compared to [PostCSS]. 169 | These benchmarks are run against [Boostrap] and [Tailwind CSS]. 170 | 171 | ```sh 172 | npm run benchmark 173 | ``` 174 | 175 | ### npm run test 176 | 177 | The **test** command tests the coverage and accuracy of the tokenizer. 178 | 179 | As of September 26, 2020, this tokenizer has 100% test coverage: 180 | 181 | ```sh 182 | npm run test 183 | ``` 184 | 185 | [Boostrap]: https://getbootstrap.com 186 | [PostCSS]: https://postcss.org 187 | [Tailwind CSS]: https://tailwindcss.com 188 | -------------------------------------------------------------------------------- /src/tokenize.scss.test.ts: -------------------------------------------------------------------------------- 1 | import { CSSToken } from './types/global/global.js' 2 | import * as fs from 'fs' 3 | import { tokenize } from './tokenize.scss.js' 4 | 5 | describe('Tokenization', () => { 6 | test('Tokenizing an empty value', () => { 7 | let sourceCSS = '' 8 | let resultCSS = '' 9 | let iterator = tokenize(sourceCSS) 10 | let iteration 11 | while (!(iteration = iterator()).done) { 12 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 13 | } 14 | expect(sourceCSS).toBe(resultCSS) 15 | }) 16 | 17 | test('Tokenizing a broken comment', () => { 18 | let sourceCSS = '/* ' 19 | let resultCSS = '' 20 | let iterator = tokenize(sourceCSS) 21 | let iteration 22 | let count = 0 23 | while (!(iteration = iterator()).done) { 24 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 25 | ++count 26 | } 27 | expect(sourceCSS).toBe(resultCSS) 28 | expect(count).toBe(1) 29 | }) 30 | 31 | test('Tokenizing usual values', () => { 32 | expect(Array.from(tokenize(`/**/`))).toHaveLength(1) 33 | expect(Array.from(tokenize(`/* */`))).toHaveLength(1) 34 | expect(Array.from(tokenize(`/***/`))).toHaveLength(1) 35 | expect(Array.from(tokenize(`/** */`))).toHaveLength(1) 36 | expect(Array.from(tokenize(`/* **/`))).toHaveLength(1) 37 | expect(Array.from(tokenize(`/*/**/`))).toHaveLength(1) 38 | 39 | expect(Array.from(tokenize(` \n \t \f `))).toHaveLength(1) 40 | 41 | expect(Array.from(tokenize(`"hello"` + `'hello'`))).toHaveLength(2) 42 | expect(Array.from(tokenize(`"\\"hello"` + `'\\'hello'` + `"hello\\""` + `'hello\\''`))).toHaveLength(4) 43 | expect(Array.from(tokenize(`"h\\"ello"` + `'h\\'ello'` + `"hell\\"o"` + `'hell\\'o'`))).toHaveLength(4) 44 | expect(Array.from(tokenize(`"\\\nhello"` + `'\\\nhello'` + `"hello\\\n"` + `'hello\\\n'`))).toHaveLength(4) 45 | expect(Array.from(tokenize(`"h\\\nello"` + `'h\\\nello'` + `"hell\\\no"` + `'hell\\\no'`))).toHaveLength(4) 46 | 47 | expect(Array.from(tokenize(`#_` + `#™` + `#A` + `#E` + `#Z` + `#a` + `#e` + `#z`))).toHaveLength(8) 48 | 49 | expect(Array.from(tokenize(`0`))).toHaveLength(1) 50 | expect(Array.from(tokenize(`+0`))).toHaveLength(1) 51 | expect(Array.from(tokenize(`-0`))).toHaveLength(1) 52 | expect(Array.from(tokenize(`.0`))).toHaveLength(1) 53 | expect(Array.from(tokenize(`+.0`))).toHaveLength(1) 54 | expect(Array.from(tokenize(`-.0`))).toHaveLength(1) 55 | expect(Array.from(tokenize(`1em`))).toHaveLength(1) 56 | expect(Array.from(tokenize(`+1em`))).toHaveLength(1) 57 | expect(Array.from(tokenize(`-1em`))).toHaveLength(1) 58 | expect(Array.from(tokenize(`.1em`))).toHaveLength(1) 59 | expect(Array.from(tokenize(`+.1em`))).toHaveLength(1) 60 | expect(Array.from(tokenize(`-.1em`))).toHaveLength(1) 61 | }) 62 | 63 | test('Tokenizing delimiters', () => { 64 | expect(Array.from(tokenize(`*`))).toHaveLength(1) 65 | expect(Array.from(tokenize(`.`))).toHaveLength(1) 66 | expect(Array.from(tokenize(`#`))).toHaveLength(1) 67 | expect(Array.from(tokenize(`@`))).toHaveLength(1) 68 | }) 69 | 70 | test('Tokenizing unusual beginnings', () => { 71 | expect(Array.from(tokenize(`@_` + `@™` + `@A` + `@E` + `@Z` + `@a` + `@e` + `@z`))).toHaveLength(8) 72 | expect(Array.from(tokenize(`@-_` + `@-™` + `@-A` + `@-E` + `@-Z` + `@-a` + `@-e` + `@-z`))).toHaveLength(8) 73 | 74 | expect(Array.from(tokenize(`--`))).toHaveLength(1) 75 | expect(Array.from(tokenize(`@--`))).toHaveLength(1) 76 | 77 | expect(Array.from(tokenize(`\\^-_` + `#\\^-_` + `@\\^-_`))).toHaveLength(3) 78 | expect(Array.from(tokenize(`-\\^-_` + `#-\\^-_` + `@-\\^-_`))).toHaveLength(3) 79 | expect(Array.from(tokenize(`-A\\^-_` + `#A\\^-_` + `@A-\\^-_`))).toHaveLength(3) 80 | }) 81 | 82 | test('Tokenizing unusual numbers', () => { 83 | let tokens: CSSToken[] 84 | 85 | tokens = Array.from(tokenize(`1em`)) 86 | expect(tokens).toHaveLength(1) 87 | 88 | tokens = [ 89 | ...Array.from(tokenize(`.25rem`)), 90 | ] 91 | expect(tokens).toHaveLength(1) 92 | 93 | tokens = [ 94 | ...Array.from(tokenize(`1_`)), 95 | ...Array.from(tokenize(`1™`)), 96 | ...Array.from(tokenize(`1A`)), 97 | ...Array.from(tokenize(`1E`)), 98 | ...Array.from(tokenize(`1Z`)), 99 | ...Array.from(tokenize(`1a`)), 100 | ...Array.from(tokenize(`1e`)), 101 | ...Array.from(tokenize(`1z`)), 102 | ] 103 | expect(tokens).toHaveLength(8) 104 | tokens = [ 105 | ...Array.from(tokenize(`1-_`)), 106 | ...Array.from(tokenize(`1-™`)), 107 | ...Array.from(tokenize(`1-A`)), 108 | ...Array.from(tokenize(`1-E`)), 109 | ...Array.from(tokenize(`1-Z`)), 110 | ...Array.from(tokenize(`1-a`)), 111 | ...Array.from(tokenize(`1-e`)), 112 | ...Array.from(tokenize(`1-z`)), 113 | ] 114 | expect(tokens).toHaveLength(8) 115 | tokens = [ 116 | ...Array.from(tokenize(`1\\^-_`)), 117 | ...Array.from(tokenize(`1-\\^-_`)), 118 | ] 119 | expect(tokens).toHaveLength(2) 120 | 121 | expect(Array.from(tokenize(`1--`))).toHaveLength(1) 122 | expect(Array.from(tokenize(`1\\^`))).toHaveLength(1) 123 | 124 | expect(Array.from(tokenize(`1e1em`))).toHaveLength(1) 125 | expect(Array.from(tokenize(`1e+1em`))).toHaveLength(1) 126 | expect(Array.from(tokenize(`1e-1em`))).toHaveLength(1) 127 | expect(Array.from(tokenize(`+1em`))).toHaveLength(1) 128 | expect(Array.from(tokenize(`+1e5em`))).toHaveLength(1) 129 | expect(Array.from(tokenize(`-1em`))).toHaveLength(1) 130 | expect(Array.from(tokenize(`-1e5em`))).toHaveLength(1) 131 | expect(Array.from(tokenize(`.1em`))).toHaveLength(1) 132 | expect(Array.from(tokenize(`+.1em`))).toHaveLength(1) 133 | expect(Array.from(tokenize(`-.1em`))).toHaveLength(1) 134 | }) 135 | 136 | test('Tokenizing unusual breaks', () => { 137 | expect(Array.from(tokenize(`@\n`))).toHaveLength(2) 138 | expect(Array.from(tokenize(`@\\\n`))).toHaveLength(3) 139 | expect(Array.from(tokenize(`@A\n`))).toHaveLength(2) 140 | expect(Array.from(tokenize(`@A\\\n`))).toHaveLength(3) 141 | expect(Array.from(tokenize(`@-\n`))).toHaveLength(3) 142 | expect(Array.from(tokenize(`@-\\\n`))).toHaveLength(4) 143 | 144 | expect(Array.from(tokenize(`1` + `\n`))).toHaveLength(2) 145 | expect(Array.from(tokenize(`1` + `\\` + `\n`))).toHaveLength(3) 146 | expect(Array.from(tokenize(`1` + `-` + `\n`))).toHaveLength(3) 147 | expect(Array.from(tokenize(`1` + `-` + `\\` + `\n`))).toHaveLength(4) 148 | 149 | expect(Array.from(tokenize(`-` + `.`))).toHaveLength(2) 150 | expect(Array.from(tokenize(`-` + `.` + `+8`))).toHaveLength(3) 151 | expect(Array.from(tokenize(`+` + `.` + `-8`))).toHaveLength(3) 152 | expect(Array.from(tokenize(`8e` + `+` + `-`))).toHaveLength(3) 153 | expect(Array.from(tokenize(`8e-` + `+`))).toHaveLength(2) 154 | }) 155 | 156 | test('Tokenizing sass variables', () => { 157 | // sass variables 158 | expect(Array.from(tokenize(`$a`))).toHaveLength(1) 159 | expect(Array.from(tokenize(`$ab`))).toHaveLength(1) 160 | expect(Array.from(tokenize(`$a-b`))).toHaveLength(1) 161 | expect(Array.from(tokenize(`$ab-`))).toHaveLength(1) 162 | expect(Array.from(tokenize(`$1`))).toHaveLength(1) 163 | expect(Array.from(tokenize(`$123`))).toHaveLength(1) 164 | expect(Array.from(tokenize(`$\\$`))).toHaveLength(1) 165 | 166 | // not quite sass variables 167 | expect(Array.from(tokenize(`-$a`))).toHaveLength(2) 168 | expect(Array.from(tokenize(`$$`))).toHaveLength(2) 169 | }) 170 | }) 171 | 172 | describe('Library accuracy', () => { 173 | test('Bootstrap', () => { 174 | let sourceCSS = fs.readFileSync('./node_modules/bootstrap/dist/css/bootstrap.css', 'utf-8') 175 | let resultCSS = '' 176 | let iterator = tokenize(sourceCSS) 177 | let iteration 178 | while (!(iteration = iterator()).done) { 179 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 180 | } 181 | expect(sourceCSS).toBe(resultCSS) 182 | }) 183 | 184 | test('Bootstrap [Symbol.iterator]', () => { 185 | let sourceCSS = fs.readFileSync('./node_modules/bootstrap/dist/css/bootstrap.css', 'utf-8') 186 | let resultCSS = Array.from(tokenize(sourceCSS), token => token.lead + token.data + token.tail).join('') 187 | expect(sourceCSS).toBe(resultCSS) 188 | }) 189 | 190 | test('Tailwind CSS', () => { 191 | let sourceCSS = fs.readFileSync('./node_modules/tailwindcss/dist/tailwind.css', 'utf-8') 192 | let resultCSS = '' 193 | let iterator = tokenize(sourceCSS) 194 | let iteration 195 | while (!(iteration = iterator()).done) { 196 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 197 | } 198 | expect(sourceCSS).toBe(resultCSS) 199 | }) 200 | 201 | test('Tailwind CSS [Symbol.iterator]', () => { 202 | let sourceCSS = fs.readFileSync('./node_modules/tailwindcss/dist/tailwind.css', 'utf-8') 203 | let resultCSS = Array.from(tokenize(sourceCSS), token => token.lead + token.data + token.tail).join('') 204 | expect(sourceCSS).toBe(resultCSS) 205 | }) 206 | }) 207 | -------------------------------------------------------------------------------- /src/tokenize.test.ts: -------------------------------------------------------------------------------- 1 | import { CSSToken } from './types/global/global.js' 2 | import * as fs from 'fs' 3 | import { tokenize } from './tokenize.js' 4 | 5 | describe('Tokenization', () => { 6 | test('Tokenizing an empty value', () => { 7 | let sourceCSS = '' 8 | let resultCSS = '' 9 | let iterator = tokenize(sourceCSS) 10 | let iteration 11 | while (!(iteration = iterator()).done) { 12 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 13 | } 14 | expect(sourceCSS).toBe(resultCSS) 15 | }) 16 | 17 | test('Tokenizing a broken comment', () => { 18 | let sourceCSS = '/* ' 19 | let resultCSS = '' 20 | let iterator = tokenize(sourceCSS) 21 | let iteration 22 | let count = 0 23 | while (!(iteration = iterator()).done) { 24 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 25 | ++count 26 | } 27 | expect(sourceCSS).toBe(resultCSS) 28 | expect(count).toBe(1) 29 | }) 30 | 31 | test('Tokenizing usual values', () => { 32 | expect(Array.from(tokenize(`/**/`))).toHaveLength(1) 33 | expect(Array.from(tokenize(`/* */`))).toHaveLength(1) 34 | expect(Array.from(tokenize(`/***/`))).toHaveLength(1) 35 | expect(Array.from(tokenize(`/** */`))).toHaveLength(1) 36 | expect(Array.from(tokenize(`/* **/`))).toHaveLength(1) 37 | expect(Array.from(tokenize(`/*/**/`))).toHaveLength(1) 38 | 39 | expect(Array.from(tokenize(` \n \t \f `))).toHaveLength(1) 40 | 41 | expect(Array.from(tokenize(`"hello"` + `'hello'`))).toHaveLength(2) 42 | expect(Array.from(tokenize(`"\\"hello"` + `'\\'hello'` + `"hello\\""` + `'hello\\''`))).toHaveLength(4) 43 | expect(Array.from(tokenize(`"h\\"ello"` + `'h\\'ello'` + `"hell\\"o"` + `'hell\\'o'`))).toHaveLength(4) 44 | expect(Array.from(tokenize(`"\\\nhello"` + `'\\\nhello'` + `"hello\\\n"` + `'hello\\\n'`))).toHaveLength(4) 45 | expect(Array.from(tokenize(`"h\\\nello"` + `'h\\\nello'` + `"hell\\\no"` + `'hell\\\no'`))).toHaveLength(4) 46 | 47 | expect(Array.from(tokenize(`#_` + `#™` + `#A` + `#E` + `#Z` + `#a` + `#e` + `#z`))).toHaveLength(8) 48 | 49 | expect(Array.from(tokenize(`0`))).toHaveLength(1) 50 | expect(Array.from(tokenize(`+0`))).toHaveLength(1) 51 | expect(Array.from(tokenize(`-0`))).toHaveLength(1) 52 | expect(Array.from(tokenize(`.0`))).toHaveLength(1) 53 | expect(Array.from(tokenize(`+.0`))).toHaveLength(1) 54 | expect(Array.from(tokenize(`-.0`))).toHaveLength(1) 55 | expect(Array.from(tokenize(`1em`))).toHaveLength(1) 56 | expect(Array.from(tokenize(`+1em`))).toHaveLength(1) 57 | expect(Array.from(tokenize(`-1em`))).toHaveLength(1) 58 | expect(Array.from(tokenize(`.1em`))).toHaveLength(1) 59 | expect(Array.from(tokenize(`+.1em`))).toHaveLength(1) 60 | expect(Array.from(tokenize(`-.1em`))).toHaveLength(1) 61 | 62 | expect(Array.from(tokenize(`0%`))).toHaveLength(1) 63 | expect(Array.from(tokenize(`+0%`))).toHaveLength(1) 64 | expect(Array.from(tokenize(`-0%`))).toHaveLength(1) 65 | expect(Array.from(tokenize(`.0%`))).toHaveLength(1) 66 | expect(Array.from(tokenize(`+.0%`))).toHaveLength(1) 67 | expect(Array.from(tokenize(`-.0%`))).toHaveLength(1) 68 | expect(Array.from(tokenize(`1%`))).toHaveLength(1) 69 | expect(Array.from(tokenize(`+1%`))).toHaveLength(1) 70 | expect(Array.from(tokenize(`-1%`))).toHaveLength(1) 71 | expect(Array.from(tokenize(`.1%`))).toHaveLength(1) 72 | expect(Array.from(tokenize(`+.1%`))).toHaveLength(1) 73 | expect(Array.from(tokenize(`-.1%`))).toHaveLength(1) 74 | }) 75 | 76 | test('Tokenizing delimiters', () => { 77 | expect(Array.from(tokenize(`*`))).toHaveLength(1) 78 | expect(Array.from(tokenize(`.`))).toHaveLength(1) 79 | expect(Array.from(tokenize(`#`))).toHaveLength(1) 80 | expect(Array.from(tokenize(`@`))).toHaveLength(1) 81 | }) 82 | 83 | test('Tokenizing unusual beginnings', () => { 84 | expect(Array.from(tokenize(`@_` + `@™` + `@A` + `@E` + `@Z` + `@a` + `@e` + `@z`))).toHaveLength(8) 85 | expect(Array.from(tokenize(`@-_` + `@-™` + `@-A` + `@-E` + `@-Z` + `@-a` + `@-e` + `@-z`))).toHaveLength(8) 86 | 87 | expect(Array.from(tokenize(`--`))).toHaveLength(1) 88 | expect(Array.from(tokenize(`@--`))).toHaveLength(1) 89 | 90 | expect(Array.from(tokenize(`\\^-_` + `#\\^-_` + `@\\^-_`))).toHaveLength(3) 91 | expect(Array.from(tokenize(`-\\^-_` + `#-\\^-_` + `@-\\^-_`))).toHaveLength(3) 92 | expect(Array.from(tokenize(`-A\\^-_` + `#A\\^-_` + `@A-\\^-_`))).toHaveLength(3) 93 | }) 94 | 95 | test('Tokenizing unusual numbers', () => { 96 | let tokens: CSSToken[] 97 | 98 | tokens = Array.from(tokenize(`1em`)) 99 | expect(tokens).toHaveLength(1) 100 | 101 | tokens = [ 102 | ...Array.from(tokenize(`.25rem`)), 103 | ] 104 | expect(tokens).toHaveLength(1) 105 | 106 | tokens = [ 107 | ...Array.from(tokenize(`1_`)), 108 | ...Array.from(tokenize(`1™`)), 109 | ...Array.from(tokenize(`1A`)), 110 | ...Array.from(tokenize(`1E`)), 111 | ...Array.from(tokenize(`1Z`)), 112 | ...Array.from(tokenize(`1a`)), 113 | ...Array.from(tokenize(`1e`)), 114 | ...Array.from(tokenize(`1z`)), 115 | ] 116 | expect(tokens).toHaveLength(8) 117 | tokens = [ 118 | ...Array.from(tokenize(`1-_`)), 119 | ...Array.from(tokenize(`1-™`)), 120 | ...Array.from(tokenize(`1-A`)), 121 | ...Array.from(tokenize(`1-E`)), 122 | ...Array.from(tokenize(`1-Z`)), 123 | ...Array.from(tokenize(`1-a`)), 124 | ...Array.from(tokenize(`1-e`)), 125 | ...Array.from(tokenize(`1-z`)), 126 | ] 127 | expect(tokens).toHaveLength(8) 128 | tokens = [ 129 | ...Array.from(tokenize(`1\\^-_`)), 130 | ...Array.from(tokenize(`1-\\^-_`)), 131 | ] 132 | expect(tokens).toHaveLength(2) 133 | 134 | expect(Array.from(tokenize(`1--`))).toHaveLength(1) 135 | expect(Array.from(tokenize(`1\\^`))).toHaveLength(1) 136 | 137 | expect(Array.from(tokenize(`1e1em`))).toHaveLength(1) 138 | expect(Array.from(tokenize(`1e+1em`))).toHaveLength(1) 139 | expect(Array.from(tokenize(`1e-1em`))).toHaveLength(1) 140 | expect(Array.from(tokenize(`+1em`))).toHaveLength(1) 141 | expect(Array.from(tokenize(`+1e5em`))).toHaveLength(1) 142 | expect(Array.from(tokenize(`-1em`))).toHaveLength(1) 143 | expect(Array.from(tokenize(`-1e5em`))).toHaveLength(1) 144 | expect(Array.from(tokenize(`.1em`))).toHaveLength(1) 145 | expect(Array.from(tokenize(`+.1em`))).toHaveLength(1) 146 | expect(Array.from(tokenize(`-.1em`))).toHaveLength(1) 147 | }) 148 | 149 | test('Tokenizing unusual breaks', () => { 150 | expect(Array.from(tokenize(`@\n`))).toHaveLength(2) 151 | expect(Array.from(tokenize(`@\\\n`))).toHaveLength(3) 152 | expect(Array.from(tokenize(`@A\n`))).toHaveLength(2) 153 | expect(Array.from(tokenize(`@A\\\n`))).toHaveLength(3) 154 | expect(Array.from(tokenize(`@-\n`))).toHaveLength(3) 155 | expect(Array.from(tokenize(`@-\\\n`))).toHaveLength(4) 156 | 157 | expect(Array.from(tokenize(`1` + `\n`))).toHaveLength(2) 158 | expect(Array.from(tokenize(`1` + `\\` + `\n`))).toHaveLength(3) 159 | expect(Array.from(tokenize(`1` + `-` + `\n`))).toHaveLength(3) 160 | expect(Array.from(tokenize(`1` + `-` + `\\` + `\n`))).toHaveLength(4) 161 | 162 | expect(Array.from(tokenize(`-` + `.`))).toHaveLength(2) 163 | expect(Array.from(tokenize(`-` + `.` + `+8`))).toHaveLength(3) 164 | expect(Array.from(tokenize(`+` + `.` + `-8`))).toHaveLength(3) 165 | expect(Array.from(tokenize(`8e` + `+` + `-`))).toHaveLength(3) 166 | expect(Array.from(tokenize(`8e-` + `+`))).toHaveLength(2) 167 | }) 168 | }) 169 | 170 | describe('Library accuracy', () => { 171 | test('Bootstrap', () => { 172 | let sourceCSS = fs.readFileSync('./node_modules/bootstrap/dist/css/bootstrap.css', 'utf-8') 173 | let resultCSS = '' 174 | let iterator = tokenize(sourceCSS) 175 | let iteration 176 | while (!(iteration = iterator()).done) { 177 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 178 | } 179 | expect(sourceCSS).toBe(resultCSS) 180 | }) 181 | 182 | test('Bootstrap [Symbol.iterator]', () => { 183 | let sourceCSS = fs.readFileSync('./node_modules/bootstrap/dist/css/bootstrap.css', 'utf-8') 184 | let resultCSS = Array.from(tokenize(sourceCSS), token => token.lead + token.data + token.tail).join('') 185 | expect(sourceCSS).toBe(resultCSS) 186 | }) 187 | 188 | test('Tailwind CSS', () => { 189 | let sourceCSS = fs.readFileSync('./node_modules/tailwindcss/dist/tailwind.css', 'utf-8') 190 | let resultCSS = '' 191 | let iterator = tokenize(sourceCSS) 192 | let iteration 193 | while (!(iteration = iterator()).done) { 194 | resultCSS += iteration.value.lead + iteration.value.data + iteration.value.tail 195 | } 196 | expect(sourceCSS).toBe(resultCSS) 197 | }) 198 | 199 | test('Tailwind CSS [Symbol.iterator]', () => { 200 | let sourceCSS = fs.readFileSync('./node_modules/tailwindcss/dist/tailwind.css', 'utf-8') 201 | let resultCSS = Array.from(tokenize(sourceCSS), token => token.lead + token.data + token.tail).join('') 202 | expect(sourceCSS).toBe(resultCSS) 203 | }) 204 | }) 205 | -------------------------------------------------------------------------------- /src/lib/code-points.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Unicode Character Codes (0x0000 - 0x0080) 3 | * @see https://www.unicode.org/Public/UCD/latest/ucd/NamesList.txt 4 | * @see https://unicode.org/charts/nameslist/n_0000.html 5 | *//** */ 6 | 7 | /** 8 | * C0 controls 9 | * ===================================================================== *//***/ 10 | 11 | /** ␀ */ export const NULL = 0x0000 12 | /** ␁ */ export const START_OF_HEADING = 0x0001 13 | /** ␂ */ export const START_OF_TEXT = 0x0002 14 | /** ␃ */ export const END_OF_TEXT = 0x0003 15 | /** ␄ */ export const END_OF_TRANSMISSION = 0x0004 16 | /** ␅ */ export const ENQUIRY = 0x0005 17 | /** ␆ */ export const ACKNOWLEDGE = 0x0006 18 | /** ␇ */ export const BELL = 0x0007 19 | /** ␈ */ export const BACKSPACE = 0x0008 20 | /** ␉ */ export const CHARACTER_TABULATION = 0x0009 21 | /** ␊ */ export const LINE_FEED = 0x000A 22 | /** ␋ */ export const LINE_TABULATION = 0x000B 23 | /** ␌ */ export const FORM_FEED = 0x000C 24 | /** ␍ */ export const CARRIAGE_RETURN = 0x000D 25 | /** ␎ */ export const SHIFT_OUT = 0x000E 26 | /** ␏ */ export const SHIFT_IN = 0x000F 27 | /** ␐ */ export const DATA_LINK_ESCAPE = 0x0010 28 | /** ␑ */ export const DEVICE_CONTROL_ONE = 0x0011 29 | /** ␒ */ export const DEVICE_CONTROL_TWO = 0x0012 30 | /** ␓ */ export const DEVICE_CONTROL_THREE = 0x0013 31 | /** ␔ */ export const DEVICE_CONTROL_FOUR = 0x0014 32 | /** ␕ */ export const NEGATIVE_ACKNOWLEDGE = 0x0015 33 | /** ␖ */ export const SYNCHRONOUS_IDLE = 0x0016 34 | /** ␗ */ export const END_OF_TRANSMISSION_BLOCK = 0x0017 35 | /** ␘ */ export const CANCEL = 0x0018 36 | /** ␙ */ export const END_OF_MEDIUM = 0x0019 37 | /** ␚ */ export const SUBSTITUTE = 0x001A 38 | /** ␛ */ export const ESCAPE = 0x001B 39 | /** ␜ */ export const INFORMATION_SEPARATOR_FOUR = 0x001C 40 | /** ␝ */ export const INFORMATION_SEPARATOR_THREE = 0x001D 41 | /** ␞ */ export const INFORMATION_SEPARATOR_TWO = 0x001E 42 | /** ␟ */ export const INFORMATION_SEPARATOR_ONE = 0x001F 43 | 44 | /** 45 | * ASCII punctuation and symbols 46 | * ===================================================================== *//***/ 47 | 48 | /** ␠ */ export const SPACE = 0x0020 49 | /** ! */ export const EXCLAMATION_MARK = 0x0021 50 | /** " */ export const QUOTATION_MARK = 0x0022 51 | /** # */ export const NUMBER_SIGN = 0x0023 52 | /** $ */ export const DOLLAR_SIGN = 0x0024 53 | /** % */ export const PERCENT_SIGN = 0x0025 54 | /** & */ export const AMPERSAND = 0x0026 55 | /** ' */ export const APOSTROPHE = 0x0027 56 | /** ( */ export const LEFT_PARENTHESIS = 0x0028 57 | /** ) */ export const RIGHT_PARENTHESIS = 0x0029 58 | /** * */ export const ASTERISK = 0x002A 59 | /** + */ export const PLUS_SIGN = 0x002B 60 | /** , */ export const COMMA = 0x002C 61 | /** - */ export const HYPHEN_MINUS = 0x002D 62 | /** . */ export const FULL_STOP = 0x002E 63 | /** / */ export const SOLIDUS = 0x002F 64 | 65 | /* 66 | * ASCII digits 67 | * ========================================================================== */ 68 | 69 | /** 0 */ export const DIGIT_ZERO = 0x0030 70 | /** 1 */ export const DIGIT_ONE = 0x0031 71 | /** 2 */ export const DIGIT_TWO = 0x0032 72 | /** 3 */ export const DIGIT_THREE = 0x0033 73 | /** 4 */ export const DIGIT_FOUR = 0x0034 74 | /** 5 */ export const DIGIT_FIVE = 0x0035 75 | /** 6 */ export const DIGIT_SIX = 0x0036 76 | /** 7 */ export const DIGIT_SEVEN = 0x0037 77 | /** 8 */ export const DIGIT_EIGHT = 0x0038 78 | /** 9 */ export const DIGIT_NINE = 0x0039 79 | 80 | /** 81 | * ASCII punctuation and symbols 82 | * ===================================================================== *//***/ 83 | 84 | /** : */ export const COLON = 0x003A 85 | /** ; */ export const SEMICOLON = 0x003B 86 | /** < */ export const LESS_THAN_SIGN = 0x003C 87 | /** = */ export const EQUALS_SIGN = 0x003D 88 | /** > */ export const GREATER_THAN_SIGN = 0x003E 89 | /** ? */ export const QUESTION_MARK = 0x003F 90 | /** @ */ export const COMMERCIAL_AT = 0x0040 91 | 92 | /** 93 | * Uppercase Latin alphabet 94 | * ===================================================================== *//***/ 95 | 96 | /** A */ export const LATIN_CAPITAL_LETTER_A = 0x0041 97 | /** B */ export const LATIN_CAPITAL_LETTER_B = 0x0042 98 | /** C */ export const LATIN_CAPITAL_LETTER_C = 0x0043 99 | /** D */ export const LATIN_CAPITAL_LETTER_D = 0x0044 100 | /** E */ export const LATIN_CAPITAL_LETTER_E = 0x0045 101 | /** F */ export const LATIN_CAPITAL_LETTER_F = 0x0046 102 | /** G */ export const LATIN_CAPITAL_LETTER_G = 0x0047 103 | /** H */ export const LATIN_CAPITAL_LETTER_H = 0x0048 104 | /** I */ export const LATIN_CAPITAL_LETTER_I = 0x0049 105 | /** J */ export const LATIN_CAPITAL_LETTER_J = 0x004A 106 | /** K */ export const LATIN_CAPITAL_LETTER_K = 0x004B 107 | /** L */ export const LATIN_CAPITAL_LETTER_L = 0x004C 108 | /** M */ export const LATIN_CAPITAL_LETTER_M = 0x004D 109 | /** N */ export const LATIN_CAPITAL_LETTER_N = 0x004E 110 | /** O */ export const LATIN_CAPITAL_LETTER_O = 0x004F 111 | /** P */ export const LATIN_CAPITAL_LETTER_P = 0x0050 112 | /** Q */ export const LATIN_CAPITAL_LETTER_Q = 0x0051 113 | /** R */ export const LATIN_CAPITAL_LETTER_R = 0x0052 114 | /** S */ export const LATIN_CAPITAL_LETTER_S = 0x0053 115 | /** T */ export const LATIN_CAPITAL_LETTER_T = 0x0054 116 | /** U */ export const LATIN_CAPITAL_LETTER_U = 0x0055 117 | /** V */ export const LATIN_CAPITAL_LETTER_V = 0x0056 118 | /** W */ export const LATIN_CAPITAL_LETTER_W = 0x0057 119 | /** X */ export const LATIN_CAPITAL_LETTER_X = 0x0058 120 | /** Y */ export const LATIN_CAPITAL_LETTER_Y = 0x0059 121 | /** Z */ export const LATIN_CAPITAL_LETTER_Z = 0x005A 122 | 123 | /** 124 | * ASCII punctuation and symbols 125 | * ===================================================================== *//***/ 126 | 127 | /** [ */ export const LEFT_SQUARE_BRACKET = 0x005B 128 | /** \ */ export const REVERSE_SOLIDUS = 0x005C 129 | /** ] */ export const RIGHT_SQUARE_BRACKET = 0x005D 130 | /** ^ */ export const CIRCUMFLEX_ACCENT = 0x005E 131 | /** _ */ export const LOW_LINE = 0x005F 132 | /** ` */ export const GRAVE_ACCENT = 0x0060 133 | 134 | /* 135 | * Lowercase Latin alphabet 136 | * ========================================================================== */ 137 | 138 | /** a */ export const LATIN_SMALL_LETTER_A = 0x0061 139 | /** b */ export const LATIN_SMALL_LETTER_B = 0x0062 140 | /** c */ export const LATIN_SMALL_LETTER_C = 0x0063 141 | /** d */ export const LATIN_SMALL_LETTER_D = 0x0064 142 | /** e */ export const LATIN_SMALL_LETTER_E = 0x0065 143 | /** f */ export const LATIN_SMALL_LETTER_F = 0x0066 144 | /** g */ export const LATIN_SMALL_LETTER_G = 0x0067 145 | /** h */ export const LATIN_SMALL_LETTER_H = 0x0068 146 | /** i */ export const LATIN_SMALL_LETTER_I = 0x0069 147 | /** j */ export const LATIN_SMALL_LETTER_J = 0x006A 148 | /** k */ export const LATIN_SMALL_LETTER_K = 0x006B 149 | /** l */ export const LATIN_SMALL_LETTER_L = 0x006C 150 | /** m */ export const LATIN_SMALL_LETTER_M = 0x006D 151 | /** n */ export const LATIN_SMALL_LETTER_N = 0x006E 152 | /** o */ export const LATIN_SMALL_LETTER_O = 0x006F 153 | /** p */ export const LATIN_SMALL_LETTER_P = 0x0070 154 | /** q */ export const LATIN_SMALL_LETTER_Q = 0x0071 155 | /** r */ export const LATIN_SMALL_LETTER_R = 0x0072 156 | /** s */ export const LATIN_SMALL_LETTER_S = 0x0073 157 | /** t */ export const LATIN_SMALL_LETTER_T = 0x0074 158 | /** u */ export const LATIN_SMALL_LETTER_U = 0x0075 159 | /** v */ export const LATIN_SMALL_LETTER_V = 0x0076 160 | /** w */ export const LATIN_SMALL_LETTER_W = 0x0077 161 | /** x */ export const LATIN_SMALL_LETTER_X = 0x0078 162 | /** y */ export const LATIN_SMALL_LETTER_Y = 0x0079 163 | /** z */ export const LATIN_SMALL_LETTER_Z = 0x007A 164 | 165 | /** 166 | * ASCII punctuation and symbols 167 | * ===================================================================== *//***/ 168 | 169 | /** { */ export const LEFT_CURLY_BRACKET = 0x007B 170 | /** | */ export const VERTICAL_LINE = 0x007C 171 | /** } */ export const RIGHT_CURLY_BRACKET = 0x007D 172 | /** ~ */ export const TILDE = 0x007E 173 | 174 | /** 175 | * Control character 176 | * ===================================================================== *//***/ 177 | 178 | /** ␡ */ export const DELETE = 0x007F 179 | 180 | /** 181 | * Non-ASCII 182 | * ===================================================================== *//***/ 183 | 184 | /** � */ export const NON_ASCII = 0x0080 185 | 186 | /** 187 | * EOF 188 | * ===================================================================== *//***/ 189 | 190 | /** ⏏ */ export const EOF = -0x0001 191 | -------------------------------------------------------------------------------- /src/lib/consume.ts: -------------------------------------------------------------------------------- 1 | import { CSSState, CSSToken } from '../types/global/global.js' 2 | 3 | import * as cp from './code-points.js' 4 | import * as is from './is.js' 5 | import * as tt from './token-types.js' 6 | 7 | const { fromCharCode } = String 8 | 9 | /** Consumes and returns a token. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 10 | export const consume = ( 11 | /** Condition of the current tokenizer. */ 12 | state: CSSState 13 | ) => { 14 | switch (true) { 15 | /* 16 | /* https://drafts.csswg.org/css-syntax/#consume-comment */ 17 | case state.codeAt0 === cp.SOLIDUS: 18 | if (state.codeAt1 === cp.ASTERISK) return consumeCommentToken(state) 19 | break 20 | /* 21 | /* https://drafts.csswg.org/css-syntax/#whitespace-token-diagram */ 22 | case is.space(state.codeAt0): 23 | return consumeSpaceToken(state) 24 | /* 25 | /* https://drafts.csswg.org/css-syntax/#string-token-diagram */ 26 | case state.codeAt0 === cp.QUOTATION_MARK: 27 | case state.codeAt0 === cp.APOSTROPHE: 28 | // "" || '' 29 | return consumeStringToken(state) 30 | /* 31 | /* https://drafts.csswg.org/css-syntax/#hash-token-diagram */ 32 | case state.codeAt0 === cp.NUMBER_SIGN: 33 | // #W 34 | if (is.identifier(state.codeAt1)) return { 35 | tick: state.tick, 36 | type: tt.HASH, 37 | code: -1, 38 | lead: consumeAnyValue(state), 39 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 40 | tail: '', 41 | } as CSSToken 42 | // #\: 43 | if (is.validEscape(state.codeAt1, state.codeAt2)) return { 44 | tick: state.tick, 45 | type: tt.HASH, 46 | code: -1, 47 | lead: consumeAnyValue(state), 48 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 49 | tail: '', 50 | } as CSSToken 51 | break 52 | /* */ 53 | /* https://drafts.csswg.org/css-syntax/#ident-token-diagram */ 54 | case state.codeAt0 === cp.REVERSE_SOLIDUS: 55 | if (is.validEscape(state.codeAt0, state.codeAt1)) return consumeIdentifierLikeToken(state, { 56 | tick: state.tick, 57 | type: tt.WORD, 58 | code: -1, 59 | lead: '', 60 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 61 | tail: '', 62 | }) 63 | break 64 | case is.identifierStart(state.codeAt0): 65 | // W 66 | return consumeIdentifierLikeToken(state, { 67 | tick: state.tick, 68 | type: tt.WORD, 69 | code: -1, 70 | lead: '', 71 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 72 | tail: '', 73 | }) 74 | case state.codeAt0 === cp.HYPHEN_MINUS: 75 | // -W 76 | if (state.codeAt1 === cp.HYPHEN_MINUS || is.identifierStart(state.codeAt1)) return consumeIdentifierLikeToken(state, { 77 | tick: state.tick, 78 | type: tt.WORD, 79 | code: -1, 80 | lead: '', 81 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 82 | tail: '', 83 | }) 84 | // -\: 85 | if (is.validEscape(state.codeAt1, state.codeAt2)) return consumeIdentifierLikeToken(state, { 86 | tick: state.tick, 87 | type: tt.WORD, 88 | code: -1, 89 | lead: '', 90 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 91 | tail: '', 92 | }) 93 | /* */ 94 | /* https://drafts.csswg.org/css-syntax/#number-token-diagram */ 95 | // -8 96 | if (is.digit(state.codeAt1)) return { 97 | tick: state.tick, 98 | type: tt.NUMBER, 99 | code: -1, 100 | lead: '', 101 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 102 | tail: consumeNumericUnitValue(state), 103 | } as CSSToken 104 | // -.8 105 | if (state.codeAt1 === cp.FULL_STOP && is.digit(state.codeAt2)) return { 106 | tick: state.tick, 107 | type: tt.NUMBER, 108 | code: -1, 109 | lead: '', 110 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 111 | tail: consumeNumericUnitValue(state), 112 | } as CSSToken 113 | case state.codeAt0 === cp.FULL_STOP: 114 | // .8 115 | if (is.digit(state.codeAt1)) return { 116 | tick: state.tick, 117 | type: tt.NUMBER, 118 | code: -1, 119 | lead: '', 120 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 121 | tail: consumeNumericUnitValue(state), 122 | } as CSSToken 123 | break 124 | case state.codeAt0 === cp.PLUS_SIGN: 125 | // +8 126 | if (is.digit(state.codeAt1)) return { 127 | tick: state.tick, 128 | type: tt.NUMBER, 129 | code: -1, 130 | lead: '', 131 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 132 | tail: consumeNumericUnitValue(state), 133 | } as CSSToken 134 | // +.8 135 | if (state.codeAt1 === cp.FULL_STOP && is.digit(state.codeAt2)) return { 136 | tick: state.tick, 137 | type: tt.NUMBER, 138 | code: -1, 139 | lead: '', 140 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 141 | tail: consumeNumericUnitValue(state), 142 | } as CSSToken 143 | break 144 | case is.digit(state.codeAt0): 145 | // 8 146 | return { 147 | tick: state.tick, 148 | type: tt.NUMBER, 149 | code: -1, 150 | lead: '', 151 | data: consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 152 | tail: consumeNumericUnitValue(state), 153 | } as CSSToken 154 | /* */ 155 | /* https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram */ 156 | case state.codeAt0 === cp.COMMERCIAL_AT: 157 | if (state.codeAt1 === cp.HYPHEN_MINUS) { 158 | // @-- 159 | if (state.codeAt2 === cp.HYPHEN_MINUS) return { 160 | tick: state.tick, 161 | type: tt.ATWORD, 162 | code: -1, 163 | lead: consumeAnyValue(state), 164 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 165 | tail: '', 166 | } as CSSToken 167 | // @-W 168 | if (is.identifierStart(state.codeAt2)) return { 169 | tick: state.tick, 170 | type: tt.ATWORD, 171 | code: -1, 172 | lead: consumeAnyValue(state), 173 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 174 | tail: '', 175 | } as CSSToken 176 | // @-\: 177 | if (is.validEscape(state.codeAt2, state.codeAt3)) return { 178 | tick: state.tick, 179 | type: tt.ATWORD, 180 | code: -1, 181 | lead: consumeAnyValue(state), 182 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 183 | tail: '', 184 | } as CSSToken 185 | } 186 | // @W 187 | if (is.identifierStart(state.codeAt1)) return { 188 | tick: state.tick, 189 | type: tt.ATWORD, 190 | code: -1, 191 | lead: consumeAnyValue(state), 192 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 193 | tail: '', 194 | } as CSSToken 195 | // @\: 196 | if (is.validEscape(state.codeAt1, state.codeAt2)) return { 197 | tick: state.tick, 198 | type: tt.ATWORD, 199 | code: -1, 200 | lead: consumeAnyValue(state), 201 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 202 | tail: '', 203 | } as CSSToken 204 | break 205 | } 206 | /* */ 207 | /* https://drafts.csswg.org/css-syntax/#typedef-delim-token */ 208 | return { 209 | tick: state.tick, 210 | type: tt.SYMBOL, 211 | code: state.codeAt0, 212 | lead: '', 213 | data: consumeAnyValue(state), 214 | tail: '', 215 | } as CSSToken 216 | } 217 | 218 | /** Consume and return a value. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 219 | const consumeAnyValue = (state: CSSState) => { 220 | const result = fromCharCode(state.codeAt0) 221 | state.next() 222 | return result 223 | } 224 | 225 | /** Consume and return an identifier value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 226 | const consumeIdentifierValue = (state: CSSState) => { 227 | let result = '' 228 | while (true) { 229 | switch (true) { 230 | case is.validEscape(state.codeAt0, state.codeAt1): 231 | result += fromCharCode(state.codeAt0) 232 | state.next() 233 | case is.identifier(state.codeAt0): 234 | result += fromCharCode(state.codeAt0) 235 | state.next() 236 | continue 237 | } 238 | break 239 | } 240 | return result 241 | } 242 | 243 | /** Consume and return an identifier or function token. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 244 | const consumeIdentifierLikeToken = (state: CSSState, token: CSSToken) => { 245 | if (state.codeAt0 === cp.LEFT_PARENTHESIS) { 246 | token.code = 40 247 | token.type = tt.FUNCTION 248 | token.lead = token.data 249 | token.data = '(' 250 | state.next() 251 | } 252 | return token 253 | } 254 | 255 | /** Consume and return a comment token. [↗](https://drafts.csswg.org/css-syntax/#consume-comment) */ 256 | const consumeCommentToken = (state: CSSState) => { 257 | const token: CSSToken = { 258 | tick: state.tick, 259 | type: tt.COMMENT, 260 | code: -1, 261 | lead: '/*', 262 | data: '', 263 | tail: '', 264 | } 265 | state.next() 266 | state.next() 267 | while (state.tick < state.size) { 268 | // @ts-ignore 269 | if (state.codeAt0 === cp.ASTERISK && state.codeAt1 === cp.SOLIDUS) { 270 | token.tail = '*/' 271 | state.next() 272 | state.next() 273 | break 274 | } 275 | token.data += consumeAnyValue(state) 276 | } 277 | return token 278 | } 279 | 280 | /** Consumes and returns a space token. [↗](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 281 | const consumeSpaceToken = (state: CSSState) => { 282 | const token: CSSToken = { 283 | tick: state.tick, 284 | type: tt.SPACE, 285 | code: -1, 286 | lead: '', 287 | data: consumeAnyValue(state), 288 | tail: '', 289 | } 290 | while (state.tick < state.size) { 291 | if (!is.space(state.codeAt0)) break 292 | token.data += consumeAnyValue(state) 293 | } 294 | return token 295 | } 296 | 297 | /** Consumes and returns a string token. [↗](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 298 | const consumeStringToken = (state: CSSState) => { 299 | const { codeAt0 } = state 300 | const token: CSSToken = { 301 | tick: state.tick, 302 | type: tt.STRING, 303 | code: -1, 304 | lead: '', 305 | data: consumeAnyValue(state), 306 | tail: '', 307 | } 308 | while (state.tick < state.size) { 309 | switch (true) { 310 | case is.validEscape(state.codeAt0, state.codeAt1): 311 | token.data += consumeAnyValue(state) 312 | default: 313 | token.data += consumeAnyValue(state) 314 | continue 315 | case state.codeAt0 === codeAt0: 316 | token.tail = consumeAnyValue(state) 317 | } 318 | break 319 | } 320 | return token 321 | } 322 | 323 | /** Consumes and returns a number value after an additive symbol. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 324 | export const consumeNumberSansAdditiveValue = (state: CSSState) => { 325 | let result = '' 326 | result += consumeDigitValue(state) 327 | if (state.codeAt0 === cp.FULL_STOP && is.digit(state.codeAt1)) result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state) 328 | return result + consumeNumberSansDecimalValue(state) 329 | } 330 | 331 | /** Consumes and returns a number value after a decimal place. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 332 | const consumeNumberSansDecimalValue = (state: CSSState) => { 333 | let result = '' 334 | result += consumeDigitValue(state) 335 | if (state.codeAt0 === cp.LATIN_CAPITAL_LETTER_E || state.codeAt0 === cp.LATIN_SMALL_LETTER_E) { 336 | switch (true) { 337 | case (state.codeAt1 === cp.PLUS_SIGN || state.codeAt1 === cp.HYPHEN_MINUS): 338 | if (!is.digit(state.codeAt2)) break 339 | result += consumeAnyValue(state) 340 | case is.digit(state.codeAt1): 341 | result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state) 342 | } 343 | } 344 | return result 345 | } 346 | 347 | /** Consumes and returns a digit value. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 348 | const consumeDigitValue = (state: CSSState) => { 349 | let result = '' 350 | while (state.tick < state.size) { 351 | if (!is.digit(state.codeAt0)) break 352 | result += consumeAnyValue(state) 353 | } 354 | return result 355 | } 356 | 357 | /** Consumes and returns a numeric unit value. [↗](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 358 | const consumeNumericUnitValue = (state: CSSState) => ( 359 | state.codeAt0 === cp.HYPHEN_MINUS 360 | ? state.codeAt1 === cp.HYPHEN_MINUS 361 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 362 | : is.identifierStart(state.codeAt1) 363 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 364 | : is.validEscape(state.codeAt1, state.codeAt2) 365 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 366 | : '' 367 | : state.codeAt0 === cp.PERCENT_SIGN 368 | ? consumeAnyValue(state) 369 | : is.identifierStart(state.codeAt0) 370 | ? consumeAnyValue(state) + consumeIdentifierValue(state) 371 | : is.validEscape(state.codeAt0, state.codeAt1) 372 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 373 | : '' 374 | ) 375 | -------------------------------------------------------------------------------- /src/lib/consume.scss.ts: -------------------------------------------------------------------------------- 1 | import { CSSState, CSSToken } from '../types/global/global.js' 2 | 3 | import * as cp from './code-points.js' 4 | import * as is from './is.js' 5 | import * as tt from './token-types.scss.js' 6 | 7 | const { fromCharCode } = String 8 | 9 | /** Consumes and returns a token. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 10 | export const consume = ( 11 | /** Condition of the current tokenizer. */ 12 | state: CSSState 13 | ) => { 14 | switch (true) { 15 | /* 16 | /* https://drafts.csswg.org/css-syntax/#consume-comment */ 17 | case state.codeAt0 === cp.SOLIDUS: 18 | if (state.codeAt1 === cp.ASTERISK) return consumeCommentToken(state) 19 | break 20 | /* 21 | /* https://drafts.csswg.org/css-syntax/#whitespace-token-diagram */ 22 | case is.space(state.codeAt0): 23 | return consumeSpaceToken(state) 24 | /* 25 | /* https://drafts.csswg.org/css-syntax/#string-token-diagram */ 26 | case state.codeAt0 === cp.QUOTATION_MARK: 27 | case state.codeAt0 === cp.APOSTROPHE: 28 | // "" || '' 29 | return consumeStringToken(state) 30 | /* 31 | /* https://drafts.csswg.org/css-syntax/#hash-token-diagram */ 32 | case state.codeAt0 === cp.NUMBER_SIGN: 33 | // #W 34 | if (is.identifier(state.codeAt1)) return { 35 | tick: state.tick, 36 | type: tt.HASH, 37 | code: -1, 38 | lead: consumeAnyValue(state), 39 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 40 | tail: '', 41 | } as CSSToken 42 | // #\: 43 | if (is.validEscape(state.codeAt1, state.codeAt2)) return { 44 | tick: state.tick, 45 | type: tt.HASH, 46 | code: -1, 47 | lead: consumeAnyValue(state), 48 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 49 | tail: '', 50 | } as CSSToken 51 | break 52 | /* 53 | /* https://sass-lang.com/documentation/variables */ 54 | case state.codeAt0 === cp.DOLLAR_SIGN: 55 | // $W 56 | if (is.identifier(state.codeAt1)) return { 57 | tick: state.tick, 58 | type: tt.VARIABLE, 59 | code: -1, 60 | lead: consumeAnyValue(state), 61 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 62 | tail: '', 63 | } as CSSToken 64 | // $\: 65 | if (is.validEscape(state.codeAt1, state.codeAt2)) return { 66 | tick: state.tick, 67 | type: tt.VARIABLE, 68 | code: -1, 69 | lead: consumeAnyValue(state), 70 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 71 | tail: '', 72 | } as CSSToken 73 | break 74 | /* */ 75 | /* https://drafts.csswg.org/css-syntax/#ident-token-diagram */ 76 | case state.codeAt0 === cp.REVERSE_SOLIDUS: 77 | if (is.validEscape(state.codeAt0, state.codeAt1)) return consumeIdentifierLikeToken(state, { 78 | tick: state.tick, 79 | type: tt.WORD, 80 | code: -1, 81 | lead: '', 82 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 83 | tail: '', 84 | }) 85 | break 86 | case is.identifierStart(state.codeAt0): 87 | // W 88 | return consumeIdentifierLikeToken(state, { 89 | tick: state.tick, 90 | type: tt.WORD, 91 | code: -1, 92 | lead: '', 93 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 94 | tail: '', 95 | }) 96 | case state.codeAt0 === cp.HYPHEN_MINUS: 97 | // -W 98 | if (state.codeAt1 === cp.HYPHEN_MINUS || is.identifierStart(state.codeAt1)) return consumeIdentifierLikeToken(state, { 99 | tick: state.tick, 100 | type: tt.WORD, 101 | code: -1, 102 | lead: '', 103 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 104 | tail: '', 105 | }) 106 | // -\: 107 | if (is.validEscape(state.codeAt1, state.codeAt2)) return consumeIdentifierLikeToken(state, { 108 | tick: state.tick, 109 | type: tt.WORD, 110 | code: -1, 111 | lead: '', 112 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 113 | tail: '', 114 | }) 115 | /* */ 116 | /* https://drafts.csswg.org/css-syntax/#number-token-diagram */ 117 | // -8 118 | if (is.digit(state.codeAt1)) return { 119 | tick: state.tick, 120 | type: tt.NUMBER, 121 | code: -1, 122 | lead: '', 123 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 124 | tail: consumeNumericUnitValue(state), 125 | } as CSSToken 126 | // -.8 127 | if (state.codeAt1 === cp.FULL_STOP && is.digit(state.codeAt2)) return { 128 | tick: state.tick, 129 | type: tt.NUMBER, 130 | code: -1, 131 | lead: '', 132 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 133 | tail: consumeNumericUnitValue(state), 134 | } as CSSToken 135 | case state.codeAt0 === cp.FULL_STOP: 136 | // .8 137 | if (is.digit(state.codeAt1)) return { 138 | tick: state.tick, 139 | type: tt.NUMBER, 140 | code: -1, 141 | lead: '', 142 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 143 | tail: consumeNumericUnitValue(state), 144 | } as CSSToken 145 | break 146 | case state.codeAt0 === cp.PLUS_SIGN: 147 | // +8 148 | if (is.digit(state.codeAt1)) return { 149 | tick: state.tick, 150 | type: tt.NUMBER, 151 | code: -1, 152 | lead: '', 153 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 154 | tail: consumeNumericUnitValue(state), 155 | } as CSSToken 156 | // +.8 157 | if (state.codeAt1 === cp.FULL_STOP && is.digit(state.codeAt2)) return { 158 | tick: state.tick, 159 | type: tt.NUMBER, 160 | code: -1, 161 | lead: '', 162 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 163 | tail: consumeNumericUnitValue(state), 164 | } as CSSToken 165 | break 166 | case is.digit(state.codeAt0): 167 | // 8 168 | return { 169 | tick: state.tick, 170 | type: tt.NUMBER, 171 | code: -1, 172 | lead: '', 173 | data: consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 174 | tail: consumeNumericUnitValue(state), 175 | } as CSSToken 176 | /* */ 177 | /* https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram */ 178 | case state.codeAt0 === cp.COMMERCIAL_AT: 179 | if (state.codeAt1 === cp.HYPHEN_MINUS) { 180 | // @-- 181 | if (state.codeAt2 === cp.HYPHEN_MINUS) return { 182 | tick: state.tick, 183 | type: tt.ATWORD, 184 | code: -1, 185 | lead: consumeAnyValue(state), 186 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 187 | tail: '', 188 | } as CSSToken 189 | // @-W 190 | if (is.identifierStart(state.codeAt2)) return { 191 | tick: state.tick, 192 | type: tt.ATWORD, 193 | code: -1, 194 | lead: consumeAnyValue(state), 195 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 196 | tail: '', 197 | } as CSSToken 198 | // @-\: 199 | if (is.validEscape(state.codeAt2, state.codeAt3)) return { 200 | tick: state.tick, 201 | type: tt.ATWORD, 202 | code: -1, 203 | lead: consumeAnyValue(state), 204 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 205 | tail: '', 206 | } as CSSToken 207 | } 208 | // @W 209 | if (is.identifierStart(state.codeAt1)) return { 210 | tick: state.tick, 211 | type: tt.ATWORD, 212 | code: -1, 213 | lead: consumeAnyValue(state), 214 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 215 | tail: '', 216 | } as CSSToken 217 | // @\: 218 | if (is.validEscape(state.codeAt1, state.codeAt2)) return { 219 | tick: state.tick, 220 | type: tt.ATWORD, 221 | code: -1, 222 | lead: consumeAnyValue(state), 223 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 224 | tail: '', 225 | } as CSSToken 226 | break 227 | } 228 | /* */ 229 | /* https://drafts.csswg.org/css-syntax/#typedef-delim-token */ 230 | return { 231 | tick: state.tick, 232 | type: tt.SYMBOL, 233 | code: state.codeAt0, 234 | lead: '', 235 | data: consumeAnyValue(state), 236 | tail: '', 237 | } as CSSToken 238 | } 239 | 240 | /** Consume and return a value. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 241 | const consumeAnyValue = (state: CSSState) => { 242 | const result = fromCharCode(state.codeAt0) 243 | state.next() 244 | return result 245 | } 246 | 247 | /** Consume and return an identifier value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 248 | const consumeIdentifierValue = (state: CSSState) => { 249 | let result = '' 250 | while (true) { 251 | switch (true) { 252 | case is.validEscape(state.codeAt0, state.codeAt1): 253 | result += fromCharCode(state.codeAt0) 254 | state.next() 255 | case is.identifier(state.codeAt0): 256 | result += fromCharCode(state.codeAt0) 257 | state.next() 258 | continue 259 | } 260 | break 261 | } 262 | return result 263 | } 264 | 265 | /** Consume and return an identifier or function token. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 266 | const consumeIdentifierLikeToken = (state: CSSState, token: CSSToken) => { 267 | if (state.codeAt0 === cp.LEFT_PARENTHESIS) { 268 | token.code = 40 269 | token.type = tt.FUNCTION 270 | token.lead = token.data 271 | token.data = '(' 272 | state.next() 273 | } 274 | return token 275 | } 276 | 277 | /** Consume and return a comment token. [↗](https://drafts.csswg.org/css-syntax/#consume-comment) */ 278 | const consumeCommentToken = (state: CSSState) => { 279 | const token: CSSToken = { 280 | tick: state.tick, 281 | type: tt.COMMENT, 282 | code: -1, 283 | lead: '/*', 284 | data: '', 285 | tail: '', 286 | } 287 | state.next() 288 | state.next() 289 | while (state.tick < state.size) { 290 | // @ts-ignore 291 | if (state.codeAt0 === cp.ASTERISK && state.codeAt1 === cp.SOLIDUS) { 292 | token.tail = '*/' 293 | state.next() 294 | state.next() 295 | break 296 | } 297 | token.data += consumeAnyValue(state) 298 | } 299 | return token 300 | } 301 | 302 | /** Consumes and returns a space token. [↗](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 303 | const consumeSpaceToken = (state: CSSState) => { 304 | const token: CSSToken = { 305 | tick: state.tick, 306 | type: tt.SPACE, 307 | code: -1, 308 | lead: '', 309 | data: consumeAnyValue(state), 310 | tail: '', 311 | } 312 | while (state.tick < state.size) { 313 | if (!is.space(state.codeAt0)) break 314 | token.data += consumeAnyValue(state) 315 | } 316 | return token 317 | } 318 | 319 | /** Consumes and returns a string token. [↗](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 320 | const consumeStringToken = (state: CSSState) => { 321 | const { codeAt0 } = state 322 | const token: CSSToken = { 323 | tick: state.tick, 324 | type: tt.STRING, 325 | code: -1, 326 | lead: '', 327 | data: consumeAnyValue(state), 328 | tail: '', 329 | } 330 | while (state.tick < state.size) { 331 | switch (true) { 332 | case is.validEscape(state.codeAt0, state.codeAt1): 333 | token.data += consumeAnyValue(state) 334 | default: 335 | token.data += consumeAnyValue(state) 336 | continue 337 | case state.codeAt0 === codeAt0: 338 | token.tail = consumeAnyValue(state) 339 | } 340 | break 341 | } 342 | return token 343 | } 344 | 345 | /** Consumes and returns a number value after an additive symbol. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 346 | export const consumeNumberSansAdditiveValue = (state: CSSState) => { 347 | let result = '' 348 | result += consumeDigitValue(state) 349 | if (state.codeAt0 === cp.FULL_STOP && is.digit(state.codeAt1)) result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state) 350 | return result + consumeNumberSansDecimalValue(state) 351 | } 352 | 353 | /** Consumes and returns a number value after a decimal place. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 354 | const consumeNumberSansDecimalValue = (state: CSSState) => { 355 | let result = '' 356 | result += consumeDigitValue(state) 357 | if (state.codeAt0 === cp.LATIN_CAPITAL_LETTER_E || state.codeAt0 === cp.LATIN_SMALL_LETTER_E) { 358 | switch (true) { 359 | case (state.codeAt1 === cp.PLUS_SIGN || state.codeAt1 === cp.HYPHEN_MINUS): 360 | if (!is.digit(state.codeAt2)) break 361 | result += consumeAnyValue(state) 362 | case is.digit(state.codeAt1): 363 | result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state) 364 | } 365 | } 366 | return result 367 | } 368 | 369 | /** Consumes and returns a digit value. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 370 | const consumeDigitValue = (state: CSSState) => { 371 | let result = '' 372 | while (state.tick < state.size) { 373 | if (!is.digit(state.codeAt0)) break 374 | result += consumeAnyValue(state) 375 | } 376 | return result 377 | } 378 | 379 | /** Consumes and returns a numeric unit value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 380 | const consumeNumericUnitValue = (state: CSSState) => ( 381 | state.codeAt0 === cp.HYPHEN_MINUS 382 | ? state.codeAt1 === cp.HYPHEN_MINUS 383 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 384 | : is.identifierStart(state.codeAt1) 385 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 386 | : is.validEscape(state.codeAt1, state.codeAt2) 387 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 388 | : '' 389 | : is.identifierStart(state.codeAt0) 390 | ? consumeAnyValue(state) + consumeIdentifierValue(state) 391 | : is.validEscape(state.codeAt0, state.codeAt1) 392 | ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) 393 | : '' 394 | ) 395 | -------------------------------------------------------------------------------- /dist/tokenize.mjs: -------------------------------------------------------------------------------- 1 | /** 2 | * Unicode Character Codes (0x0000 - 0x0080) 3 | * @see https://www.unicode.org/Public/UCD/latest/ucd/NamesList.txt 4 | * @see https://unicode.org/charts/nameslist/n_0000.html 5 | */ /** */ 6 | 7 | /** ␉ */ 8 | const CHARACTER_TABULATION = 0x0009; 9 | /** ␊ */ 10 | const LINE_FEED = 0x000A; 11 | /** ␌ */ 12 | const FORM_FEED = 0x000C; 13 | /** ␍ */ 14 | const CARRIAGE_RETURN = 0x000D; 15 | 16 | /** 17 | * ASCII punctuation and symbols 18 | * ===================================================================== */ /***/ 19 | 20 | /** ␠ */ 21 | const SPACE$1 = 0x0020; 22 | /** " */ 23 | const QUOTATION_MARK = 0x0022; 24 | /** # */ 25 | const NUMBER_SIGN = 0x0023; 26 | /** % */ 27 | const PERCENT_SIGN = 0x0025; 28 | /** ' */ 29 | const APOSTROPHE = 0x0027; 30 | /** ( */ 31 | const LEFT_PARENTHESIS = 0x0028; 32 | /** * */ 33 | const ASTERISK = 0x002A; 34 | /** + */ 35 | const PLUS_SIGN = 0x002B; 36 | /** - */ 37 | const HYPHEN_MINUS = 0x002D; 38 | /** . */ 39 | const FULL_STOP = 0x002E; 40 | /** / */ 41 | const SOLIDUS = 0x002F; 42 | 43 | /* 44 | * ASCII digits 45 | * ========================================================================== */ 46 | 47 | /** 0 */ 48 | const DIGIT_ZERO = 0x0030; 49 | /** 9 */ 50 | const DIGIT_NINE = 0x0039; 51 | /** @ */ 52 | const COMMERCIAL_AT = 0x0040; 53 | 54 | /** 55 | * Uppercase Latin alphabet 56 | * ===================================================================== */ /***/ 57 | 58 | /** A */ 59 | const LATIN_CAPITAL_LETTER_A = 0x0041; 60 | /** E */ 61 | const LATIN_CAPITAL_LETTER_E = 0x0045; 62 | /** Z */ 63 | const LATIN_CAPITAL_LETTER_Z = 0x005A; 64 | /** \ */ 65 | const REVERSE_SOLIDUS = 0x005C; 66 | /** _ */ 67 | const LOW_LINE = 0x005F; 68 | 69 | /* 70 | * Lowercase Latin alphabet 71 | * ========================================================================== */ 72 | 73 | /** a */ 74 | const LATIN_SMALL_LETTER_A = 0x0061; 75 | /** e */ 76 | const LATIN_SMALL_LETTER_E = 0x0065; 77 | /** z */ 78 | const LATIN_SMALL_LETTER_Z = 0x007A; 79 | 80 | /** 81 | * Non-ASCII 82 | * ===================================================================== */ /***/ 83 | 84 | /** � */ 85 | const NON_ASCII = 0x0080; 86 | 87 | /** Returns whether the unicode value is a digit. [↗](https://drafts.csswg.org/css-syntax/#digit) */ 88 | const digit = code => code >= DIGIT_ZERO && code <= DIGIT_NINE; 89 | 90 | /** Returns whether the unicode value is an identifier. [↗](https://drafts.csswg.org/css-syntax/#identifier-code-point) */ 91 | const identifier = code => identifierStart(code) || code >= DIGIT_ZERO && code <= DIGIT_NINE || code === HYPHEN_MINUS; 92 | 93 | /** Returns whether the unicode value is an identifier-start. [↗](https://drafts.csswg.org/css-syntax/#identifier-start-code-point) */ 94 | const identifierStart = code => code === LOW_LINE || code >= NON_ASCII || code >= LATIN_CAPITAL_LETTER_A && code <= LATIN_CAPITAL_LETTER_Z || code >= LATIN_SMALL_LETTER_A && code <= LATIN_SMALL_LETTER_Z; 95 | 96 | /** Returns whether the unicode value is a space. [↗](https://drafts.csswg.org/css-syntax/#whitespace) */ 97 | const space = code => code === CHARACTER_TABULATION || code === LINE_FEED || code === FORM_FEED || code === CARRIAGE_RETURN || code === SPACE$1; 98 | 99 | /** Returns whether the unicode values are a valid escape. [↗](https://drafts.csswg.org/css-syntax/#starts-with-a-valid-escape) */ 100 | const validEscape = (code1of2, code2of2) => code1of2 === REVERSE_SOLIDUS && !space(code2of2); 101 | 102 | /** [``](https://drafts.csswg.org/css-syntax/#typedef-delim-token) */ 103 | const SYMBOL = 0x0001; 104 | 105 | /** [``](https://drafts.csswg.org/css-syntax/#comment-diagram) */ 106 | const COMMENT = 0x0002; 107 | 108 | /** [``](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 109 | const SPACE = 0x0003; 110 | 111 | /** [``](https://drafts.csswg.org/css-syntax/#ident-token-diagram) */ 112 | const WORD = 0x0004; 113 | 114 | /** [``](https://drafts.csswg.org/css-syntax/#function-token-diagram) */ 115 | const FUNCTION = 0x0005; 116 | 117 | /** [``](https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram) */ 118 | const ATWORD = 0x0006; 119 | 120 | /** [``](https://drafts.csswg.org/css-syntax/#hash-token-diagram) */ 121 | const HASH = 0x0007; 122 | 123 | /** [``](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 124 | const STRING = 0x0008; 125 | 126 | /** [``](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 127 | const NUMBER = 0x0009; 128 | 129 | const { 130 | fromCharCode 131 | } = String; 132 | 133 | /** Consumes and returns a token. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 134 | const consume = state => { 135 | switch (true) { 136 | /* 137 | /* https://drafts.csswg.org/css-syntax/#consume-comment */ 138 | case state.codeAt0 === SOLIDUS: 139 | if (state.codeAt1 === ASTERISK) return consumeCommentToken(state); 140 | break; 141 | /* 142 | /* https://drafts.csswg.org/css-syntax/#whitespace-token-diagram */ 143 | case space(state.codeAt0): 144 | return consumeSpaceToken(state); 145 | /* 146 | /* https://drafts.csswg.org/css-syntax/#string-token-diagram */ 147 | case state.codeAt0 === QUOTATION_MARK: 148 | case state.codeAt0 === APOSTROPHE: 149 | // "" || '' 150 | return consumeStringToken(state); 151 | /* 152 | /* https://drafts.csswg.org/css-syntax/#hash-token-diagram */ 153 | case state.codeAt0 === NUMBER_SIGN: 154 | // #W 155 | if (identifier(state.codeAt1)) return { 156 | tick: state.tick, 157 | type: HASH, 158 | code: -1, 159 | lead: consumeAnyValue(state), 160 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 161 | tail: '' 162 | }; 163 | // #\: 164 | if (validEscape(state.codeAt1, state.codeAt2)) return { 165 | tick: state.tick, 166 | type: HASH, 167 | code: -1, 168 | lead: consumeAnyValue(state), 169 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 170 | tail: '' 171 | }; 172 | break; 173 | /* */ 174 | /* https://drafts.csswg.org/css-syntax/#ident-token-diagram */ 175 | case state.codeAt0 === REVERSE_SOLIDUS: 176 | if (validEscape(state.codeAt0, state.codeAt1)) return consumeIdentifierLikeToken(state, { 177 | tick: state.tick, 178 | type: WORD, 179 | code: -1, 180 | lead: '', 181 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 182 | tail: '' 183 | }); 184 | break; 185 | case identifierStart(state.codeAt0): 186 | // W 187 | return consumeIdentifierLikeToken(state, { 188 | tick: state.tick, 189 | type: WORD, 190 | code: -1, 191 | lead: '', 192 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 193 | tail: '' 194 | }); 195 | case state.codeAt0 === HYPHEN_MINUS: 196 | // -W 197 | if (state.codeAt1 === HYPHEN_MINUS || identifierStart(state.codeAt1)) return consumeIdentifierLikeToken(state, { 198 | tick: state.tick, 199 | type: WORD, 200 | code: -1, 201 | lead: '', 202 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 203 | tail: '' 204 | }); 205 | // -\: 206 | if (validEscape(state.codeAt1, state.codeAt2)) return consumeIdentifierLikeToken(state, { 207 | tick: state.tick, 208 | type: WORD, 209 | code: -1, 210 | lead: '', 211 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 212 | tail: '' 213 | }); 214 | /* */ 215 | /* https://drafts.csswg.org/css-syntax/#number-token-diagram */ 216 | // -8 217 | if (digit(state.codeAt1)) return { 218 | tick: state.tick, 219 | type: NUMBER, 220 | code: -1, 221 | lead: '', 222 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 223 | tail: consumeNumericUnitValue(state) 224 | }; 225 | // -.8 226 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 227 | tick: state.tick, 228 | type: NUMBER, 229 | code: -1, 230 | lead: '', 231 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 232 | tail: consumeNumericUnitValue(state) 233 | }; 234 | case state.codeAt0 === FULL_STOP: 235 | // .8 236 | if (digit(state.codeAt1)) return { 237 | tick: state.tick, 238 | type: NUMBER, 239 | code: -1, 240 | lead: '', 241 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 242 | tail: consumeNumericUnitValue(state) 243 | }; 244 | break; 245 | case state.codeAt0 === PLUS_SIGN: 246 | // +8 247 | if (digit(state.codeAt1)) return { 248 | tick: state.tick, 249 | type: NUMBER, 250 | code: -1, 251 | lead: '', 252 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 253 | tail: consumeNumericUnitValue(state) 254 | }; 255 | // +.8 256 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 257 | tick: state.tick, 258 | type: NUMBER, 259 | code: -1, 260 | lead: '', 261 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 262 | tail: consumeNumericUnitValue(state) 263 | }; 264 | break; 265 | case digit(state.codeAt0): 266 | // 8 267 | return { 268 | tick: state.tick, 269 | type: NUMBER, 270 | code: -1, 271 | lead: '', 272 | data: consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 273 | tail: consumeNumericUnitValue(state) 274 | }; 275 | /* */ 276 | /* https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram */ 277 | case state.codeAt0 === COMMERCIAL_AT: 278 | if (state.codeAt1 === HYPHEN_MINUS) { 279 | // @-- 280 | if (state.codeAt2 === HYPHEN_MINUS) return { 281 | tick: state.tick, 282 | type: ATWORD, 283 | code: -1, 284 | lead: consumeAnyValue(state), 285 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 286 | tail: '' 287 | }; 288 | // @-W 289 | if (identifierStart(state.codeAt2)) return { 290 | tick: state.tick, 291 | type: ATWORD, 292 | code: -1, 293 | lead: consumeAnyValue(state), 294 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 295 | tail: '' 296 | }; 297 | // @-\: 298 | if (validEscape(state.codeAt2, state.codeAt3)) return { 299 | tick: state.tick, 300 | type: ATWORD, 301 | code: -1, 302 | lead: consumeAnyValue(state), 303 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 304 | tail: '' 305 | }; 306 | } 307 | // @W 308 | if (identifierStart(state.codeAt1)) return { 309 | tick: state.tick, 310 | type: ATWORD, 311 | code: -1, 312 | lead: consumeAnyValue(state), 313 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 314 | tail: '' 315 | }; 316 | // @\: 317 | if (validEscape(state.codeAt1, state.codeAt2)) return { 318 | tick: state.tick, 319 | type: ATWORD, 320 | code: -1, 321 | lead: consumeAnyValue(state), 322 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 323 | tail: '' 324 | }; 325 | break; 326 | } 327 | /* */ 328 | /* https://drafts.csswg.org/css-syntax/#typedef-delim-token */ 329 | return { 330 | tick: state.tick, 331 | type: SYMBOL, 332 | code: state.codeAt0, 333 | lead: '', 334 | data: consumeAnyValue(state), 335 | tail: '' 336 | }; 337 | }; 338 | 339 | /** Consume and return a value. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 340 | const consumeAnyValue = state => { 341 | const result = fromCharCode(state.codeAt0); 342 | state.next(); 343 | return result; 344 | }; 345 | 346 | /** Consume and return an identifier value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 347 | const consumeIdentifierValue = state => { 348 | let result = ''; 349 | while (true) { 350 | switch (true) { 351 | case validEscape(state.codeAt0, state.codeAt1): 352 | result += fromCharCode(state.codeAt0); 353 | state.next(); 354 | case identifier(state.codeAt0): 355 | result += fromCharCode(state.codeAt0); 356 | state.next(); 357 | continue; 358 | } 359 | break; 360 | } 361 | return result; 362 | }; 363 | 364 | /** Consume and return an identifier or function token. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 365 | const consumeIdentifierLikeToken = (state, token) => { 366 | if (state.codeAt0 === LEFT_PARENTHESIS) { 367 | token.code = 40; 368 | token.type = FUNCTION; 369 | token.lead = token.data; 370 | token.data = '('; 371 | state.next(); 372 | } 373 | return token; 374 | }; 375 | 376 | /** Consume and return a comment token. [↗](https://drafts.csswg.org/css-syntax/#consume-comment) */ 377 | const consumeCommentToken = state => { 378 | const token = { 379 | tick: state.tick, 380 | type: COMMENT, 381 | code: -1, 382 | lead: '/*', 383 | data: '', 384 | tail: '' 385 | }; 386 | state.next(); 387 | state.next(); 388 | while (state.tick < state.size) { 389 | // @ts-ignore 390 | if (state.codeAt0 === ASTERISK && state.codeAt1 === SOLIDUS) { 391 | token.tail = '*/'; 392 | state.next(); 393 | state.next(); 394 | break; 395 | } 396 | token.data += consumeAnyValue(state); 397 | } 398 | return token; 399 | }; 400 | 401 | /** Consumes and returns a space token. [↗](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 402 | const consumeSpaceToken = state => { 403 | const token = { 404 | tick: state.tick, 405 | type: SPACE, 406 | code: -1, 407 | lead: '', 408 | data: consumeAnyValue(state), 409 | tail: '' 410 | }; 411 | while (state.tick < state.size) { 412 | if (!space(state.codeAt0)) break; 413 | token.data += consumeAnyValue(state); 414 | } 415 | return token; 416 | }; 417 | 418 | /** Consumes and returns a string token. [↗](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 419 | const consumeStringToken = state => { 420 | const { 421 | codeAt0 422 | } = state; 423 | const token = { 424 | tick: state.tick, 425 | type: STRING, 426 | code: -1, 427 | lead: '', 428 | data: consumeAnyValue(state), 429 | tail: '' 430 | }; 431 | while (state.tick < state.size) { 432 | switch (true) { 433 | case validEscape(state.codeAt0, state.codeAt1): 434 | token.data += consumeAnyValue(state); 435 | default: 436 | token.data += consumeAnyValue(state); 437 | continue; 438 | case state.codeAt0 === codeAt0: 439 | token.tail = consumeAnyValue(state); 440 | } 441 | break; 442 | } 443 | return token; 444 | }; 445 | 446 | /** Consumes and returns a number value after an additive symbol. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 447 | const consumeNumberSansAdditiveValue = state => { 448 | let result = ''; 449 | result += consumeDigitValue(state); 450 | if (state.codeAt0 === FULL_STOP && digit(state.codeAt1)) result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 451 | return result + consumeNumberSansDecimalValue(state); 452 | }; 453 | 454 | /** Consumes and returns a number value after a decimal place. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 455 | const consumeNumberSansDecimalValue = state => { 456 | let result = ''; 457 | result += consumeDigitValue(state); 458 | if (state.codeAt0 === LATIN_CAPITAL_LETTER_E || state.codeAt0 === LATIN_SMALL_LETTER_E) { 459 | switch (true) { 460 | case state.codeAt1 === PLUS_SIGN || state.codeAt1 === HYPHEN_MINUS: 461 | if (!digit(state.codeAt2)) break; 462 | result += consumeAnyValue(state); 463 | case digit(state.codeAt1): 464 | result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 465 | } 466 | } 467 | return result; 468 | }; 469 | 470 | /** Consumes and returns a digit value. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 471 | const consumeDigitValue = state => { 472 | let result = ''; 473 | while (state.tick < state.size) { 474 | if (!digit(state.codeAt0)) break; 475 | result += consumeAnyValue(state); 476 | } 477 | return result; 478 | }; 479 | 480 | /** Consumes and returns a numeric unit value. [↗](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 481 | const consumeNumericUnitValue = state => state.codeAt0 === HYPHEN_MINUS ? state.codeAt1 === HYPHEN_MINUS ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : identifierStart(state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt1, state.codeAt2) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : '' : state.codeAt0 === PERCENT_SIGN ? consumeAnyValue(state) : identifierStart(state.codeAt0) ? consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt0, state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : ''; 482 | 483 | /** Returns a CSS iterator to yield tokens from the given CSS data. */ 484 | const tokenize = data => { 485 | let size = data.length; 486 | let tick = 0; 487 | 488 | /** Condition of the current tokenizer. */ 489 | let state = { 490 | data, 491 | size, 492 | tick, 493 | codeAt0: tick + 0 < size ? data.charCodeAt(tick + 0) : -1, 494 | codeAt1: tick + 1 < size ? data.charCodeAt(tick + 1) : -1, 495 | codeAt2: tick + 2 < size ? data.charCodeAt(tick + 2) : -1, 496 | codeAt3: tick + 3 < size ? data.charCodeAt(tick + 3) : -1, 497 | /** Advances the unicode characters being read from the CSS data by one position. */ 498 | next() { 499 | state.tick = ++tick; 500 | state.codeAt0 = state.codeAt1; 501 | state.codeAt1 = state.codeAt2; 502 | state.codeAt2 = state.codeAt3; 503 | state.codeAt3 = tick + 3 < size ? data.charCodeAt(tick + 3) : -1; 504 | return tick >= size; 505 | } 506 | }; 507 | 508 | /** Returns the most recent state and token yielded from the CSS iterator. */ 509 | const iterator = () => state.tick >= state.size ? { 510 | done: true, 511 | value: { 512 | tick: state.tick, 513 | type: 0, 514 | code: -2, 515 | lead: '', 516 | data: '', 517 | tail: '' 518 | } 519 | } : { 520 | done: false, 521 | value: consume(state) 522 | }; 523 | iterator[Symbol.iterator] = () => ({ 524 | next: iterator 525 | }); 526 | return iterator; 527 | }; 528 | 529 | export { tokenize }; 530 | //# sourceMappingURL=tokenize.mjs.map 531 | -------------------------------------------------------------------------------- /dist/tokenize.cjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Unicode Character Codes (0x0000 - 0x0080) 5 | * @see https://www.unicode.org/Public/UCD/latest/ucd/NamesList.txt 6 | * @see https://unicode.org/charts/nameslist/n_0000.html 7 | */ /** */ 8 | 9 | /** ␉ */ 10 | const CHARACTER_TABULATION = 0x0009; 11 | /** ␊ */ 12 | const LINE_FEED = 0x000A; 13 | /** ␌ */ 14 | const FORM_FEED = 0x000C; 15 | /** ␍ */ 16 | const CARRIAGE_RETURN = 0x000D; 17 | 18 | /** 19 | * ASCII punctuation and symbols 20 | * ===================================================================== */ /***/ 21 | 22 | /** ␠ */ 23 | const SPACE$1 = 0x0020; 24 | /** " */ 25 | const QUOTATION_MARK = 0x0022; 26 | /** # */ 27 | const NUMBER_SIGN = 0x0023; 28 | /** % */ 29 | const PERCENT_SIGN = 0x0025; 30 | /** ' */ 31 | const APOSTROPHE = 0x0027; 32 | /** ( */ 33 | const LEFT_PARENTHESIS = 0x0028; 34 | /** * */ 35 | const ASTERISK = 0x002A; 36 | /** + */ 37 | const PLUS_SIGN = 0x002B; 38 | /** - */ 39 | const HYPHEN_MINUS = 0x002D; 40 | /** . */ 41 | const FULL_STOP = 0x002E; 42 | /** / */ 43 | const SOLIDUS = 0x002F; 44 | 45 | /* 46 | * ASCII digits 47 | * ========================================================================== */ 48 | 49 | /** 0 */ 50 | const DIGIT_ZERO = 0x0030; 51 | /** 9 */ 52 | const DIGIT_NINE = 0x0039; 53 | /** @ */ 54 | const COMMERCIAL_AT = 0x0040; 55 | 56 | /** 57 | * Uppercase Latin alphabet 58 | * ===================================================================== */ /***/ 59 | 60 | /** A */ 61 | const LATIN_CAPITAL_LETTER_A = 0x0041; 62 | /** E */ 63 | const LATIN_CAPITAL_LETTER_E = 0x0045; 64 | /** Z */ 65 | const LATIN_CAPITAL_LETTER_Z = 0x005A; 66 | /** \ */ 67 | const REVERSE_SOLIDUS = 0x005C; 68 | /** _ */ 69 | const LOW_LINE = 0x005F; 70 | 71 | /* 72 | * Lowercase Latin alphabet 73 | * ========================================================================== */ 74 | 75 | /** a */ 76 | const LATIN_SMALL_LETTER_A = 0x0061; 77 | /** e */ 78 | const LATIN_SMALL_LETTER_E = 0x0065; 79 | /** z */ 80 | const LATIN_SMALL_LETTER_Z = 0x007A; 81 | 82 | /** 83 | * Non-ASCII 84 | * ===================================================================== */ /***/ 85 | 86 | /** � */ 87 | const NON_ASCII = 0x0080; 88 | 89 | /** Returns whether the unicode value is a digit. [↗](https://drafts.csswg.org/css-syntax/#digit) */ 90 | const digit = code => code >= DIGIT_ZERO && code <= DIGIT_NINE; 91 | 92 | /** Returns whether the unicode value is an identifier. [↗](https://drafts.csswg.org/css-syntax/#identifier-code-point) */ 93 | const identifier = code => identifierStart(code) || code >= DIGIT_ZERO && code <= DIGIT_NINE || code === HYPHEN_MINUS; 94 | 95 | /** Returns whether the unicode value is an identifier-start. [↗](https://drafts.csswg.org/css-syntax/#identifier-start-code-point) */ 96 | const identifierStart = code => code === LOW_LINE || code >= NON_ASCII || code >= LATIN_CAPITAL_LETTER_A && code <= LATIN_CAPITAL_LETTER_Z || code >= LATIN_SMALL_LETTER_A && code <= LATIN_SMALL_LETTER_Z; 97 | 98 | /** Returns whether the unicode value is a space. [↗](https://drafts.csswg.org/css-syntax/#whitespace) */ 99 | const space = code => code === CHARACTER_TABULATION || code === LINE_FEED || code === FORM_FEED || code === CARRIAGE_RETURN || code === SPACE$1; 100 | 101 | /** Returns whether the unicode values are a valid escape. [↗](https://drafts.csswg.org/css-syntax/#starts-with-a-valid-escape) */ 102 | const validEscape = (code1of2, code2of2) => code1of2 === REVERSE_SOLIDUS && !space(code2of2); 103 | 104 | /** [``](https://drafts.csswg.org/css-syntax/#typedef-delim-token) */ 105 | const SYMBOL = 0x0001; 106 | 107 | /** [``](https://drafts.csswg.org/css-syntax/#comment-diagram) */ 108 | const COMMENT = 0x0002; 109 | 110 | /** [``](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 111 | const SPACE = 0x0003; 112 | 113 | /** [``](https://drafts.csswg.org/css-syntax/#ident-token-diagram) */ 114 | const WORD = 0x0004; 115 | 116 | /** [``](https://drafts.csswg.org/css-syntax/#function-token-diagram) */ 117 | const FUNCTION = 0x0005; 118 | 119 | /** [``](https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram) */ 120 | const ATWORD = 0x0006; 121 | 122 | /** [``](https://drafts.csswg.org/css-syntax/#hash-token-diagram) */ 123 | const HASH = 0x0007; 124 | 125 | /** [``](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 126 | const STRING = 0x0008; 127 | 128 | /** [``](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 129 | const NUMBER = 0x0009; 130 | 131 | const { 132 | fromCharCode 133 | } = String; 134 | 135 | /** Consumes and returns a token. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 136 | const consume = state => { 137 | switch (true) { 138 | /* 139 | /* https://drafts.csswg.org/css-syntax/#consume-comment */ 140 | case state.codeAt0 === SOLIDUS: 141 | if (state.codeAt1 === ASTERISK) return consumeCommentToken(state); 142 | break; 143 | /* 144 | /* https://drafts.csswg.org/css-syntax/#whitespace-token-diagram */ 145 | case space(state.codeAt0): 146 | return consumeSpaceToken(state); 147 | /* 148 | /* https://drafts.csswg.org/css-syntax/#string-token-diagram */ 149 | case state.codeAt0 === QUOTATION_MARK: 150 | case state.codeAt0 === APOSTROPHE: 151 | // "" || '' 152 | return consumeStringToken(state); 153 | /* 154 | /* https://drafts.csswg.org/css-syntax/#hash-token-diagram */ 155 | case state.codeAt0 === NUMBER_SIGN: 156 | // #W 157 | if (identifier(state.codeAt1)) return { 158 | tick: state.tick, 159 | type: HASH, 160 | code: -1, 161 | lead: consumeAnyValue(state), 162 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 163 | tail: '' 164 | }; 165 | // #\: 166 | if (validEscape(state.codeAt1, state.codeAt2)) return { 167 | tick: state.tick, 168 | type: HASH, 169 | code: -1, 170 | lead: consumeAnyValue(state), 171 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 172 | tail: '' 173 | }; 174 | break; 175 | /* */ 176 | /* https://drafts.csswg.org/css-syntax/#ident-token-diagram */ 177 | case state.codeAt0 === REVERSE_SOLIDUS: 178 | if (validEscape(state.codeAt0, state.codeAt1)) return consumeIdentifierLikeToken(state, { 179 | tick: state.tick, 180 | type: WORD, 181 | code: -1, 182 | lead: '', 183 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 184 | tail: '' 185 | }); 186 | break; 187 | case identifierStart(state.codeAt0): 188 | // W 189 | return consumeIdentifierLikeToken(state, { 190 | tick: state.tick, 191 | type: WORD, 192 | code: -1, 193 | lead: '', 194 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 195 | tail: '' 196 | }); 197 | case state.codeAt0 === HYPHEN_MINUS: 198 | // -W 199 | if (state.codeAt1 === HYPHEN_MINUS || identifierStart(state.codeAt1)) return consumeIdentifierLikeToken(state, { 200 | tick: state.tick, 201 | type: WORD, 202 | code: -1, 203 | lead: '', 204 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 205 | tail: '' 206 | }); 207 | // -\: 208 | if (validEscape(state.codeAt1, state.codeAt2)) return consumeIdentifierLikeToken(state, { 209 | tick: state.tick, 210 | type: WORD, 211 | code: -1, 212 | lead: '', 213 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 214 | tail: '' 215 | }); 216 | /* */ 217 | /* https://drafts.csswg.org/css-syntax/#number-token-diagram */ 218 | // -8 219 | if (digit(state.codeAt1)) return { 220 | tick: state.tick, 221 | type: NUMBER, 222 | code: -1, 223 | lead: '', 224 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 225 | tail: consumeNumericUnitValue(state) 226 | }; 227 | // -.8 228 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 229 | tick: state.tick, 230 | type: NUMBER, 231 | code: -1, 232 | lead: '', 233 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 234 | tail: consumeNumericUnitValue(state) 235 | }; 236 | case state.codeAt0 === FULL_STOP: 237 | // .8 238 | if (digit(state.codeAt1)) return { 239 | tick: state.tick, 240 | type: NUMBER, 241 | code: -1, 242 | lead: '', 243 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 244 | tail: consumeNumericUnitValue(state) 245 | }; 246 | break; 247 | case state.codeAt0 === PLUS_SIGN: 248 | // +8 249 | if (digit(state.codeAt1)) return { 250 | tick: state.tick, 251 | type: NUMBER, 252 | code: -1, 253 | lead: '', 254 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 255 | tail: consumeNumericUnitValue(state) 256 | }; 257 | // +.8 258 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 259 | tick: state.tick, 260 | type: NUMBER, 261 | code: -1, 262 | lead: '', 263 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 264 | tail: consumeNumericUnitValue(state) 265 | }; 266 | break; 267 | case digit(state.codeAt0): 268 | // 8 269 | return { 270 | tick: state.tick, 271 | type: NUMBER, 272 | code: -1, 273 | lead: '', 274 | data: consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 275 | tail: consumeNumericUnitValue(state) 276 | }; 277 | /* */ 278 | /* https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram */ 279 | case state.codeAt0 === COMMERCIAL_AT: 280 | if (state.codeAt1 === HYPHEN_MINUS) { 281 | // @-- 282 | if (state.codeAt2 === HYPHEN_MINUS) return { 283 | tick: state.tick, 284 | type: ATWORD, 285 | code: -1, 286 | lead: consumeAnyValue(state), 287 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 288 | tail: '' 289 | }; 290 | // @-W 291 | if (identifierStart(state.codeAt2)) return { 292 | tick: state.tick, 293 | type: ATWORD, 294 | code: -1, 295 | lead: consumeAnyValue(state), 296 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 297 | tail: '' 298 | }; 299 | // @-\: 300 | if (validEscape(state.codeAt2, state.codeAt3)) return { 301 | tick: state.tick, 302 | type: ATWORD, 303 | code: -1, 304 | lead: consumeAnyValue(state), 305 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 306 | tail: '' 307 | }; 308 | } 309 | // @W 310 | if (identifierStart(state.codeAt1)) return { 311 | tick: state.tick, 312 | type: ATWORD, 313 | code: -1, 314 | lead: consumeAnyValue(state), 315 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 316 | tail: '' 317 | }; 318 | // @\: 319 | if (validEscape(state.codeAt1, state.codeAt2)) return { 320 | tick: state.tick, 321 | type: ATWORD, 322 | code: -1, 323 | lead: consumeAnyValue(state), 324 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 325 | tail: '' 326 | }; 327 | break; 328 | } 329 | /* */ 330 | /* https://drafts.csswg.org/css-syntax/#typedef-delim-token */ 331 | return { 332 | tick: state.tick, 333 | type: SYMBOL, 334 | code: state.codeAt0, 335 | lead: '', 336 | data: consumeAnyValue(state), 337 | tail: '' 338 | }; 339 | }; 340 | 341 | /** Consume and return a value. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 342 | const consumeAnyValue = state => { 343 | const result = fromCharCode(state.codeAt0); 344 | state.next(); 345 | return result; 346 | }; 347 | 348 | /** Consume and return an identifier value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 349 | const consumeIdentifierValue = state => { 350 | let result = ''; 351 | while (true) { 352 | switch (true) { 353 | case validEscape(state.codeAt0, state.codeAt1): 354 | result += fromCharCode(state.codeAt0); 355 | state.next(); 356 | case identifier(state.codeAt0): 357 | result += fromCharCode(state.codeAt0); 358 | state.next(); 359 | continue; 360 | } 361 | break; 362 | } 363 | return result; 364 | }; 365 | 366 | /** Consume and return an identifier or function token. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 367 | const consumeIdentifierLikeToken = (state, token) => { 368 | if (state.codeAt0 === LEFT_PARENTHESIS) { 369 | token.code = 40; 370 | token.type = FUNCTION; 371 | token.lead = token.data; 372 | token.data = '('; 373 | state.next(); 374 | } 375 | return token; 376 | }; 377 | 378 | /** Consume and return a comment token. [↗](https://drafts.csswg.org/css-syntax/#consume-comment) */ 379 | const consumeCommentToken = state => { 380 | const token = { 381 | tick: state.tick, 382 | type: COMMENT, 383 | code: -1, 384 | lead: '/*', 385 | data: '', 386 | tail: '' 387 | }; 388 | state.next(); 389 | state.next(); 390 | while (state.tick < state.size) { 391 | // @ts-ignore 392 | if (state.codeAt0 === ASTERISK && state.codeAt1 === SOLIDUS) { 393 | token.tail = '*/'; 394 | state.next(); 395 | state.next(); 396 | break; 397 | } 398 | token.data += consumeAnyValue(state); 399 | } 400 | return token; 401 | }; 402 | 403 | /** Consumes and returns a space token. [↗](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 404 | const consumeSpaceToken = state => { 405 | const token = { 406 | tick: state.tick, 407 | type: SPACE, 408 | code: -1, 409 | lead: '', 410 | data: consumeAnyValue(state), 411 | tail: '' 412 | }; 413 | while (state.tick < state.size) { 414 | if (!space(state.codeAt0)) break; 415 | token.data += consumeAnyValue(state); 416 | } 417 | return token; 418 | }; 419 | 420 | /** Consumes and returns a string token. [↗](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 421 | const consumeStringToken = state => { 422 | const { 423 | codeAt0 424 | } = state; 425 | const token = { 426 | tick: state.tick, 427 | type: STRING, 428 | code: -1, 429 | lead: '', 430 | data: consumeAnyValue(state), 431 | tail: '' 432 | }; 433 | while (state.tick < state.size) { 434 | switch (true) { 435 | case validEscape(state.codeAt0, state.codeAt1): 436 | token.data += consumeAnyValue(state); 437 | default: 438 | token.data += consumeAnyValue(state); 439 | continue; 440 | case state.codeAt0 === codeAt0: 441 | token.tail = consumeAnyValue(state); 442 | } 443 | break; 444 | } 445 | return token; 446 | }; 447 | 448 | /** Consumes and returns a number value after an additive symbol. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 449 | const consumeNumberSansAdditiveValue = state => { 450 | let result = ''; 451 | result += consumeDigitValue(state); 452 | if (state.codeAt0 === FULL_STOP && digit(state.codeAt1)) result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 453 | return result + consumeNumberSansDecimalValue(state); 454 | }; 455 | 456 | /** Consumes and returns a number value after a decimal place. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 457 | const consumeNumberSansDecimalValue = state => { 458 | let result = ''; 459 | result += consumeDigitValue(state); 460 | if (state.codeAt0 === LATIN_CAPITAL_LETTER_E || state.codeAt0 === LATIN_SMALL_LETTER_E) { 461 | switch (true) { 462 | case state.codeAt1 === PLUS_SIGN || state.codeAt1 === HYPHEN_MINUS: 463 | if (!digit(state.codeAt2)) break; 464 | result += consumeAnyValue(state); 465 | case digit(state.codeAt1): 466 | result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 467 | } 468 | } 469 | return result; 470 | }; 471 | 472 | /** Consumes and returns a digit value. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 473 | const consumeDigitValue = state => { 474 | let result = ''; 475 | while (state.tick < state.size) { 476 | if (!digit(state.codeAt0)) break; 477 | result += consumeAnyValue(state); 478 | } 479 | return result; 480 | }; 481 | 482 | /** Consumes and returns a numeric unit value. [↗](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 483 | const consumeNumericUnitValue = state => state.codeAt0 === HYPHEN_MINUS ? state.codeAt1 === HYPHEN_MINUS ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : identifierStart(state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt1, state.codeAt2) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : '' : state.codeAt0 === PERCENT_SIGN ? consumeAnyValue(state) : identifierStart(state.codeAt0) ? consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt0, state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : ''; 484 | 485 | /** Returns a CSS iterator to yield tokens from the given CSS data. */ 486 | const tokenize = data => { 487 | let size = data.length; 488 | let tick = 0; 489 | 490 | /** Condition of the current tokenizer. */ 491 | let state = { 492 | data, 493 | size, 494 | tick, 495 | codeAt0: tick + 0 < size ? data.charCodeAt(tick + 0) : -1, 496 | codeAt1: tick + 1 < size ? data.charCodeAt(tick + 1) : -1, 497 | codeAt2: tick + 2 < size ? data.charCodeAt(tick + 2) : -1, 498 | codeAt3: tick + 3 < size ? data.charCodeAt(tick + 3) : -1, 499 | /** Advances the unicode characters being read from the CSS data by one position. */ 500 | next() { 501 | state.tick = ++tick; 502 | state.codeAt0 = state.codeAt1; 503 | state.codeAt1 = state.codeAt2; 504 | state.codeAt2 = state.codeAt3; 505 | state.codeAt3 = tick + 3 < size ? data.charCodeAt(tick + 3) : -1; 506 | return tick >= size; 507 | } 508 | }; 509 | 510 | /** Returns the most recent state and token yielded from the CSS iterator. */ 511 | const iterator = () => state.tick >= state.size ? { 512 | done: true, 513 | value: { 514 | tick: state.tick, 515 | type: 0, 516 | code: -2, 517 | lead: '', 518 | data: '', 519 | tail: '' 520 | } 521 | } : { 522 | done: false, 523 | value: consume(state) 524 | }; 525 | iterator[Symbol.iterator] = () => ({ 526 | next: iterator 527 | }); 528 | return iterator; 529 | }; 530 | 531 | exports.tokenize = tokenize; 532 | //# sourceMappingURL=tokenize.cjs.map 533 | -------------------------------------------------------------------------------- /dist/tokenizeSCSS.mjs: -------------------------------------------------------------------------------- 1 | /** 2 | * Unicode Character Codes (0x0000 - 0x0080) 3 | * @see https://www.unicode.org/Public/UCD/latest/ucd/NamesList.txt 4 | * @see https://unicode.org/charts/nameslist/n_0000.html 5 | */ /** */ 6 | 7 | /** ␉ */ 8 | const CHARACTER_TABULATION = 0x0009; 9 | /** ␊ */ 10 | const LINE_FEED = 0x000A; 11 | /** ␌ */ 12 | const FORM_FEED = 0x000C; 13 | /** ␍ */ 14 | const CARRIAGE_RETURN = 0x000D; 15 | 16 | /** 17 | * ASCII punctuation and symbols 18 | * ===================================================================== */ /***/ 19 | 20 | /** ␠ */ 21 | const SPACE$1 = 0x0020; 22 | /** " */ 23 | const QUOTATION_MARK = 0x0022; 24 | /** # */ 25 | const NUMBER_SIGN = 0x0023; 26 | /** $ */ 27 | const DOLLAR_SIGN = 0x0024; 28 | /** ' */ 29 | const APOSTROPHE = 0x0027; 30 | /** ( */ 31 | const LEFT_PARENTHESIS = 0x0028; 32 | /** * */ 33 | const ASTERISK = 0x002A; 34 | /** + */ 35 | const PLUS_SIGN = 0x002B; 36 | /** - */ 37 | const HYPHEN_MINUS = 0x002D; 38 | /** . */ 39 | const FULL_STOP = 0x002E; 40 | /** / */ 41 | const SOLIDUS = 0x002F; 42 | 43 | /* 44 | * ASCII digits 45 | * ========================================================================== */ 46 | 47 | /** 0 */ 48 | const DIGIT_ZERO = 0x0030; 49 | /** 9 */ 50 | const DIGIT_NINE = 0x0039; 51 | /** @ */ 52 | const COMMERCIAL_AT = 0x0040; 53 | 54 | /** 55 | * Uppercase Latin alphabet 56 | * ===================================================================== */ /***/ 57 | 58 | /** A */ 59 | const LATIN_CAPITAL_LETTER_A = 0x0041; 60 | /** E */ 61 | const LATIN_CAPITAL_LETTER_E = 0x0045; 62 | /** Z */ 63 | const LATIN_CAPITAL_LETTER_Z = 0x005A; 64 | /** \ */ 65 | const REVERSE_SOLIDUS = 0x005C; 66 | /** _ */ 67 | const LOW_LINE = 0x005F; 68 | 69 | /* 70 | * Lowercase Latin alphabet 71 | * ========================================================================== */ 72 | 73 | /** a */ 74 | const LATIN_SMALL_LETTER_A = 0x0061; 75 | /** e */ 76 | const LATIN_SMALL_LETTER_E = 0x0065; 77 | /** z */ 78 | const LATIN_SMALL_LETTER_Z = 0x007A; 79 | 80 | /** 81 | * Non-ASCII 82 | * ===================================================================== */ /***/ 83 | 84 | /** � */ 85 | const NON_ASCII = 0x0080; 86 | 87 | /** Returns whether the unicode value is a digit. [↗](https://drafts.csswg.org/css-syntax/#digit) */ 88 | const digit = code => code >= DIGIT_ZERO && code <= DIGIT_NINE; 89 | 90 | /** Returns whether the unicode value is an identifier. [↗](https://drafts.csswg.org/css-syntax/#identifier-code-point) */ 91 | const identifier = code => identifierStart(code) || code >= DIGIT_ZERO && code <= DIGIT_NINE || code === HYPHEN_MINUS; 92 | 93 | /** Returns whether the unicode value is an identifier-start. [↗](https://drafts.csswg.org/css-syntax/#identifier-start-code-point) */ 94 | const identifierStart = code => code === LOW_LINE || code >= NON_ASCII || code >= LATIN_CAPITAL_LETTER_A && code <= LATIN_CAPITAL_LETTER_Z || code >= LATIN_SMALL_LETTER_A && code <= LATIN_SMALL_LETTER_Z; 95 | 96 | /** Returns whether the unicode value is a space. [↗](https://drafts.csswg.org/css-syntax/#whitespace) */ 97 | const space = code => code === CHARACTER_TABULATION || code === LINE_FEED || code === FORM_FEED || code === CARRIAGE_RETURN || code === SPACE$1; 98 | 99 | /** Returns whether the unicode values are a valid escape. [↗](https://drafts.csswg.org/css-syntax/#starts-with-a-valid-escape) */ 100 | const validEscape = (code1of2, code2of2) => code1of2 === REVERSE_SOLIDUS && !space(code2of2); 101 | 102 | /** [``](https://drafts.csswg.org/css-syntax/#typedef-delim-token) */ 103 | const SYMBOL = 0x0001; 104 | 105 | /** [``](https://drafts.csswg.org/css-syntax/#comment-diagram) */ 106 | const COMMENT = 0x0002; 107 | 108 | /** [``](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 109 | const SPACE = 0x0003; 110 | 111 | /** [``](https://drafts.csswg.org/css-syntax/#ident-token-diagram) */ 112 | const WORD = 0x0004; 113 | 114 | /** [``](https://drafts.csswg.org/css-syntax/#function-token-diagram) */ 115 | const FUNCTION = 0x0005; 116 | 117 | /** [``](https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram) */ 118 | const ATWORD = 0x0006; 119 | 120 | /** [``](https://drafts.csswg.org/css-syntax/#hash-token-diagram) */ 121 | const HASH = 0x0007; 122 | 123 | /** [``](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 124 | const STRING = 0x0008; 125 | 126 | /** [``](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 127 | const NUMBER = 0x0009; 128 | 129 | /** [``](https://sass-lang.com/documentation/variables) */ 130 | const VARIABLE = 0x0010; 131 | 132 | const { 133 | fromCharCode 134 | } = String; 135 | 136 | /** Consumes and returns a token. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 137 | const consume = state => { 138 | switch (true) { 139 | /* 140 | /* https://drafts.csswg.org/css-syntax/#consume-comment */ 141 | case state.codeAt0 === SOLIDUS: 142 | if (state.codeAt1 === ASTERISK) return consumeCommentToken(state); 143 | break; 144 | /* 145 | /* https://drafts.csswg.org/css-syntax/#whitespace-token-diagram */ 146 | case space(state.codeAt0): 147 | return consumeSpaceToken(state); 148 | /* 149 | /* https://drafts.csswg.org/css-syntax/#string-token-diagram */ 150 | case state.codeAt0 === QUOTATION_MARK: 151 | case state.codeAt0 === APOSTROPHE: 152 | // "" || '' 153 | return consumeStringToken(state); 154 | /* 155 | /* https://drafts.csswg.org/css-syntax/#hash-token-diagram */ 156 | case state.codeAt0 === NUMBER_SIGN: 157 | // #W 158 | if (identifier(state.codeAt1)) return { 159 | tick: state.tick, 160 | type: HASH, 161 | code: -1, 162 | lead: consumeAnyValue(state), 163 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 164 | tail: '' 165 | }; 166 | // #\: 167 | if (validEscape(state.codeAt1, state.codeAt2)) return { 168 | tick: state.tick, 169 | type: HASH, 170 | code: -1, 171 | lead: consumeAnyValue(state), 172 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 173 | tail: '' 174 | }; 175 | break; 176 | /* 177 | /* https://sass-lang.com/documentation/variables */ 178 | case state.codeAt0 === DOLLAR_SIGN: 179 | // $W 180 | if (identifier(state.codeAt1)) return { 181 | tick: state.tick, 182 | type: VARIABLE, 183 | code: -1, 184 | lead: consumeAnyValue(state), 185 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 186 | tail: '' 187 | }; 188 | // $\: 189 | if (validEscape(state.codeAt1, state.codeAt2)) return { 190 | tick: state.tick, 191 | type: VARIABLE, 192 | code: -1, 193 | lead: consumeAnyValue(state), 194 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 195 | tail: '' 196 | }; 197 | break; 198 | /* */ 199 | /* https://drafts.csswg.org/css-syntax/#ident-token-diagram */ 200 | case state.codeAt0 === REVERSE_SOLIDUS: 201 | if (validEscape(state.codeAt0, state.codeAt1)) return consumeIdentifierLikeToken(state, { 202 | tick: state.tick, 203 | type: WORD, 204 | code: -1, 205 | lead: '', 206 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 207 | tail: '' 208 | }); 209 | break; 210 | case identifierStart(state.codeAt0): 211 | // W 212 | return consumeIdentifierLikeToken(state, { 213 | tick: state.tick, 214 | type: WORD, 215 | code: -1, 216 | lead: '', 217 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 218 | tail: '' 219 | }); 220 | case state.codeAt0 === HYPHEN_MINUS: 221 | // -W 222 | if (state.codeAt1 === HYPHEN_MINUS || identifierStart(state.codeAt1)) return consumeIdentifierLikeToken(state, { 223 | tick: state.tick, 224 | type: WORD, 225 | code: -1, 226 | lead: '', 227 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 228 | tail: '' 229 | }); 230 | // -\: 231 | if (validEscape(state.codeAt1, state.codeAt2)) return consumeIdentifierLikeToken(state, { 232 | tick: state.tick, 233 | type: WORD, 234 | code: -1, 235 | lead: '', 236 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 237 | tail: '' 238 | }); 239 | /* */ 240 | /* https://drafts.csswg.org/css-syntax/#number-token-diagram */ 241 | // -8 242 | if (digit(state.codeAt1)) return { 243 | tick: state.tick, 244 | type: NUMBER, 245 | code: -1, 246 | lead: '', 247 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 248 | tail: consumeNumericUnitValue(state) 249 | }; 250 | // -.8 251 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 252 | tick: state.tick, 253 | type: NUMBER, 254 | code: -1, 255 | lead: '', 256 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 257 | tail: consumeNumericUnitValue(state) 258 | }; 259 | case state.codeAt0 === FULL_STOP: 260 | // .8 261 | if (digit(state.codeAt1)) return { 262 | tick: state.tick, 263 | type: NUMBER, 264 | code: -1, 265 | lead: '', 266 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 267 | tail: consumeNumericUnitValue(state) 268 | }; 269 | break; 270 | case state.codeAt0 === PLUS_SIGN: 271 | // +8 272 | if (digit(state.codeAt1)) return { 273 | tick: state.tick, 274 | type: NUMBER, 275 | code: -1, 276 | lead: '', 277 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 278 | tail: consumeNumericUnitValue(state) 279 | }; 280 | // +.8 281 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 282 | tick: state.tick, 283 | type: NUMBER, 284 | code: -1, 285 | lead: '', 286 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 287 | tail: consumeNumericUnitValue(state) 288 | }; 289 | break; 290 | case digit(state.codeAt0): 291 | // 8 292 | return { 293 | tick: state.tick, 294 | type: NUMBER, 295 | code: -1, 296 | lead: '', 297 | data: consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 298 | tail: consumeNumericUnitValue(state) 299 | }; 300 | /* */ 301 | /* https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram */ 302 | case state.codeAt0 === COMMERCIAL_AT: 303 | if (state.codeAt1 === HYPHEN_MINUS) { 304 | // @-- 305 | if (state.codeAt2 === HYPHEN_MINUS) return { 306 | tick: state.tick, 307 | type: ATWORD, 308 | code: -1, 309 | lead: consumeAnyValue(state), 310 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 311 | tail: '' 312 | }; 313 | // @-W 314 | if (identifierStart(state.codeAt2)) return { 315 | tick: state.tick, 316 | type: ATWORD, 317 | code: -1, 318 | lead: consumeAnyValue(state), 319 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 320 | tail: '' 321 | }; 322 | // @-\: 323 | if (validEscape(state.codeAt2, state.codeAt3)) return { 324 | tick: state.tick, 325 | type: ATWORD, 326 | code: -1, 327 | lead: consumeAnyValue(state), 328 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 329 | tail: '' 330 | }; 331 | } 332 | // @W 333 | if (identifierStart(state.codeAt1)) return { 334 | tick: state.tick, 335 | type: ATWORD, 336 | code: -1, 337 | lead: consumeAnyValue(state), 338 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 339 | tail: '' 340 | }; 341 | // @\: 342 | if (validEscape(state.codeAt1, state.codeAt2)) return { 343 | tick: state.tick, 344 | type: ATWORD, 345 | code: -1, 346 | lead: consumeAnyValue(state), 347 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 348 | tail: '' 349 | }; 350 | break; 351 | } 352 | /* */ 353 | /* https://drafts.csswg.org/css-syntax/#typedef-delim-token */ 354 | return { 355 | tick: state.tick, 356 | type: SYMBOL, 357 | code: state.codeAt0, 358 | lead: '', 359 | data: consumeAnyValue(state), 360 | tail: '' 361 | }; 362 | }; 363 | 364 | /** Consume and return a value. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 365 | const consumeAnyValue = state => { 366 | const result = fromCharCode(state.codeAt0); 367 | state.next(); 368 | return result; 369 | }; 370 | 371 | /** Consume and return an identifier value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 372 | const consumeIdentifierValue = state => { 373 | let result = ''; 374 | while (true) { 375 | switch (true) { 376 | case validEscape(state.codeAt0, state.codeAt1): 377 | result += fromCharCode(state.codeAt0); 378 | state.next(); 379 | case identifier(state.codeAt0): 380 | result += fromCharCode(state.codeAt0); 381 | state.next(); 382 | continue; 383 | } 384 | break; 385 | } 386 | return result; 387 | }; 388 | 389 | /** Consume and return an identifier or function token. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 390 | const consumeIdentifierLikeToken = (state, token) => { 391 | if (state.codeAt0 === LEFT_PARENTHESIS) { 392 | token.code = 40; 393 | token.type = FUNCTION; 394 | token.lead = token.data; 395 | token.data = '('; 396 | state.next(); 397 | } 398 | return token; 399 | }; 400 | 401 | /** Consume and return a comment token. [↗](https://drafts.csswg.org/css-syntax/#consume-comment) */ 402 | const consumeCommentToken = state => { 403 | const token = { 404 | tick: state.tick, 405 | type: COMMENT, 406 | code: -1, 407 | lead: '/*', 408 | data: '', 409 | tail: '' 410 | }; 411 | state.next(); 412 | state.next(); 413 | while (state.tick < state.size) { 414 | // @ts-ignore 415 | if (state.codeAt0 === ASTERISK && state.codeAt1 === SOLIDUS) { 416 | token.tail = '*/'; 417 | state.next(); 418 | state.next(); 419 | break; 420 | } 421 | token.data += consumeAnyValue(state); 422 | } 423 | return token; 424 | }; 425 | 426 | /** Consumes and returns a space token. [↗](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 427 | const consumeSpaceToken = state => { 428 | const token = { 429 | tick: state.tick, 430 | type: SPACE, 431 | code: -1, 432 | lead: '', 433 | data: consumeAnyValue(state), 434 | tail: '' 435 | }; 436 | while (state.tick < state.size) { 437 | if (!space(state.codeAt0)) break; 438 | token.data += consumeAnyValue(state); 439 | } 440 | return token; 441 | }; 442 | 443 | /** Consumes and returns a string token. [↗](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 444 | const consumeStringToken = state => { 445 | const { 446 | codeAt0 447 | } = state; 448 | const token = { 449 | tick: state.tick, 450 | type: STRING, 451 | code: -1, 452 | lead: '', 453 | data: consumeAnyValue(state), 454 | tail: '' 455 | }; 456 | while (state.tick < state.size) { 457 | switch (true) { 458 | case validEscape(state.codeAt0, state.codeAt1): 459 | token.data += consumeAnyValue(state); 460 | default: 461 | token.data += consumeAnyValue(state); 462 | continue; 463 | case state.codeAt0 === codeAt0: 464 | token.tail = consumeAnyValue(state); 465 | } 466 | break; 467 | } 468 | return token; 469 | }; 470 | 471 | /** Consumes and returns a number value after an additive symbol. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 472 | const consumeNumberSansAdditiveValue = state => { 473 | let result = ''; 474 | result += consumeDigitValue(state); 475 | if (state.codeAt0 === FULL_STOP && digit(state.codeAt1)) result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 476 | return result + consumeNumberSansDecimalValue(state); 477 | }; 478 | 479 | /** Consumes and returns a number value after a decimal place. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 480 | const consumeNumberSansDecimalValue = state => { 481 | let result = ''; 482 | result += consumeDigitValue(state); 483 | if (state.codeAt0 === LATIN_CAPITAL_LETTER_E || state.codeAt0 === LATIN_SMALL_LETTER_E) { 484 | switch (true) { 485 | case state.codeAt1 === PLUS_SIGN || state.codeAt1 === HYPHEN_MINUS: 486 | if (!digit(state.codeAt2)) break; 487 | result += consumeAnyValue(state); 488 | case digit(state.codeAt1): 489 | result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 490 | } 491 | } 492 | return result; 493 | }; 494 | 495 | /** Consumes and returns a digit value. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 496 | const consumeDigitValue = state => { 497 | let result = ''; 498 | while (state.tick < state.size) { 499 | if (!digit(state.codeAt0)) break; 500 | result += consumeAnyValue(state); 501 | } 502 | return result; 503 | }; 504 | 505 | /** Consumes and returns a numeric unit value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 506 | const consumeNumericUnitValue = state => state.codeAt0 === HYPHEN_MINUS ? state.codeAt1 === HYPHEN_MINUS ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : identifierStart(state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt1, state.codeAt2) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : '' : identifierStart(state.codeAt0) ? consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt0, state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : ''; 507 | 508 | /** Returns a CSS iterator to yield tokens from the given CSS data. */ 509 | const tokenize = data => { 510 | let size = data.length; 511 | let tick = 0; 512 | 513 | /** Condition of the current tokenizer. */ 514 | let state = { 515 | data, 516 | size, 517 | tick, 518 | codeAt0: tick + 0 < size ? data.charCodeAt(tick + 0) : -1, 519 | codeAt1: tick + 1 < size ? data.charCodeAt(tick + 1) : -1, 520 | codeAt2: tick + 2 < size ? data.charCodeAt(tick + 2) : -1, 521 | codeAt3: tick + 3 < size ? data.charCodeAt(tick + 3) : -1, 522 | /** Advances the unicode characters being read from the CSS data by one position. */ 523 | next() { 524 | state.tick = ++tick; 525 | state.codeAt0 = state.codeAt1; 526 | state.codeAt1 = state.codeAt2; 527 | state.codeAt2 = state.codeAt3; 528 | state.codeAt3 = tick + 3 < size ? data.charCodeAt(tick + 3) : -1; 529 | return tick >= size; 530 | } 531 | }; 532 | 533 | /** Returns the most recent state and token yielded from the CSS iterator. */ 534 | const iterator = () => state.tick >= state.size ? { 535 | done: true, 536 | value: { 537 | tick: state.tick, 538 | type: 0, 539 | code: -2, 540 | lead: '', 541 | data: '', 542 | tail: '' 543 | } 544 | } : { 545 | done: false, 546 | value: consume(state) 547 | }; 548 | iterator[Symbol.iterator] = () => ({ 549 | next: iterator 550 | }); 551 | return iterator; 552 | }; 553 | 554 | export { tokenize }; 555 | //# sourceMappingURL=tokenizeSCSS.mjs.map 556 | -------------------------------------------------------------------------------- /dist/tokenizeSCSS.cjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Unicode Character Codes (0x0000 - 0x0080) 5 | * @see https://www.unicode.org/Public/UCD/latest/ucd/NamesList.txt 6 | * @see https://unicode.org/charts/nameslist/n_0000.html 7 | */ /** */ 8 | 9 | /** ␉ */ 10 | const CHARACTER_TABULATION = 0x0009; 11 | /** ␊ */ 12 | const LINE_FEED = 0x000A; 13 | /** ␌ */ 14 | const FORM_FEED = 0x000C; 15 | /** ␍ */ 16 | const CARRIAGE_RETURN = 0x000D; 17 | 18 | /** 19 | * ASCII punctuation and symbols 20 | * ===================================================================== */ /***/ 21 | 22 | /** ␠ */ 23 | const SPACE$1 = 0x0020; 24 | /** " */ 25 | const QUOTATION_MARK = 0x0022; 26 | /** # */ 27 | const NUMBER_SIGN = 0x0023; 28 | /** $ */ 29 | const DOLLAR_SIGN = 0x0024; 30 | /** ' */ 31 | const APOSTROPHE = 0x0027; 32 | /** ( */ 33 | const LEFT_PARENTHESIS = 0x0028; 34 | /** * */ 35 | const ASTERISK = 0x002A; 36 | /** + */ 37 | const PLUS_SIGN = 0x002B; 38 | /** - */ 39 | const HYPHEN_MINUS = 0x002D; 40 | /** . */ 41 | const FULL_STOP = 0x002E; 42 | /** / */ 43 | const SOLIDUS = 0x002F; 44 | 45 | /* 46 | * ASCII digits 47 | * ========================================================================== */ 48 | 49 | /** 0 */ 50 | const DIGIT_ZERO = 0x0030; 51 | /** 9 */ 52 | const DIGIT_NINE = 0x0039; 53 | /** @ */ 54 | const COMMERCIAL_AT = 0x0040; 55 | 56 | /** 57 | * Uppercase Latin alphabet 58 | * ===================================================================== */ /***/ 59 | 60 | /** A */ 61 | const LATIN_CAPITAL_LETTER_A = 0x0041; 62 | /** E */ 63 | const LATIN_CAPITAL_LETTER_E = 0x0045; 64 | /** Z */ 65 | const LATIN_CAPITAL_LETTER_Z = 0x005A; 66 | /** \ */ 67 | const REVERSE_SOLIDUS = 0x005C; 68 | /** _ */ 69 | const LOW_LINE = 0x005F; 70 | 71 | /* 72 | * Lowercase Latin alphabet 73 | * ========================================================================== */ 74 | 75 | /** a */ 76 | const LATIN_SMALL_LETTER_A = 0x0061; 77 | /** e */ 78 | const LATIN_SMALL_LETTER_E = 0x0065; 79 | /** z */ 80 | const LATIN_SMALL_LETTER_Z = 0x007A; 81 | 82 | /** 83 | * Non-ASCII 84 | * ===================================================================== */ /***/ 85 | 86 | /** � */ 87 | const NON_ASCII = 0x0080; 88 | 89 | /** Returns whether the unicode value is a digit. [↗](https://drafts.csswg.org/css-syntax/#digit) */ 90 | const digit = code => code >= DIGIT_ZERO && code <= DIGIT_NINE; 91 | 92 | /** Returns whether the unicode value is an identifier. [↗](https://drafts.csswg.org/css-syntax/#identifier-code-point) */ 93 | const identifier = code => identifierStart(code) || code >= DIGIT_ZERO && code <= DIGIT_NINE || code === HYPHEN_MINUS; 94 | 95 | /** Returns whether the unicode value is an identifier-start. [↗](https://drafts.csswg.org/css-syntax/#identifier-start-code-point) */ 96 | const identifierStart = code => code === LOW_LINE || code >= NON_ASCII || code >= LATIN_CAPITAL_LETTER_A && code <= LATIN_CAPITAL_LETTER_Z || code >= LATIN_SMALL_LETTER_A && code <= LATIN_SMALL_LETTER_Z; 97 | 98 | /** Returns whether the unicode value is a space. [↗](https://drafts.csswg.org/css-syntax/#whitespace) */ 99 | const space = code => code === CHARACTER_TABULATION || code === LINE_FEED || code === FORM_FEED || code === CARRIAGE_RETURN || code === SPACE$1; 100 | 101 | /** Returns whether the unicode values are a valid escape. [↗](https://drafts.csswg.org/css-syntax/#starts-with-a-valid-escape) */ 102 | const validEscape = (code1of2, code2of2) => code1of2 === REVERSE_SOLIDUS && !space(code2of2); 103 | 104 | /** [``](https://drafts.csswg.org/css-syntax/#typedef-delim-token) */ 105 | const SYMBOL = 0x0001; 106 | 107 | /** [``](https://drafts.csswg.org/css-syntax/#comment-diagram) */ 108 | const COMMENT = 0x0002; 109 | 110 | /** [``](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 111 | const SPACE = 0x0003; 112 | 113 | /** [``](https://drafts.csswg.org/css-syntax/#ident-token-diagram) */ 114 | const WORD = 0x0004; 115 | 116 | /** [``](https://drafts.csswg.org/css-syntax/#function-token-diagram) */ 117 | const FUNCTION = 0x0005; 118 | 119 | /** [``](https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram) */ 120 | const ATWORD = 0x0006; 121 | 122 | /** [``](https://drafts.csswg.org/css-syntax/#hash-token-diagram) */ 123 | const HASH = 0x0007; 124 | 125 | /** [``](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 126 | const STRING = 0x0008; 127 | 128 | /** [``](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */ 129 | const NUMBER = 0x0009; 130 | 131 | /** [``](https://sass-lang.com/documentation/variables) */ 132 | const VARIABLE = 0x0010; 133 | 134 | const { 135 | fromCharCode 136 | } = String; 137 | 138 | /** Consumes and returns a token. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 139 | const consume = state => { 140 | switch (true) { 141 | /* 142 | /* https://drafts.csswg.org/css-syntax/#consume-comment */ 143 | case state.codeAt0 === SOLIDUS: 144 | if (state.codeAt1 === ASTERISK) return consumeCommentToken(state); 145 | break; 146 | /* 147 | /* https://drafts.csswg.org/css-syntax/#whitespace-token-diagram */ 148 | case space(state.codeAt0): 149 | return consumeSpaceToken(state); 150 | /* 151 | /* https://drafts.csswg.org/css-syntax/#string-token-diagram */ 152 | case state.codeAt0 === QUOTATION_MARK: 153 | case state.codeAt0 === APOSTROPHE: 154 | // "" || '' 155 | return consumeStringToken(state); 156 | /* 157 | /* https://drafts.csswg.org/css-syntax/#hash-token-diagram */ 158 | case state.codeAt0 === NUMBER_SIGN: 159 | // #W 160 | if (identifier(state.codeAt1)) return { 161 | tick: state.tick, 162 | type: HASH, 163 | code: -1, 164 | lead: consumeAnyValue(state), 165 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 166 | tail: '' 167 | }; 168 | // #\: 169 | if (validEscape(state.codeAt1, state.codeAt2)) return { 170 | tick: state.tick, 171 | type: HASH, 172 | code: -1, 173 | lead: consumeAnyValue(state), 174 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 175 | tail: '' 176 | }; 177 | break; 178 | /* 179 | /* https://sass-lang.com/documentation/variables */ 180 | case state.codeAt0 === DOLLAR_SIGN: 181 | // $W 182 | if (identifier(state.codeAt1)) return { 183 | tick: state.tick, 184 | type: VARIABLE, 185 | code: -1, 186 | lead: consumeAnyValue(state), 187 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 188 | tail: '' 189 | }; 190 | // $\: 191 | if (validEscape(state.codeAt1, state.codeAt2)) return { 192 | tick: state.tick, 193 | type: VARIABLE, 194 | code: -1, 195 | lead: consumeAnyValue(state), 196 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 197 | tail: '' 198 | }; 199 | break; 200 | /* */ 201 | /* https://drafts.csswg.org/css-syntax/#ident-token-diagram */ 202 | case state.codeAt0 === REVERSE_SOLIDUS: 203 | if (validEscape(state.codeAt0, state.codeAt1)) return consumeIdentifierLikeToken(state, { 204 | tick: state.tick, 205 | type: WORD, 206 | code: -1, 207 | lead: '', 208 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 209 | tail: '' 210 | }); 211 | break; 212 | case identifierStart(state.codeAt0): 213 | // W 214 | return consumeIdentifierLikeToken(state, { 215 | tick: state.tick, 216 | type: WORD, 217 | code: -1, 218 | lead: '', 219 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 220 | tail: '' 221 | }); 222 | case state.codeAt0 === HYPHEN_MINUS: 223 | // -W 224 | if (state.codeAt1 === HYPHEN_MINUS || identifierStart(state.codeAt1)) return consumeIdentifierLikeToken(state, { 225 | tick: state.tick, 226 | type: WORD, 227 | code: -1, 228 | lead: '', 229 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 230 | tail: '' 231 | }); 232 | // -\: 233 | if (validEscape(state.codeAt1, state.codeAt2)) return consumeIdentifierLikeToken(state, { 234 | tick: state.tick, 235 | type: WORD, 236 | code: -1, 237 | lead: '', 238 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 239 | tail: '' 240 | }); 241 | /* */ 242 | /* https://drafts.csswg.org/css-syntax/#number-token-diagram */ 243 | // -8 244 | if (digit(state.codeAt1)) return { 245 | tick: state.tick, 246 | type: NUMBER, 247 | code: -1, 248 | lead: '', 249 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 250 | tail: consumeNumericUnitValue(state) 251 | }; 252 | // -.8 253 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 254 | tick: state.tick, 255 | type: NUMBER, 256 | code: -1, 257 | lead: '', 258 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 259 | tail: consumeNumericUnitValue(state) 260 | }; 261 | case state.codeAt0 === FULL_STOP: 262 | // .8 263 | if (digit(state.codeAt1)) return { 264 | tick: state.tick, 265 | type: NUMBER, 266 | code: -1, 267 | lead: '', 268 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 269 | tail: consumeNumericUnitValue(state) 270 | }; 271 | break; 272 | case state.codeAt0 === PLUS_SIGN: 273 | // +8 274 | if (digit(state.codeAt1)) return { 275 | tick: state.tick, 276 | type: NUMBER, 277 | code: -1, 278 | lead: '', 279 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 280 | tail: consumeNumericUnitValue(state) 281 | }; 282 | // +.8 283 | if (state.codeAt1 === FULL_STOP && digit(state.codeAt2)) return { 284 | tick: state.tick, 285 | type: NUMBER, 286 | code: -1, 287 | lead: '', 288 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state), 289 | tail: consumeNumericUnitValue(state) 290 | }; 291 | break; 292 | case digit(state.codeAt0): 293 | // 8 294 | return { 295 | tick: state.tick, 296 | type: NUMBER, 297 | code: -1, 298 | lead: '', 299 | data: consumeAnyValue(state) + consumeNumberSansAdditiveValue(state), 300 | tail: consumeNumericUnitValue(state) 301 | }; 302 | /* */ 303 | /* https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram */ 304 | case state.codeAt0 === COMMERCIAL_AT: 305 | if (state.codeAt1 === HYPHEN_MINUS) { 306 | // @-- 307 | if (state.codeAt2 === HYPHEN_MINUS) return { 308 | tick: state.tick, 309 | type: ATWORD, 310 | code: -1, 311 | lead: consumeAnyValue(state), 312 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 313 | tail: '' 314 | }; 315 | // @-W 316 | if (identifierStart(state.codeAt2)) return { 317 | tick: state.tick, 318 | type: ATWORD, 319 | code: -1, 320 | lead: consumeAnyValue(state), 321 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 322 | tail: '' 323 | }; 324 | // @-\: 325 | if (validEscape(state.codeAt2, state.codeAt3)) return { 326 | tick: state.tick, 327 | type: ATWORD, 328 | code: -1, 329 | lead: consumeAnyValue(state), 330 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 331 | tail: '' 332 | }; 333 | } 334 | // @W 335 | if (identifierStart(state.codeAt1)) return { 336 | tick: state.tick, 337 | type: ATWORD, 338 | code: -1, 339 | lead: consumeAnyValue(state), 340 | data: consumeAnyValue(state) + consumeIdentifierValue(state), 341 | tail: '' 342 | }; 343 | // @\: 344 | if (validEscape(state.codeAt1, state.codeAt2)) return { 345 | tick: state.tick, 346 | type: ATWORD, 347 | code: -1, 348 | lead: consumeAnyValue(state), 349 | data: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state), 350 | tail: '' 351 | }; 352 | break; 353 | } 354 | /* */ 355 | /* https://drafts.csswg.org/css-syntax/#typedef-delim-token */ 356 | return { 357 | tick: state.tick, 358 | type: SYMBOL, 359 | code: state.codeAt0, 360 | lead: '', 361 | data: consumeAnyValue(state), 362 | tail: '' 363 | }; 364 | }; 365 | 366 | /** Consume and return a value. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */ 367 | const consumeAnyValue = state => { 368 | const result = fromCharCode(state.codeAt0); 369 | state.next(); 370 | return result; 371 | }; 372 | 373 | /** Consume and return an identifier value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 374 | const consumeIdentifierValue = state => { 375 | let result = ''; 376 | while (true) { 377 | switch (true) { 378 | case validEscape(state.codeAt0, state.codeAt1): 379 | result += fromCharCode(state.codeAt0); 380 | state.next(); 381 | case identifier(state.codeAt0): 382 | result += fromCharCode(state.codeAt0); 383 | state.next(); 384 | continue; 385 | } 386 | break; 387 | } 388 | return result; 389 | }; 390 | 391 | /** Consume and return an identifier or function token. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 392 | const consumeIdentifierLikeToken = (state, token) => { 393 | if (state.codeAt0 === LEFT_PARENTHESIS) { 394 | token.code = 40; 395 | token.type = FUNCTION; 396 | token.lead = token.data; 397 | token.data = '('; 398 | state.next(); 399 | } 400 | return token; 401 | }; 402 | 403 | /** Consume and return a comment token. [↗](https://drafts.csswg.org/css-syntax/#consume-comment) */ 404 | const consumeCommentToken = state => { 405 | const token = { 406 | tick: state.tick, 407 | type: COMMENT, 408 | code: -1, 409 | lead: '/*', 410 | data: '', 411 | tail: '' 412 | }; 413 | state.next(); 414 | state.next(); 415 | while (state.tick < state.size) { 416 | // @ts-ignore 417 | if (state.codeAt0 === ASTERISK && state.codeAt1 === SOLIDUS) { 418 | token.tail = '*/'; 419 | state.next(); 420 | state.next(); 421 | break; 422 | } 423 | token.data += consumeAnyValue(state); 424 | } 425 | return token; 426 | }; 427 | 428 | /** Consumes and returns a space token. [↗](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */ 429 | const consumeSpaceToken = state => { 430 | const token = { 431 | tick: state.tick, 432 | type: SPACE, 433 | code: -1, 434 | lead: '', 435 | data: consumeAnyValue(state), 436 | tail: '' 437 | }; 438 | while (state.tick < state.size) { 439 | if (!space(state.codeAt0)) break; 440 | token.data += consumeAnyValue(state); 441 | } 442 | return token; 443 | }; 444 | 445 | /** Consumes and returns a string token. [↗](https://drafts.csswg.org/css-syntax/#string-token-diagram) */ 446 | const consumeStringToken = state => { 447 | const { 448 | codeAt0 449 | } = state; 450 | const token = { 451 | tick: state.tick, 452 | type: STRING, 453 | code: -1, 454 | lead: '', 455 | data: consumeAnyValue(state), 456 | tail: '' 457 | }; 458 | while (state.tick < state.size) { 459 | switch (true) { 460 | case validEscape(state.codeAt0, state.codeAt1): 461 | token.data += consumeAnyValue(state); 462 | default: 463 | token.data += consumeAnyValue(state); 464 | continue; 465 | case state.codeAt0 === codeAt0: 466 | token.tail = consumeAnyValue(state); 467 | } 468 | break; 469 | } 470 | return token; 471 | }; 472 | 473 | /** Consumes and returns a number value after an additive symbol. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 474 | const consumeNumberSansAdditiveValue = state => { 475 | let result = ''; 476 | result += consumeDigitValue(state); 477 | if (state.codeAt0 === FULL_STOP && digit(state.codeAt1)) result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 478 | return result + consumeNumberSansDecimalValue(state); 479 | }; 480 | 481 | /** Consumes and returns a number value after a decimal place. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 482 | const consumeNumberSansDecimalValue = state => { 483 | let result = ''; 484 | result += consumeDigitValue(state); 485 | if (state.codeAt0 === LATIN_CAPITAL_LETTER_E || state.codeAt0 === LATIN_SMALL_LETTER_E) { 486 | switch (true) { 487 | case state.codeAt1 === PLUS_SIGN || state.codeAt1 === HYPHEN_MINUS: 488 | if (!digit(state.codeAt2)) break; 489 | result += consumeAnyValue(state); 490 | case digit(state.codeAt1): 491 | result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state); 492 | } 493 | } 494 | return result; 495 | }; 496 | 497 | /** Consumes and returns a digit value. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */ 498 | const consumeDigitValue = state => { 499 | let result = ''; 500 | while (state.tick < state.size) { 501 | if (!digit(state.codeAt0)) break; 502 | result += consumeAnyValue(state); 503 | } 504 | return result; 505 | }; 506 | 507 | /** Consumes and returns a numeric unit value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */ 508 | const consumeNumericUnitValue = state => state.codeAt0 === HYPHEN_MINUS ? state.codeAt1 === HYPHEN_MINUS ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : identifierStart(state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt1, state.codeAt2) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : '' : identifierStart(state.codeAt0) ? consumeAnyValue(state) + consumeIdentifierValue(state) : validEscape(state.codeAt0, state.codeAt1) ? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state) : ''; 509 | 510 | /** Returns a CSS iterator to yield tokens from the given CSS data. */ 511 | const tokenize = data => { 512 | let size = data.length; 513 | let tick = 0; 514 | 515 | /** Condition of the current tokenizer. */ 516 | let state = { 517 | data, 518 | size, 519 | tick, 520 | codeAt0: tick + 0 < size ? data.charCodeAt(tick + 0) : -1, 521 | codeAt1: tick + 1 < size ? data.charCodeAt(tick + 1) : -1, 522 | codeAt2: tick + 2 < size ? data.charCodeAt(tick + 2) : -1, 523 | codeAt3: tick + 3 < size ? data.charCodeAt(tick + 3) : -1, 524 | /** Advances the unicode characters being read from the CSS data by one position. */ 525 | next() { 526 | state.tick = ++tick; 527 | state.codeAt0 = state.codeAt1; 528 | state.codeAt1 = state.codeAt2; 529 | state.codeAt2 = state.codeAt3; 530 | state.codeAt3 = tick + 3 < size ? data.charCodeAt(tick + 3) : -1; 531 | return tick >= size; 532 | } 533 | }; 534 | 535 | /** Returns the most recent state and token yielded from the CSS iterator. */ 536 | const iterator = () => state.tick >= state.size ? { 537 | done: true, 538 | value: { 539 | tick: state.tick, 540 | type: 0, 541 | code: -2, 542 | lead: '', 543 | data: '', 544 | tail: '' 545 | } 546 | } : { 547 | done: false, 548 | value: consume(state) 549 | }; 550 | iterator[Symbol.iterator] = () => ({ 551 | next: iterator 552 | }); 553 | return iterator; 554 | }; 555 | 556 | exports.tokenize = tokenize; 557 | //# sourceMappingURL=tokenizeSCSS.cjs.map 558 | -------------------------------------------------------------------------------- /dist/tokenize.cjs.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"tokenize.cjs","sources":["../src/lib/code-points.ts","../src/lib/is.ts","../src/lib/token-types.ts","../src/lib/consume.ts","../src/tokenize.ts"],"sourcesContent":["/**\n * Unicode Character Codes (0x0000 - 0x0080)\n * @see https://www.unicode.org/Public/UCD/latest/ucd/NamesList.txt\n * @see https://unicode.org/charts/nameslist/n_0000.html\n *//** */\n\n/**\n * C0 controls\n * ===================================================================== *//***/\n\n/** ␀ */ export const NULL = 0x0000\n/** ␁ */ export const START_OF_HEADING = 0x0001\n/** ␂ */ export const START_OF_TEXT = 0x0002\n/** ␃ */ export const END_OF_TEXT = 0x0003\n/** ␄ */ export const END_OF_TRANSMISSION = 0x0004\n/** ␅ */ export const ENQUIRY = 0x0005\n/** ␆ */ export const ACKNOWLEDGE = 0x0006\n/** ␇ */ export const BELL = 0x0007\n/** ␈ */ export const BACKSPACE = 0x0008\n/** ␉ */ export const CHARACTER_TABULATION = 0x0009\n/** ␊ */ export const LINE_FEED = 0x000A\n/** ␋ */ export const LINE_TABULATION = 0x000B\n/** ␌ */ export const FORM_FEED = 0x000C\n/** ␍ */ export const CARRIAGE_RETURN = 0x000D\n/** ␎ */ export const SHIFT_OUT = 0x000E\n/** ␏ */ export const SHIFT_IN = 0x000F\n/** ␐ */ export const DATA_LINK_ESCAPE = 0x0010\n/** ␑ */ export const DEVICE_CONTROL_ONE = 0x0011\n/** ␒ */ export const DEVICE_CONTROL_TWO = 0x0012\n/** ␓ */ export const DEVICE_CONTROL_THREE = 0x0013\n/** ␔ */ export const DEVICE_CONTROL_FOUR = 0x0014\n/** ␕ */ export const NEGATIVE_ACKNOWLEDGE = 0x0015\n/** ␖ */ export const SYNCHRONOUS_IDLE = 0x0016\n/** ␗ */ export const END_OF_TRANSMISSION_BLOCK = 0x0017\n/** ␘ */ export const CANCEL = 0x0018\n/** ␙ */ export const END_OF_MEDIUM = 0x0019\n/** ␚ */ export const SUBSTITUTE = 0x001A\n/** ␛ */ export const ESCAPE = 0x001B\n/** ␜ */ export const INFORMATION_SEPARATOR_FOUR = 0x001C\n/** ␝ */ export const INFORMATION_SEPARATOR_THREE = 0x001D\n/** ␞ */ export const INFORMATION_SEPARATOR_TWO = 0x001E\n/** ␟ */ export const INFORMATION_SEPARATOR_ONE = 0x001F\n\n/**\n * ASCII punctuation and symbols\n * ===================================================================== *//***/\n\n/** ␠ */ export const SPACE = 0x0020\n/** ! */ export const EXCLAMATION_MARK = 0x0021\n/** \" */ export const QUOTATION_MARK = 0x0022\n/** # */ export const NUMBER_SIGN = 0x0023\n/** $ */ export const DOLLAR_SIGN = 0x0024\n/** % */ export const PERCENT_SIGN = 0x0025\n/** & */ export const AMPERSAND = 0x0026\n/** ' */ export const APOSTROPHE = 0x0027\n/** ( */ export const LEFT_PARENTHESIS = 0x0028\n/** ) */ export const RIGHT_PARENTHESIS = 0x0029\n/** * */ export const ASTERISK = 0x002A\n/** + */ export const PLUS_SIGN = 0x002B\n/** , */ export const COMMA = 0x002C\n/** - */ export const HYPHEN_MINUS = 0x002D\n/** . */ export const FULL_STOP = 0x002E\n/** / */ export const SOLIDUS = 0x002F\n\n/*\n * ASCII digits\n * ========================================================================== */\n\n/** 0 */ export const DIGIT_ZERO = 0x0030\n/** 1 */ export const DIGIT_ONE = 0x0031\n/** 2 */ export const DIGIT_TWO = 0x0032\n/** 3 */ export const DIGIT_THREE = 0x0033\n/** 4 */ export const DIGIT_FOUR = 0x0034\n/** 5 */ export const DIGIT_FIVE = 0x0035\n/** 6 */ export const DIGIT_SIX = 0x0036\n/** 7 */ export const DIGIT_SEVEN = 0x0037\n/** 8 */ export const DIGIT_EIGHT = 0x0038\n/** 9 */ export const DIGIT_NINE = 0x0039\n\n/**\n * ASCII punctuation and symbols\n * ===================================================================== *//***/\n\n/** : */ export const COLON = 0x003A\n/** ; */ export const SEMICOLON = 0x003B\n/** < */ export const LESS_THAN_SIGN = 0x003C\n/** = */ export const EQUALS_SIGN = 0x003D\n/** > */ export const GREATER_THAN_SIGN = 0x003E\n/** ? */ export const QUESTION_MARK = 0x003F\n/** @ */ export const COMMERCIAL_AT = 0x0040\n\n/**\n * Uppercase Latin alphabet\n * ===================================================================== *//***/\n\n/** A */ export const LATIN_CAPITAL_LETTER_A = 0x0041\n/** B */ export const LATIN_CAPITAL_LETTER_B = 0x0042\n/** C */ export const LATIN_CAPITAL_LETTER_C = 0x0043\n/** D */ export const LATIN_CAPITAL_LETTER_D = 0x0044\n/** E */ export const LATIN_CAPITAL_LETTER_E = 0x0045\n/** F */ export const LATIN_CAPITAL_LETTER_F = 0x0046\n/** G */ export const LATIN_CAPITAL_LETTER_G = 0x0047\n/** H */ export const LATIN_CAPITAL_LETTER_H = 0x0048\n/** I */ export const LATIN_CAPITAL_LETTER_I = 0x0049\n/** J */ export const LATIN_CAPITAL_LETTER_J = 0x004A\n/** K */ export const LATIN_CAPITAL_LETTER_K = 0x004B\n/** L */ export const LATIN_CAPITAL_LETTER_L = 0x004C\n/** M */ export const LATIN_CAPITAL_LETTER_M = 0x004D\n/** N */ export const LATIN_CAPITAL_LETTER_N = 0x004E\n/** O */ export const LATIN_CAPITAL_LETTER_O = 0x004F\n/** P */ export const LATIN_CAPITAL_LETTER_P = 0x0050\n/** Q */ export const LATIN_CAPITAL_LETTER_Q = 0x0051\n/** R */ export const LATIN_CAPITAL_LETTER_R = 0x0052\n/** S */ export const LATIN_CAPITAL_LETTER_S = 0x0053\n/** T */ export const LATIN_CAPITAL_LETTER_T = 0x0054\n/** U */ export const LATIN_CAPITAL_LETTER_U = 0x0055\n/** V */ export const LATIN_CAPITAL_LETTER_V = 0x0056\n/** W */ export const LATIN_CAPITAL_LETTER_W = 0x0057\n/** X */ export const LATIN_CAPITAL_LETTER_X = 0x0058\n/** Y */ export const LATIN_CAPITAL_LETTER_Y = 0x0059\n/** Z */ export const LATIN_CAPITAL_LETTER_Z = 0x005A\n\n/**\n * ASCII punctuation and symbols\n * ===================================================================== *//***/\n\n/** [ */ export const LEFT_SQUARE_BRACKET = 0x005B\n/** \\ */ export const REVERSE_SOLIDUS = 0x005C\n/** ] */ export const RIGHT_SQUARE_BRACKET = 0x005D\n/** ^ */ export const CIRCUMFLEX_ACCENT = 0x005E\n/** _ */ export const LOW_LINE = 0x005F\n/** ` */ export const GRAVE_ACCENT = 0x0060\n\n/*\n * Lowercase Latin alphabet\n * ========================================================================== */\n\n/** a */ export const LATIN_SMALL_LETTER_A = 0x0061\n/** b */ export const LATIN_SMALL_LETTER_B = 0x0062\n/** c */ export const LATIN_SMALL_LETTER_C = 0x0063\n/** d */ export const LATIN_SMALL_LETTER_D = 0x0064\n/** e */ export const LATIN_SMALL_LETTER_E = 0x0065\n/** f */ export const LATIN_SMALL_LETTER_F = 0x0066\n/** g */ export const LATIN_SMALL_LETTER_G = 0x0067\n/** h */ export const LATIN_SMALL_LETTER_H = 0x0068\n/** i */ export const LATIN_SMALL_LETTER_I = 0x0069\n/** j */ export const LATIN_SMALL_LETTER_J = 0x006A\n/** k */ export const LATIN_SMALL_LETTER_K = 0x006B\n/** l */ export const LATIN_SMALL_LETTER_L = 0x006C\n/** m */ export const LATIN_SMALL_LETTER_M = 0x006D\n/** n */ export const LATIN_SMALL_LETTER_N = 0x006E\n/** o */ export const LATIN_SMALL_LETTER_O = 0x006F\n/** p */ export const LATIN_SMALL_LETTER_P = 0x0070\n/** q */ export const LATIN_SMALL_LETTER_Q = 0x0071\n/** r */ export const LATIN_SMALL_LETTER_R = 0x0072\n/** s */ export const LATIN_SMALL_LETTER_S = 0x0073\n/** t */ export const LATIN_SMALL_LETTER_T = 0x0074\n/** u */ export const LATIN_SMALL_LETTER_U = 0x0075\n/** v */ export const LATIN_SMALL_LETTER_V = 0x0076\n/** w */ export const LATIN_SMALL_LETTER_W = 0x0077\n/** x */ export const LATIN_SMALL_LETTER_X = 0x0078\n/** y */ export const LATIN_SMALL_LETTER_Y = 0x0079\n/** z */ export const LATIN_SMALL_LETTER_Z = 0x007A\n\n/**\n * ASCII punctuation and symbols\n * ===================================================================== *//***/\n\n/** { */ export const LEFT_CURLY_BRACKET = 0x007B\n/** | */ export const VERTICAL_LINE = 0x007C\n/** } */ export const RIGHT_CURLY_BRACKET = 0x007D\n/** ~ */ export const TILDE = 0x007E\n\n/**\n * Control character\n * ===================================================================== *//***/\n\n/** ␡ */ export const DELETE = 0x007F\n\n/**\n * Non-ASCII\n * ===================================================================== *//***/\n\n/** � */ export const NON_ASCII = 0x0080\n\n/**\n * EOF\n * ===================================================================== *//***/\n\n/** ⏏ */ export const EOF = -0x0001\n","import * as cp from './code-points.js'\n\n/** Returns whether the unicode value is a digit. [↗](https://drafts.csswg.org/css-syntax/#digit) */\nexport const digit = (code: number) => code >= cp.DIGIT_ZERO && code <= cp.DIGIT_NINE\n\n/** Returns whether the unicode value is an identifier. [↗](https://drafts.csswg.org/css-syntax/#identifier-code-point) */\nexport const identifier = (code: number) => (\n\tidentifierStart(code) ||\n\t(code >= cp.DIGIT_ZERO && code <= cp.DIGIT_NINE) ||\n\t(code === cp.HYPHEN_MINUS)\n)\n\n/** Returns whether the unicode value is an identifier-start. [↗](https://drafts.csswg.org/css-syntax/#identifier-start-code-point) */\nexport const identifierStart = (code: number) => (\n\t(code === cp.LOW_LINE) ||\n\t(code >= cp.NON_ASCII) ||\n\t(code >= cp.LATIN_CAPITAL_LETTER_A && code <= cp.LATIN_CAPITAL_LETTER_Z) ||\n\t(code >= cp.LATIN_SMALL_LETTER_A && code <= cp.LATIN_SMALL_LETTER_Z)\n)\n\n/** Returns whether the unicode value is a space. [↗](https://drafts.csswg.org/css-syntax/#whitespace) */\nexport const space = (code: number) => (\n\tcode === cp.CHARACTER_TABULATION\n\t|| code === cp.LINE_FEED\n\t|| code === cp.FORM_FEED\n\t|| code === cp.CARRIAGE_RETURN\n\t|| code === cp.SPACE\n)\n\n/** Returns whether the unicode values are a valid escape. [↗](https://drafts.csswg.org/css-syntax/#starts-with-a-valid-escape) */\nexport const validEscape = (code1of2: number, code2of2: number) => (\n\tcode1of2 === cp.REVERSE_SOLIDUS\n\t&& !space(code2of2)\n)\n","/** [``](https://drafts.csswg.org/css-syntax/#typedef-delim-token) */\nexport const SYMBOL = 0x0001\n\n/** [``](https://drafts.csswg.org/css-syntax/#comment-diagram) */\nexport const COMMENT = 0x0002\n\n/** [``](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */\nexport const SPACE = 0x0003\n\n/** [``](https://drafts.csswg.org/css-syntax/#ident-token-diagram) */\nexport const WORD = 0x0004\n\n/** [``](https://drafts.csswg.org/css-syntax/#function-token-diagram) */\nexport const FUNCTION = 0x0005\n\n/** [``](https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram) */\nexport const ATWORD = 0x0006\n\n/** [``](https://drafts.csswg.org/css-syntax/#hash-token-diagram) */\nexport const HASH = 0x0007\n\n/** [``](https://drafts.csswg.org/css-syntax/#string-token-diagram) */\nexport const STRING = 0x0008\n\n/** [``](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */\nexport const NUMBER = 0x0009\n","import { CSSState, CSSToken } from '../types/global/global.js'\n\nimport * as cp from './code-points.js'\nimport * as is from './is.js'\nimport * as tt from './token-types.js'\n\nconst { fromCharCode } = String\n\n/** Consumes and returns a token. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */\nexport const consume = (\n\t/** Condition of the current tokenizer. */\n\tstate: CSSState\n) => {\n\tswitch (true) {\n\t\t/* \n\t\t/* https://drafts.csswg.org/css-syntax/#consume-comment */\n\t\tcase state.codeAt0 === cp.SOLIDUS:\n\t\t\tif (state.codeAt1 === cp.ASTERISK) return consumeCommentToken(state)\n\t\t\tbreak\n\t\t/* \n\t\t/* https://drafts.csswg.org/css-syntax/#whitespace-token-diagram */\n\t\tcase is.space(state.codeAt0):\n\t\t\treturn consumeSpaceToken(state)\n\t\t/* \n\t\t/* https://drafts.csswg.org/css-syntax/#string-token-diagram */\n\t\tcase state.codeAt0 === cp.QUOTATION_MARK:\n\t\tcase state.codeAt0 === cp.APOSTROPHE:\n\t\t\t// \"\" || ''\n\t\t\treturn consumeStringToken(state)\n\t\t/* \n\t\t/* https://drafts.csswg.org/css-syntax/#hash-token-diagram */\n\t\tcase state.codeAt0 === cp.NUMBER_SIGN:\n\t\t\t// #W\n\t\t\tif (is.identifier(state.codeAt1)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.HASH,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: consumeAnyValue(state),\n\t\t\t\tdata: consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t} as CSSToken\n\t\t\t// #\\:\n\t\t\tif (is.validEscape(state.codeAt1, state.codeAt2)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.HASH,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: consumeAnyValue(state),\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t} as CSSToken\n\t\t\tbreak\n\t\t/* */\n\t\t/* https://drafts.csswg.org/css-syntax/#ident-token-diagram */\n\t\tcase state.codeAt0 === cp.REVERSE_SOLIDUS:\n\t\t\tif (is.validEscape(state.codeAt0, state.codeAt1)) return consumeIdentifierLikeToken(state, {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.WORD,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t})\n\t\t\tbreak\n\t\tcase is.identifierStart(state.codeAt0):\n\t\t\t// W\n\t\t\treturn consumeIdentifierLikeToken(state, {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.WORD,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t})\n\t\tcase state.codeAt0 === cp.HYPHEN_MINUS:\n\t\t\t// -W\n\t\t\tif (state.codeAt1 === cp.HYPHEN_MINUS || is.identifierStart(state.codeAt1)) return consumeIdentifierLikeToken(state, {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.WORD,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t})\n\t\t\t// -\\:\n\t\t\tif (is.validEscape(state.codeAt1, state.codeAt2)) return consumeIdentifierLikeToken(state, {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.WORD,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t})\n\t\t/* */\n\t\t/* https://drafts.csswg.org/css-syntax/#number-token-diagram */\n\t\t\t// -8\n\t\t\tif (is.digit(state.codeAt1)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.NUMBER,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state),\n\t\t\t\ttail: consumeNumericUnitValue(state),\n\t\t\t} as CSSToken\n\t\t\t// -.8\n\t\t\tif (state.codeAt1 === cp.FULL_STOP && is.digit(state.codeAt2)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.NUMBER,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state),\n\t\t\t\ttail: consumeNumericUnitValue(state),\n\t\t\t} as CSSToken\n\t\tcase state.codeAt0 === cp.FULL_STOP:\n\t\t\t// .8\n\t\t\tif (is.digit(state.codeAt1)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.NUMBER,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state),\n\t\t\t\ttail: consumeNumericUnitValue(state),\n\t\t\t} as CSSToken\n\t\t\tbreak\n\t\tcase state.codeAt0 === cp.PLUS_SIGN:\n\t\t\t// +8\n\t\t\tif (is.digit(state.codeAt1)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.NUMBER,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansAdditiveValue(state),\n\t\t\t\ttail: consumeNumericUnitValue(state),\n\t\t\t} as CSSToken\n\t\t\t// +.8\n\t\t\tif (state.codeAt1 === cp.FULL_STOP && is.digit(state.codeAt2)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.NUMBER,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeNumberSansDecimalValue(state),\n\t\t\t\ttail: consumeNumericUnitValue(state),\n\t\t\t} as CSSToken\n\t\t\tbreak\n\t\tcase is.digit(state.codeAt0):\n\t\t\t// 8\n\t\t\treturn {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.NUMBER,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: '',\n\t\t\t\tdata: consumeAnyValue(state) + consumeNumberSansAdditiveValue(state),\n\t\t\t\ttail: consumeNumericUnitValue(state),\n\t\t\t} as CSSToken\n\t\t/* */\n\t\t/* https://drafts.csswg.org/css-syntax/#at-keyword-token-diagram */\n\t\tcase state.codeAt0 === cp.COMMERCIAL_AT:\n\t\t\tif (state.codeAt1 === cp.HYPHEN_MINUS) {\n\t\t\t\t// @--\n\t\t\t\tif (state.codeAt2 === cp.HYPHEN_MINUS) return {\n\t\t\t\t\ttick: state.tick,\n\t\t\t\t\ttype: tt.ATWORD,\n\t\t\t\t\tcode: -1,\n\t\t\t\t\tlead: consumeAnyValue(state),\n\t\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\t\ttail: '',\n\t\t\t\t} as CSSToken\n\t\t\t\t// @-W\n\t\t\t\tif (is.identifierStart(state.codeAt2)) return {\n\t\t\t\t\ttick: state.tick,\n\t\t\t\t\ttype: tt.ATWORD,\n\t\t\t\t\tcode: -1,\n\t\t\t\t\tlead: consumeAnyValue(state),\n\t\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\t\ttail: '',\n\t\t\t\t} as CSSToken\n\t\t\t\t// @-\\:\n\t\t\t\tif (is.validEscape(state.codeAt2, state.codeAt3)) return {\n\t\t\t\t\ttick: state.tick,\n\t\t\t\t\ttype: tt.ATWORD,\n\t\t\t\t\tcode: -1,\n\t\t\t\t\tlead: consumeAnyValue(state),\n\t\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\t\ttail: '',\n\t\t\t\t} as CSSToken\n\t\t\t}\n\t\t\t// @W\n\t\t\tif (is.identifierStart(state.codeAt1)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.ATWORD,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: consumeAnyValue(state),\n\t\t\t\tdata: consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t} as CSSToken\n\t\t\t// @\\:\n\t\t\tif (is.validEscape(state.codeAt1, state.codeAt2)) return {\n\t\t\t\ttick: state.tick,\n\t\t\t\ttype: tt.ATWORD,\n\t\t\t\tcode: -1,\n\t\t\t\tlead: consumeAnyValue(state),\n\t\t\t\tdata: consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state),\n\t\t\t\ttail: '',\n\t\t\t} as CSSToken\n\t\t\tbreak\n\t}\n\t/* */\n\t/* https://drafts.csswg.org/css-syntax/#typedef-delim-token */\n\treturn {\n\t\ttick: state.tick,\n\t\ttype: tt.SYMBOL,\n\t\tcode: state.codeAt0,\n\t\tlead: '',\n\t\tdata: consumeAnyValue(state),\n\t\ttail: '',\n\t} as CSSToken\n}\n\n/** Consume and return a value. [↗](https://drafts.csswg.org/css-syntax/#consume-token) */\nconst consumeAnyValue = (state: CSSState) => {\n\tconst result = fromCharCode(state.codeAt0)\n\tstate.next()\n\treturn result\n}\n\n/** Consume and return an identifier value. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */\nconst consumeIdentifierValue = (state: CSSState) => {\n\tlet result = ''\n\twhile (true) {\n\t\tswitch (true) {\n\t\t\tcase is.validEscape(state.codeAt0, state.codeAt1):\n\t\t\t\tresult += fromCharCode(state.codeAt0)\n\t\t\t\tstate.next()\n\t\t\tcase is.identifier(state.codeAt0):\n\t\t\t\tresult += fromCharCode(state.codeAt0)\n\t\t\t\tstate.next()\n\t\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\treturn result\n}\n\n/** Consume and return an identifier or function token. [↗](https://drafts.csswg.org/css-syntax/#consume-an-identifier) */\nconst consumeIdentifierLikeToken = (state: CSSState, token: CSSToken) => {\n\tif (state.codeAt0 === cp.LEFT_PARENTHESIS) {\n\t\ttoken.code = 40\n\t\ttoken.type = tt.FUNCTION\n\t\ttoken.lead = token.data\n\t\ttoken.data = '('\n\t\tstate.next()\n\t}\n\treturn token\n}\n\n/** Consume and return a comment token. [↗](https://drafts.csswg.org/css-syntax/#consume-comment) */\nconst consumeCommentToken = (state: CSSState) => {\n\tconst token: CSSToken = {\n\t\ttick: state.tick,\n\t\ttype: tt.COMMENT,\n\t\tcode: -1,\n\t\tlead: '/*',\n\t\tdata: '',\n\t\ttail: '',\n\t}\n\tstate.next()\n\tstate.next()\n\twhile (state.tick < state.size) {\n\t\t// @ts-ignore\n\t\tif (state.codeAt0 === cp.ASTERISK && state.codeAt1 === cp.SOLIDUS) {\n\t\t\ttoken.tail = '*/'\n\t\t\tstate.next()\n\t\t\tstate.next()\n\t\t\tbreak\n\t\t}\n\t\ttoken.data += consumeAnyValue(state)\n\t}\n\treturn token\n}\n\n/** Consumes and returns a space token. [↗](https://drafts.csswg.org/css-syntax/#whitespace-token-diagram) */\nconst consumeSpaceToken = (state: CSSState) => {\n\tconst token: CSSToken = {\n\t\ttick: state.tick,\n\t\ttype: tt.SPACE,\n\t\tcode: -1,\n\t\tlead: '',\n\t\tdata: consumeAnyValue(state),\n\t\ttail: '',\n\t}\n\twhile (state.tick < state.size) {\n\t\tif (!is.space(state.codeAt0)) break\n\t\ttoken.data += consumeAnyValue(state)\n\t}\n\treturn token\n}\n\n/** Consumes and returns a string token. [↗](https://drafts.csswg.org/css-syntax/#string-token-diagram) */\nconst consumeStringToken = (state: CSSState) => {\n\tconst { codeAt0 } = state\n\tconst token: CSSToken = {\n\t\ttick: state.tick,\n\t\ttype: tt.STRING,\n\t\tcode: -1,\n\t\tlead: '',\n\t\tdata: consumeAnyValue(state),\n\t\ttail: '',\n\t}\n\twhile (state.tick < state.size) {\n\t\tswitch (true) {\n\t\t\tcase is.validEscape(state.codeAt0, state.codeAt1):\n\t\t\t\ttoken.data += consumeAnyValue(state)\n\t\t\tdefault:\n\t\t\t\ttoken.data += consumeAnyValue(state)\n\t\t\t\tcontinue\n\t\t\tcase state.codeAt0 === codeAt0:\n\t\t\t\ttoken.tail = consumeAnyValue(state)\n\t\t}\n\t\tbreak\n\t}\n\treturn token\n}\n\n/** Consumes and returns a number value after an additive symbol. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */\nexport const consumeNumberSansAdditiveValue = (state: CSSState) => {\n\tlet result = ''\n\tresult += consumeDigitValue(state)\n\tif (state.codeAt0 === cp.FULL_STOP && is.digit(state.codeAt1)) result += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state)\n\treturn result + consumeNumberSansDecimalValue(state)\n}\n\n/** Consumes and returns a number value after a decimal place. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */\nconst consumeNumberSansDecimalValue = (state: CSSState) => {\n\tlet result = ''\n\tresult += consumeDigitValue(state)\n\tif (state.codeAt0 === cp.LATIN_CAPITAL_LETTER_E || state.codeAt0 === cp.LATIN_SMALL_LETTER_E) {\n\t\tswitch (true) {\n\t\t\tcase (state.codeAt1 === cp.PLUS_SIGN || state.codeAt1 === cp.HYPHEN_MINUS):\n\t\t\t\tif (!is.digit(state.codeAt2)) break\n\t\t\t\tresult += consumeAnyValue(state)\n\t\t\tcase is.digit(state.codeAt1):\n\t\t\t\tresult += consumeAnyValue(state) + consumeAnyValue(state) + consumeDigitValue(state)\n\t\t}\n\t}\n\treturn result\n}\n\n/** Consumes and returns a digit value. [↗](https://drafts.csswg.org/css-syntax/#consume-a-number) */\nconst consumeDigitValue = (state: CSSState) => {\n\tlet result = ''\n\twhile (state.tick < state.size) {\n\t\tif (!is.digit(state.codeAt0)) break\n\t\tresult += consumeAnyValue(state)\n\t}\n\treturn result\n}\n\n/** Consumes and returns a numeric unit value. [↗](https://drafts.csswg.org/css-syntax/#consume-numeric-token) */\nconst consumeNumericUnitValue = (state: CSSState) => (\n\tstate.codeAt0 === cp.HYPHEN_MINUS\n\t\t? state.codeAt1 === cp.HYPHEN_MINUS\n\t\t\t? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state)\n\t\t: is.identifierStart(state.codeAt1)\n\t\t\t? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state)\n\t\t: is.validEscape(state.codeAt1, state.codeAt2)\n\t\t\t? consumeAnyValue(state) + consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state)\n\t\t: ''\n\t: state.codeAt0 === cp.PERCENT_SIGN\n\t\t? consumeAnyValue(state)\n\t: is.identifierStart(state.codeAt0)\n\t\t? consumeAnyValue(state) + consumeIdentifierValue(state)\n\t: is.validEscape(state.codeAt0, state.codeAt1)\n\t\t? consumeAnyValue(state) + consumeAnyValue(state) + consumeIdentifierValue(state)\n\t: ''\n)\n","import { CSSState, CSSIterator, CSSIteration } from './types/global/global.js'\nimport { consume } from './lib/consume.js'\n\n/** Returns a CSS iterator to yield tokens from the given CSS data. */\nexport const tokenize = (/** CSS data. */ data: string) => {\n\tlet size = data.length\n\tlet tick = 0\n\n\t/** Condition of the current tokenizer. */\n\tlet state: CSSState = {\n\t\tdata,\n\t\tsize,\n\t\ttick,\n\t\tcodeAt0: tick + 0 < size ? data.charCodeAt(tick + 0) : -1,\n\t\tcodeAt1: tick + 1 < size ? data.charCodeAt(tick + 1) : -1,\n\t\tcodeAt2: tick + 2 < size ? data.charCodeAt(tick + 2) : -1,\n\t\tcodeAt3: tick + 3 < size ? data.charCodeAt(tick + 3) : -1,\n\t\t/** Advances the unicode characters being read from the CSS data by one position. */\n\t\tnext() {\n\t\t\tstate.tick = ++tick\n\t\t\tstate.codeAt0 = state.codeAt1\n\t\t\tstate.codeAt1 = state.codeAt2\n\t\t\tstate.codeAt2 = state.codeAt3\n\t\t\tstate.codeAt3 = tick + 3 < size ? data.charCodeAt(tick + 3) : -1\n\t\t\treturn tick >= size\n\t\t}\n\t}\n\n\t/** Returns the most recent state and token yielded from the CSS iterator. */\n\tconst iterator: CSSIterator = ((): CSSIteration => (\n\t\tstate.tick >= state.size\n\t\t\t? {\n\t\t\t\tdone: true,\n\t\t\t\tvalue: { tick: state.tick, type: 0, code: -2, lead: '', data: '', tail: '' }\n\t\t\t}\n\t\t: {\n\t\t\tdone: false,\n\t\t\tvalue: consume(state),\n\t\t}\n\t)) as CSSIterator\n\n\titerator[Symbol.iterator] = () => ({ next: iterator })\n\n\treturn iterator\n}\n"],"names":["CHARACTER_TABULATION","LINE_FEED","FORM_FEED","CARRIAGE_RETURN","SPACE","QUOTATION_MARK","NUMBER_SIGN","PERCENT_SIGN","APOSTROPHE","LEFT_PARENTHESIS","ASTERISK","PLUS_SIGN","HYPHEN_MINUS","FULL_STOP","SOLIDUS","DIGIT_ZERO","DIGIT_NINE","COMMERCIAL_AT","LATIN_CAPITAL_LETTER_A","LATIN_CAPITAL_LETTER_E","LATIN_CAPITAL_LETTER_Z","REVERSE_SOLIDUS","LOW_LINE","LATIN_SMALL_LETTER_A","LATIN_SMALL_LETTER_E","LATIN_SMALL_LETTER_Z","NON_ASCII","digit","code","cp","identifier","identifierStart","space","validEscape","code1of2","code2of2","SYMBOL","COMMENT","WORD","FUNCTION","ATWORD","HASH","STRING","NUMBER","fromCharCode","String","consume","state","codeAt0","codeAt1","consumeCommentToken","is","consumeSpaceToken","consumeStringToken","tick","type","tt","lead","consumeAnyValue","data","consumeIdentifierValue","tail","codeAt2","consumeIdentifierLikeToken","consumeNumberSansAdditiveValue","consumeNumericUnitValue","consumeNumberSansDecimalValue","codeAt3","result","next","token","size","consumeDigitValue","tokenize","length","charCodeAt","iterator","done","value","Symbol"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;;AAeA;AAAgB,MAAMA,oBAAoB,GAAU,MAAM;AAC1D;AAAgB,MAAMC,SAAS,GAAqB,MAAM;AAE1D;AAAgB,MAAMC,SAAS,GAAqB,MAAM;AAC1D;AAAgB,MAAMC,eAAe,GAAe,MAAM;;AAoB1D;AACA;AACA;;AAEA;AAAgB,MAAMC,OAAK,GAAyB,MAAM;AAE1D;AAAgB,MAAMC,cAAc,GAAgB,MAAM;AAC1D;AAAgB,MAAMC,WAAW,GAAmB,MAAM;AAE1D;AAAgB,MAAMC,YAAY,GAAkB,MAAM;AAE1D;AAAgB,MAAMC,UAAU,GAAoB,MAAM;AAC1D;AAAgB,MAAMC,gBAAgB,GAAc,MAAM;AAE1D;AAAgB,MAAMC,QAAQ,GAAsB,MAAM;AAC1D;AAAgB,MAAMC,SAAS,GAAqB,MAAM;AAE1D;AAAgB,MAAMC,YAAY,GAAkB,MAAM;AAC1D;AAAgB,MAAMC,SAAS,GAAqB,MAAM;AAC1D;AAAgB,MAAMC,OAAO,GAAuB,MAAM;;AAE1D;AACA;AACA;;AAEA;AAAgB,MAAMC,UAAU,GAAoB,MAAM;AAS1D;AAAgB,MAAMC,UAAU,GAAoB,MAAM;AAY1D;AAAgB,MAAMC,aAAa,GAAiB,MAAM;;AAE1D;AACA;AACA;;AAEA;AAAgB,MAAMC,sBAAsB,GAAQ,MAAM;AAI1D;AAAgB,MAAMC,sBAAsB,GAAQ,MAAM;AAqB1D;AAAgB,MAAMC,sBAAsB,GAAQ,MAAM;AAO1D;AAAgB,MAAMC,eAAe,GAAe,MAAM;AAG1D;AAAgB,MAAMC,QAAQ,GAAsB,MAAM;;AAG1D;AACA;AACA;;AAEA;AAAgB,MAAMC,oBAAoB,GAAU,MAAM;AAI1D;AAAgB,MAAMC,oBAAoB,GAAU,MAAM;AAqB1D;AAAgB,MAAMC,oBAAoB,GAAU,MAAM;;AAiB1D;AACA;AACA;;AAEA;AAAgB,MAAMC,SAAS,GAAqB,MAAM;;ACrL1D;AACO,MAAMC,KAAK,GAAIC,IAAY,IAAKA,IAAI,IAAIC,UAAa,IAAID,IAAI,IAAIC,UAAa;;AAErF;AACO,MAAMC,UAAU,GAAIF,IAAY,IACtCG,eAAe,CAACH,IAAI,CAAC,IACpBA,IAAI,IAAIC,UAAa,IAAID,IAAI,IAAIC,UAAc,IAC/CD,IAAI,KAAKC,YACV;;AAED;AACO,MAAME,eAAe,GAAIH,IAAY,IAC1CA,IAAI,KAAKC,QAAW,IACpBD,IAAI,IAAIC,SAAa,IACrBD,IAAI,IAAIC,sBAAyB,IAAID,IAAI,IAAIC,sBAA0B,IACvED,IAAI,IAAIC,oBAAuB,IAAID,IAAI,IAAIC,oBAC5C;;AAED;AACO,MAAMG,KAAK,GAAIJ,IAAY,IACjCA,IAAI,KAAKC,oBAAuB,IAC7BD,IAAI,KAAKC,SAAY,IACrBD,IAAI,KAAKC,SAAY,IACrBD,IAAI,KAAKC,eAAkB,IAC3BD,IAAI,KAAKC,OACZ;;AAED;AACO,MAAMI,WAAW,GAAGA,CAACC,QAAgB,EAAEC,QAAgB,KAC7DD,QAAQ,KAAKL,eAAkB,IAC5B,CAACG,KAAK,CAACG,QAAQ,CAClB;;ACjCD;AACO,MAAMC,MAAM,GAAwB,MAAM;;AAEjD;AACO,MAAMC,OAAO,GAAuB,MAAM;;AAEjD;AACO,MAAMjC,KAAK,GAAyB,MAAM;;AAEjD;AACO,MAAMkC,IAAI,GAA0B,MAAM;;AAEjD;AACO,MAAMC,QAAQ,GAAsB,MAAM;;AAEjD;AACO,MAAMC,MAAM,GAAwB,MAAM;;AAEjD;AACO,MAAMC,IAAI,GAA0B,MAAM;;AAEjD;AACO,MAAMC,MAAM,GAAwB,MAAM;;AAEjD;AACO,MAAMC,MAAM,GAAwB,MAAM;;ACnBjD,MAAM;AAAEC,EAAAA;AAAa,CAAC,GAAGC,MAAM;;AAE/B;AACO,MAAMC,OAAO,GAEnBC,KAAe,IACX;AACJ,EAAA,QAAQ,IAAI;AACX;AACF;AACE,IAAA,KAAKA,KAAK,CAACC,OAAO,KAAKnB,OAAU;AAChC,MAAA,IAAIkB,KAAK,CAACE,OAAO,KAAKpB,QAAW,EAAE,OAAOqB,mBAAmB,CAACH,KAAK,CAAC;AACpE,MAAA;AACD;AACF;AACE,IAAA,KAAKI,KAAQ,CAACJ,KAAK,CAACC,OAAO,CAAC;MAC3B,OAAOI,iBAAiB,CAACL,KAAK,CAAC;AAChC;AACF;AACE,IAAA,KAAKA,KAAK,CAACC,OAAO,KAAKnB,cAAiB;AACxC,IAAA,KAAKkB,KAAK,CAACC,OAAO,KAAKnB,UAAa;AACnC;MACA,OAAOwB,kBAAkB,CAACN,KAAK,CAAC;AACjC;AACF;AACE,IAAA,KAAKA,KAAK,CAACC,OAAO,KAAKnB,WAAc;AACpC;MACA,IAAIsB,UAAa,CAACJ,KAAK,CAACE,OAAO,CAAC,EAAE,OAAO;QACxCK,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,IAAO;QACb5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAEC,eAAe,CAACX,KAAK,CAAC;QAC5BY,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AAC5Dc,QAAAA,IAAI,EAAE;OACN;AACD;AACA,MAAA,IAAIV,WAAc,CAACJ,KAAK,CAACE,OAAO,EAAEF,KAAK,CAACe,OAAO,CAAC,EAAE,OAAO;QACxDR,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,IAAO;QACb5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAEC,eAAe,CAACX,KAAK,CAAC;AAC5BY,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AACrFc,QAAAA,IAAI,EAAE;OACN;AACD,MAAA;AACD;AACA;AACA,IAAA,KAAKd,KAAK,CAACC,OAAO,KAAKnB,eAAkB;AACxC,MAAA,IAAIsB,WAAc,CAACJ,KAAK,CAACC,OAAO,EAAED,KAAK,CAACE,OAAO,CAAC,EAAE,OAAOc,0BAA0B,CAAChB,KAAK,EAAE;QAC1FO,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,IAAO;QACb5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AAC9Gc,QAAAA,IAAI,EAAE;AACP,OAAC,CAAC;AACF,MAAA;AACD,IAAA,KAAKV,eAAkB,CAACJ,KAAK,CAACC,OAAO,CAAC;AACrC;MACA,OAAOe,0BAA0B,CAAChB,KAAK,EAAE;QACxCO,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,IAAO;QACb5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;QACRE,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AAC5Dc,QAAAA,IAAI,EAAE;AACP,OAAC,CAAC;AACH,IAAA,KAAKd,KAAK,CAACC,OAAO,KAAKnB,YAAe;AACrC;MACA,IAAIkB,KAAK,CAACE,OAAO,KAAKpB,YAAe,IAAIsB,eAAkB,CAACJ,KAAK,CAACE,OAAO,CAAC,EAAE,OAAOc,0BAA0B,CAAChB,KAAK,EAAE;QACpHO,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,IAAO;QACb5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AACrFc,QAAAA,IAAI,EAAE;AACP,OAAC,CAAC;AACF;AACA,MAAA,IAAIV,WAAc,CAACJ,KAAK,CAACE,OAAO,EAAEF,KAAK,CAACe,OAAO,CAAC,EAAE,OAAOC,0BAA0B,CAAChB,KAAK,EAAE;QAC1FO,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,IAAO;QACb5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AAC9Gc,QAAAA,IAAI,EAAE;AACP,OAAC,CAAC;AACH;AACA;AACC;MACA,IAAIV,KAAQ,CAACJ,KAAK,CAACE,OAAO,CAAC,EAAE,OAAO;QACnCK,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGiB,8BAA8B,CAACjB,KAAK,CAAC;QAC7Fc,IAAI,EAAEI,uBAAuB,CAAClB,KAAK;OACnC;AACD;AACA,MAAA,IAAIA,KAAK,CAACE,OAAO,KAAKpB,SAAY,IAAIsB,KAAQ,CAACJ,KAAK,CAACe,OAAO,CAAC,EAAE,OAAO;QACrER,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGmB,6BAA6B,CAACnB,KAAK,CAAC;QACrHc,IAAI,EAAEI,uBAAuB,CAAClB,KAAK;OACnC;AACF,IAAA,KAAKA,KAAK,CAACC,OAAO,KAAKnB,SAAY;AAClC;MACA,IAAIsB,KAAQ,CAACJ,KAAK,CAACE,OAAO,CAAC,EAAE,OAAO;QACnCK,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGmB,6BAA6B,CAACnB,KAAK,CAAC;QAC5Fc,IAAI,EAAEI,uBAAuB,CAAClB,KAAK;OACnC;AACD,MAAA;AACD,IAAA,KAAKA,KAAK,CAACC,OAAO,KAAKnB,SAAY;AAClC;MACA,IAAIsB,KAAQ,CAACJ,KAAK,CAACE,OAAO,CAAC,EAAE,OAAO;QACnCK,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGiB,8BAA8B,CAACjB,KAAK,CAAC;QAC7Fc,IAAI,EAAEI,uBAAuB,CAAClB,KAAK;OACnC;AACD;AACA,MAAA,IAAIA,KAAK,CAACE,OAAO,KAAKpB,SAAY,IAAIsB,KAAQ,CAACJ,KAAK,CAACe,OAAO,CAAC,EAAE,OAAO;QACrER,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;AACRE,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGmB,6BAA6B,CAACnB,KAAK,CAAC;QACrHc,IAAI,EAAEI,uBAAuB,CAAClB,KAAK;OACnC;AACD,MAAA;AACD,IAAA,KAAKI,KAAQ,CAACJ,KAAK,CAACC,OAAO,CAAC;AAC3B;MACA,OAAO;QACNM,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAE,EAAE;QACRE,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGiB,8BAA8B,CAACjB,KAAK,CAAC;QACpEc,IAAI,EAAEI,uBAAuB,CAAClB,KAAK;OACnC;AACF;AACA;AACA,IAAA,KAAKA,KAAK,CAACC,OAAO,KAAKnB,aAAgB;AACtC,MAAA,IAAIkB,KAAK,CAACE,OAAO,KAAKpB,YAAe,EAAE;AACtC;QACA,IAAIkB,KAAK,CAACe,OAAO,KAAKjC,YAAe,EAAE,OAAO;UAC7CyB,IAAI,EAAEP,KAAK,CAACO,IAAI;UAChBC,IAAI,EAAEC,MAAS;UACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,UAAAA,IAAI,EAAEC,eAAe,CAACX,KAAK,CAAC;AAC5BY,UAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AACrFc,UAAAA,IAAI,EAAE;SACN;AACD;QACA,IAAIV,eAAkB,CAACJ,KAAK,CAACe,OAAO,CAAC,EAAE,OAAO;UAC7CR,IAAI,EAAEP,KAAK,CAACO,IAAI;UAChBC,IAAI,EAAEC,MAAS;UACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,UAAAA,IAAI,EAAEC,eAAe,CAACX,KAAK,CAAC;AAC5BY,UAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AACrFc,UAAAA,IAAI,EAAE;SACN;AACD;AACA,QAAA,IAAIV,WAAc,CAACJ,KAAK,CAACe,OAAO,EAAEf,KAAK,CAACoB,OAAO,CAAC,EAAE,OAAO;UACxDb,IAAI,EAAEP,KAAK,CAACO,IAAI;UAChBC,IAAI,EAAEC,MAAS;UACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,UAAAA,IAAI,EAAEC,eAAe,CAACX,KAAK,CAAC;AAC5BY,UAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AAC9Gc,UAAAA,IAAI,EAAE;SACN;AACF;AACA;MACA,IAAIV,eAAkB,CAACJ,KAAK,CAACE,OAAO,CAAC,EAAE,OAAO;QAC7CK,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAEC,eAAe,CAACX,KAAK,CAAC;QAC5BY,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AAC5Dc,QAAAA,IAAI,EAAE;OACN;AACD;AACA,MAAA,IAAIV,WAAc,CAACJ,KAAK,CAACE,OAAO,EAAEF,KAAK,CAACe,OAAO,CAAC,EAAE,OAAO;QACxDR,IAAI,EAAEP,KAAK,CAACO,IAAI;QAChBC,IAAI,EAAEC,MAAS;QACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,QAAAA,IAAI,EAAEC,eAAe,CAACX,KAAK,CAAC;AAC5BY,QAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC;AACrFc,QAAAA,IAAI,EAAE;OACN;AACD,MAAA;AACF;AACA;AACA;EACA,OAAO;IACNP,IAAI,EAAEP,KAAK,CAACO,IAAI;IAChBC,IAAI,EAAEC,MAAS;IACf5B,IAAI,EAAEmB,KAAK,CAACC,OAAO;AACnBS,IAAAA,IAAI,EAAE,EAAE;AACRE,IAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC;AAC5Bc,IAAAA,IAAI,EAAE;GACN;AACF,CAAC;;AAED;AACA,MAAMH,eAAe,GAAIX,KAAe,IAAK;AAC5C,EAAA,MAAMqB,MAAM,GAAGxB,YAAY,CAACG,KAAK,CAACC,OAAO,CAAC;EAC1CD,KAAK,CAACsB,IAAI,EAAE;AACZ,EAAA,OAAOD,MAAM;AACd,CAAC;;AAED;AACA,MAAMR,sBAAsB,GAAIb,KAAe,IAAK;EACnD,IAAIqB,MAAM,GAAG,EAAE;AACf,EAAA,OAAO,IAAI,EAAE;AACZ,IAAA,QAAQ,IAAI;MACX,KAAKjB,WAAc,CAACJ,KAAK,CAACC,OAAO,EAAED,KAAK,CAACE,OAAO,CAAC;AAChDmB,QAAAA,MAAM,IAAIxB,YAAY,CAACG,KAAK,CAACC,OAAO,CAAC;QACrCD,KAAK,CAACsB,IAAI,EAAE;AACb,MAAA,KAAKlB,UAAa,CAACJ,KAAK,CAACC,OAAO,CAAC;AAChCoB,QAAAA,MAAM,IAAIxB,YAAY,CAACG,KAAK,CAACC,OAAO,CAAC;QACrCD,KAAK,CAACsB,IAAI,EAAE;AACZ,QAAA;AACF;AACA,IAAA;AACD;AACA,EAAA,OAAOD,MAAM;AACd,CAAC;;AAED;AACA,MAAML,0BAA0B,GAAGA,CAAChB,KAAe,EAAEuB,KAAe,KAAK;AACxE,EAAA,IAAIvB,KAAK,CAACC,OAAO,KAAKnB,gBAAmB,EAAE;IAC1CyC,KAAK,CAAC1C,IAAI,GAAG,EAAE;AACf0C,IAAAA,KAAK,CAACf,IAAI,GAAGC,QAAW;AACxBc,IAAAA,KAAK,CAACb,IAAI,GAAGa,KAAK,CAACX,IAAI;IACvBW,KAAK,CAACX,IAAI,GAAG,GAAG;IAChBZ,KAAK,CAACsB,IAAI,EAAE;AACb;AACA,EAAA,OAAOC,KAAK;AACb,CAAC;;AAED;AACA,MAAMpB,mBAAmB,GAAIH,KAAe,IAAK;AAChD,EAAA,MAAMuB,KAAe,GAAG;IACvBhB,IAAI,EAAEP,KAAK,CAACO,IAAI;IAChBC,IAAI,EAAEC,OAAU;IAChB5B,IAAI,EAAE,CAAC,CAAC;AACR6B,IAAAA,IAAI,EAAE,IAAI;AACVE,IAAAA,IAAI,EAAE,EAAE;AACRE,IAAAA,IAAI,EAAE;GACN;EACDd,KAAK,CAACsB,IAAI,EAAE;EACZtB,KAAK,CAACsB,IAAI,EAAE;AACZ,EAAA,OAAOtB,KAAK,CAACO,IAAI,GAAGP,KAAK,CAACwB,IAAI,EAAE;AAC/B;AACA,IAAA,IAAIxB,KAAK,CAACC,OAAO,KAAKnB,QAAW,IAAIkB,KAAK,CAACE,OAAO,KAAKpB,OAAU,EAAE;MAClEyC,KAAK,CAACT,IAAI,GAAG,IAAI;MACjBd,KAAK,CAACsB,IAAI,EAAE;MACZtB,KAAK,CAACsB,IAAI,EAAE;AACZ,MAAA;AACD;AACAC,IAAAA,KAAK,CAACX,IAAI,IAAID,eAAe,CAACX,KAAK,CAAC;AACrC;AACA,EAAA,OAAOuB,KAAK;AACb,CAAC;;AAED;AACA,MAAMlB,iBAAiB,GAAIL,KAAe,IAAK;AAC9C,EAAA,MAAMuB,KAAe,GAAG;IACvBhB,IAAI,EAAEP,KAAK,CAACO,IAAI;IAChBC,IAAI,EAAEC,KAAQ;IACd5B,IAAI,EAAE,CAAC,CAAC;AACR6B,IAAAA,IAAI,EAAE,EAAE;AACRE,IAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC;AAC5Bc,IAAAA,IAAI,EAAE;GACN;AACD,EAAA,OAAOd,KAAK,CAACO,IAAI,GAAGP,KAAK,CAACwB,IAAI,EAAE;IAC/B,IAAI,CAACpB,KAAQ,CAACJ,KAAK,CAACC,OAAO,CAAC,EAAE;AAC9BsB,IAAAA,KAAK,CAACX,IAAI,IAAID,eAAe,CAACX,KAAK,CAAC;AACrC;AACA,EAAA,OAAOuB,KAAK;AACb,CAAC;;AAED;AACA,MAAMjB,kBAAkB,GAAIN,KAAe,IAAK;EAC/C,MAAM;AAAEC,IAAAA;AAAQ,GAAC,GAAGD,KAAK;AACzB,EAAA,MAAMuB,KAAe,GAAG;IACvBhB,IAAI,EAAEP,KAAK,CAACO,IAAI;IAChBC,IAAI,EAAEC,MAAS;IACf5B,IAAI,EAAE,CAAC,CAAC;AACR6B,IAAAA,IAAI,EAAE,EAAE;AACRE,IAAAA,IAAI,EAAED,eAAe,CAACX,KAAK,CAAC;AAC5Bc,IAAAA,IAAI,EAAE;GACN;AACD,EAAA,OAAOd,KAAK,CAACO,IAAI,GAAGP,KAAK,CAACwB,IAAI,EAAE;AAC/B,IAAA,QAAQ,IAAI;MACX,KAAKpB,WAAc,CAACJ,KAAK,CAACC,OAAO,EAAED,KAAK,CAACE,OAAO,CAAC;AAChDqB,QAAAA,KAAK,CAACX,IAAI,IAAID,eAAe,CAACX,KAAK,CAAC;AACrC,MAAA;AACCuB,QAAAA,KAAK,CAACX,IAAI,IAAID,eAAe,CAACX,KAAK,CAAC;AACpC,QAAA;AACD,MAAA,KAAKA,KAAK,CAACC,OAAO,KAAKA,OAAO;AAC7BsB,QAAAA,KAAK,CAACT,IAAI,GAAGH,eAAe,CAACX,KAAK,CAAC;AACrC;AACA,IAAA;AACD;AACA,EAAA,OAAOuB,KAAK;AACb,CAAC;;AAED;AACO,MAAMN,8BAA8B,GAAIjB,KAAe,IAAK;EAClE,IAAIqB,MAAM,GAAG,EAAE;AACfA,EAAAA,MAAM,IAAII,iBAAiB,CAACzB,KAAK,CAAC;AAClC,EAAA,IAAIA,KAAK,CAACC,OAAO,KAAKnB,SAAY,IAAIsB,KAAQ,CAACJ,KAAK,CAACE,OAAO,CAAC,EAAEmB,MAAM,IAAIV,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGyB,iBAAiB,CAACzB,KAAK,CAAC;AACnJ,EAAA,OAAOqB,MAAM,GAAGF,6BAA6B,CAACnB,KAAK,CAAC;AACrD,CAAC;;AAED;AACA,MAAMmB,6BAA6B,GAAInB,KAAe,IAAK;EAC1D,IAAIqB,MAAM,GAAG,EAAE;AACfA,EAAAA,MAAM,IAAII,iBAAiB,CAACzB,KAAK,CAAC;AAClC,EAAA,IAAIA,KAAK,CAACC,OAAO,KAAKnB,sBAAyB,IAAIkB,KAAK,CAACC,OAAO,KAAKnB,oBAAuB,EAAE;AAC7F,IAAA,QAAQ,IAAI;AACX,MAAA,KAAMkB,KAAK,CAACE,OAAO,KAAKpB,SAAY,IAAIkB,KAAK,CAACE,OAAO,KAAKpB,YAAe;QACxE,IAAI,CAACsB,KAAQ,CAACJ,KAAK,CAACe,OAAO,CAAC,EAAE;AAC9BM,QAAAA,MAAM,IAAIV,eAAe,CAACX,KAAK,CAAC;AACjC,MAAA,KAAKI,KAAQ,CAACJ,KAAK,CAACE,OAAO,CAAC;AAC3BmB,QAAAA,MAAM,IAAIV,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGyB,iBAAiB,CAACzB,KAAK,CAAC;AACtF;AACD;AACA,EAAA,OAAOqB,MAAM;AACd,CAAC;;AAED;AACA,MAAMI,iBAAiB,GAAIzB,KAAe,IAAK;EAC9C,IAAIqB,MAAM,GAAG,EAAE;AACf,EAAA,OAAOrB,KAAK,CAACO,IAAI,GAAGP,KAAK,CAACwB,IAAI,EAAE;IAC/B,IAAI,CAACpB,KAAQ,CAACJ,KAAK,CAACC,OAAO,CAAC,EAAE;AAC9BoB,IAAAA,MAAM,IAAIV,eAAe,CAACX,KAAK,CAAC;AACjC;AACA,EAAA,OAAOqB,MAAM;AACd,CAAC;;AAED;AACA,MAAMH,uBAAuB,GAAIlB,KAAe,IAC/CA,KAAK,CAACC,OAAO,KAAKnB,YAAe,GAC9BkB,KAAK,CAACE,OAAO,KAAKpB,YAAe,GAChC6B,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC,GAChFI,eAAkB,CAACJ,KAAK,CAACE,OAAO,CAAC,GAChCS,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC,GAChFI,WAAc,CAACJ,KAAK,CAACE,OAAO,EAAEF,KAAK,CAACe,OAAO,CAAC,GAC3CJ,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC,GACzG,EAAE,GACHA,KAAK,CAACC,OAAO,KAAKnB,YAAe,GAChC6B,eAAe,CAACX,KAAK,CAAC,GACvBI,eAAkB,CAACJ,KAAK,CAACC,OAAO,CAAC,GAChCU,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC,GACvDI,WAAc,CAACJ,KAAK,CAACC,OAAO,EAAED,KAAK,CAACE,OAAO,CAAC,GAC3CS,eAAe,CAACX,KAAK,CAAC,GAAGW,eAAe,CAACX,KAAK,CAAC,GAAGa,sBAAsB,CAACb,KAAK,CAAC,GAChF,EACF;;AClXD;AACa0B,MAAAA,QAAQ,GAAqBd,IAAY,IAAK;AAC1D,EAAA,IAAIY,IAAI,GAAGZ,IAAI,CAACe,MAAM;EACtB,IAAIpB,IAAI,GAAG,CAAC;;AAEZ;AACA,EAAA,IAAIP,KAAe,GAAG;IACrBY,IAAI;IACJY,IAAI;IACJjB,IAAI;AACJN,IAAAA,OAAO,EAAEM,IAAI,GAAG,CAAC,GAAGiB,IAAI,GAAGZ,IAAI,CAACgB,UAAU,CAACrB,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACzDL,IAAAA,OAAO,EAAEK,IAAI,GAAG,CAAC,GAAGiB,IAAI,GAAGZ,IAAI,CAACgB,UAAU,CAACrB,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACzDQ,IAAAA,OAAO,EAAER,IAAI,GAAG,CAAC,GAAGiB,IAAI,GAAGZ,IAAI,CAACgB,UAAU,CAACrB,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACzDa,IAAAA,OAAO,EAAEb,IAAI,GAAG,CAAC,GAAGiB,IAAI,GAAGZ,IAAI,CAACgB,UAAU,CAACrB,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACzD;AACAe,IAAAA,IAAIA,GAAG;AACNtB,MAAAA,KAAK,CAACO,IAAI,GAAG,EAAEA,IAAI;AACnBP,MAAAA,KAAK,CAACC,OAAO,GAAGD,KAAK,CAACE,OAAO;AAC7BF,MAAAA,KAAK,CAACE,OAAO,GAAGF,KAAK,CAACe,OAAO;AAC7Bf,MAAAA,KAAK,CAACe,OAAO,GAAGf,KAAK,CAACoB,OAAO;AAC7BpB,MAAAA,KAAK,CAACoB,OAAO,GAAGb,IAAI,GAAG,CAAC,GAAGiB,IAAI,GAAGZ,IAAI,CAACgB,UAAU,CAACrB,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;MAChE,OAAOA,IAAI,IAAIiB,IAAI;AACpB;GACA;;AAED;EACA,MAAMK,QAAqB,GAAIA,MAC9B7B,KAAK,CAACO,IAAI,IAAIP,KAAK,CAACwB,IAAI,GACrB;AACDM,IAAAA,IAAI,EAAE,IAAI;AACVC,IAAAA,KAAK,EAAE;MAAExB,IAAI,EAAEP,KAAK,CAACO,IAAI;AAAEC,MAAAA,IAAI,EAAE,CAAC;MAAE3B,IAAI,EAAE,CAAC,CAAC;AAAE6B,MAAAA,IAAI,EAAE,EAAE;AAAEE,MAAAA,IAAI,EAAE,EAAE;AAAEE,MAAAA,IAAI,EAAE;AAAG;AAC5E,GAAC,GACA;AACDgB,IAAAA,IAAI,EAAE,KAAK;IACXC,KAAK,EAAEhC,OAAO,CAACC,KAAK;GAEL;AAEjB6B,EAAAA,QAAQ,CAACG,MAAM,CAACH,QAAQ,CAAC,GAAG,OAAO;AAAEP,IAAAA,IAAI,EAAEO;AAAS,GAAC,CAAC;AAEtD,EAAA,OAAOA,QAAQ;AAChB;;;;"} --------------------------------------------------------------------------------