├── src ├── react │ ├── index.ts │ ├── utils.ts │ └── renderer.ts ├── vue │ ├── index.ts │ ├── cached.ts │ └── renderer.ts ├── index.ts ├── types.ts ├── stream.ts └── tokenizer.ts ├── .github ├── FUNDING.yml └── workflows │ ├── release.yml │ └── ci.yml ├── .npmrc ├── CONTRIBUTING.md ├── .gitignore ├── netlify.toml ├── taze.config.ts ├── playground ├── src │ ├── main.ts │ ├── App.vue │ ├── renderer │ │ ├── types.ts │ │ ├── vue.ts │ │ └── react.tsx │ ├── fixture.ts │ └── Playground.vue ├── tsconfig.json ├── index.html ├── package.json └── vite.config.ts ├── eslint.config.js ├── test ├── output │ ├── slice-0.html │ ├── slice-1.html │ ├── slice-2.html │ ├── slice-3.html │ ├── merged.html │ ├── onepass.html │ └── stream-1.html ├── utils.ts ├── stream.test.ts └── tokenizer.test.ts ├── tsconfig.json ├── LICENSE.md ├── .vscode └── settings.json ├── pnpm-workspace.yaml ├── package.json └── README.md /src/react/index.ts: -------------------------------------------------------------------------------- 1 | export * from './renderer' 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [antfu] 2 | opencollective: antfu 3 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | ignore-workspace-root-check=true 2 | shell-emulator=true 3 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Please refer to https://github.com/antfu/contribute 2 | -------------------------------------------------------------------------------- /src/vue/index.ts: -------------------------------------------------------------------------------- 1 | export * from './cached' 2 | export * from './renderer' 3 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './stream' 2 | export * from './tokenizer' 3 | export * from './types' 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .DS_Store 3 | .idea 4 | *.log 5 | *.tgz 6 | coverage 7 | dist 8 | lib-cov 9 | logs 10 | node_modules 11 | temp 12 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | publish = "playground/dist" 3 | command = "pnpm run play:build" 4 | 5 | [build.environment] 6 | NODE_VERSION = "23" 7 | -------------------------------------------------------------------------------- /taze.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'taze' 2 | 3 | export default defineConfig({ 4 | exclude: [ 5 | '@shikijs/core', 6 | ], 7 | }) 8 | -------------------------------------------------------------------------------- /playground/src/main.ts: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import App from './App.vue' 3 | 4 | import '@unocss/reset/tailwind.css' 5 | import 'uno.css' 6 | 7 | createApp(App).mount('#app') 8 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | import antfu from '@antfu/eslint-config' 3 | 4 | export default antfu( 5 | { 6 | type: 'lib', 7 | react: true, 8 | pnpm: true, 9 | }, 10 | ) 11 | -------------------------------------------------------------------------------- /test/output/slice-0.html: -------------------------------------------------------------------------------- 1 |

2 | <script setup
-------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | release: 10 | uses: sxzz/workflows/.github/workflows/release.yml@v1 11 | with: 12 | publish: true 13 | permissions: 14 | contents: write 15 | id-token: write 16 | -------------------------------------------------------------------------------- /playground/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "jsx": "preserve", 5 | "lib": ["ESNext", "DOM"], 6 | "module": "ESNext", 7 | "moduleResolution": "Bundler", 8 | "resolveJsonModule": true, 9 | "strict": true, 10 | "strictNullChecks": true, 11 | "esModuleInterop": true, 12 | "skipDefaultLibCheck": true, 13 | "skipLibCheck": true 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /playground/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Shiki Stream Demo 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "jsx": "preserve", 5 | "lib": ["ESNext", "DOM"], 6 | "module": "ESNext", 7 | "moduleResolution": "Bundler", 8 | "resolveJsonModule": true, 9 | "allowImportingTsExtensions": true, 10 | "strict": true, 11 | "strictNullChecks": true, 12 | "noEmit": true, 13 | "esModuleInterop": true, 14 | "skipDefaultLibCheck": true, 15 | "skipLibCheck": true 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /playground/src/App.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 15 | 16 | 27 | -------------------------------------------------------------------------------- /playground/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "playground", 3 | "type": "module", 4 | "version": "0.0.0", 5 | "private": true, 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "preview": "vite preview" 10 | }, 11 | "dependencies": { 12 | "vue": "catalog:" 13 | }, 14 | "devDependencies": { 15 | "@types/react-dom": "catalog:", 16 | "@unocss/reset": "catalog:", 17 | "@vitejs/plugin-react": "catalog:", 18 | "@vitejs/plugin-vue": "catalog:", 19 | "@vueuse/core": "catalog:", 20 | "react-dom": "catalog:", 21 | "typescript": "catalog:", 22 | "unocss": "catalog:", 23 | "vite": "catalog:", 24 | "vite-plugin-solid": "catalog:" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /playground/src/renderer/types.ts: -------------------------------------------------------------------------------- 1 | import type { ThemedToken } from 'shiki/core' 2 | import type { RecallToken } from '../../../src' 3 | 4 | export type RendererType = 'vue' | 'react' 5 | 6 | export interface RendererUpdatePayload { 7 | stream: ReadableStream 8 | } 9 | 10 | export interface RendererFactoryResult { 11 | dispose: () => void 12 | mount: (element: HTMLElement, payload: RendererUpdatePayload) => void 13 | update: (payload: RendererUpdatePayload) => void 14 | } 15 | 16 | export interface RendererFactoryOptions { 17 | onStart?: () => void 18 | onEnd?: () => void 19 | } 20 | 21 | export type RendererFactory = (options: RendererFactoryOptions) => RendererFactoryResult 22 | -------------------------------------------------------------------------------- /test/output/slice-1.html: -------------------------------------------------------------------------------- 1 |
<script setup lang="ts">
2 | impo
-------------------------------------------------------------------------------- /playground/src/fixture.ts: -------------------------------------------------------------------------------- 1 | export const vueBefore = ` 15 | 16 | 19 | 20 | ` 26 | 27 | export const vueAfter = ` 33 | 34 | 37 | 38 | ` 44 | -------------------------------------------------------------------------------- /test/output/slice-2.html: -------------------------------------------------------------------------------- 1 |
import { ref } from 'vue'
2 | 
3 | const
-------------------------------------------------------------------------------- /src/react/utils.ts: -------------------------------------------------------------------------------- 1 | // source: https://github.com/sanity-io/use-effect-event/blob/main/src/useEffectEvent.ts 2 | import { useCallback, useInsertionEffect, useRef } from 'react' 3 | 4 | /** 5 | * This is a ponyfill of the upcoming `useEffectEvent` hook that'll arrive in React 19. 6 | * https://19.react.dev/learn/separating-events-from-effects#declaring-an-effect-event 7 | * To learn more about the ponyfill itself, see: https://blog.bitsrc.io/a-look-inside-the-useevent-polyfill-from-the-new-react-docs-d1c4739e8072 8 | * @public 9 | */ 10 | export function useEffectEvent< 11 | const T extends ( 12 | ...args: 13 | any[] 14 | ) => void, 15 | >(fn: T): T { 16 | const ref = useRef(null) 17 | useInsertionEffect(() => { 18 | ref.current = fn 19 | }, [fn]) 20 | return useCallback((...args: any) => { 21 | const latestFn = ref.current! 22 | return latestFn(...args) 23 | }, []) as unknown as T 24 | } 25 | -------------------------------------------------------------------------------- /playground/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { fileURLToPath } from 'node:url' 2 | // import { svelte as Svelte } from '@sveltejs/vite-plugin-svelte' 3 | import React from '@vitejs/plugin-react' 4 | import Vue from '@vitejs/plugin-vue' 5 | import UnoCSS from 'unocss/vite' 6 | import { defineConfig } from 'vite' 7 | // import Solid from 'vite-plugin-solid' 8 | 9 | export default defineConfig({ 10 | resolve: { 11 | alias: { 12 | 'shiki-stream/vue': fileURLToPath(new URL('../src/vue/index.ts', import.meta.url)), 13 | 'shiki-stream/react': fileURLToPath(new URL('../src/react/index.ts', import.meta.url)), 14 | 'shiki-stream': fileURLToPath(new URL('../src/index.ts', import.meta.url)), 15 | }, 16 | }, 17 | plugins: [ 18 | Vue(), 19 | UnoCSS(), 20 | // Svelte(), 21 | // Solid({ include: ['src/renderer/solid.tsx', '../src/solid/**'] }), 22 | React({ include: ['src/renderer/react.tsx', '../src/react/**'] }), 23 | ], 24 | }) 25 | -------------------------------------------------------------------------------- /playground/src/renderer/vue.ts: -------------------------------------------------------------------------------- 1 | import type { App } from 'vue' 2 | import type { RendererFactory, RendererFactoryResult } from './types' 3 | import { createApp, h, reactive } from 'vue' 4 | import { ShikiStreamRenderer } from '../../../src/vue' 5 | 6 | export const createRendererVue: RendererFactory = (options): RendererFactoryResult => { 7 | let app: App | undefined 8 | 9 | const props = reactive({ 10 | onStreamStart: options.onStart, 11 | onStreamEnd: options.onEnd, 12 | }) 13 | 14 | return { 15 | mount: (element, payload) => { 16 | Object.assign(props, payload) 17 | if (app) 18 | return 19 | app = createApp({ 20 | render: () => h(ShikiStreamRenderer, props as any), 21 | }) 22 | app.mount(element) 23 | // eslint-disable-next-line no-console 24 | console.log('Vue renderer mounted') 25 | }, 26 | 27 | update: (payload) => { 28 | Object.assign(props, payload) 29 | }, 30 | 31 | dispose: () => { 32 | app?.unmount() 33 | app = undefined 34 | }, 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /test/utils.ts: -------------------------------------------------------------------------------- 1 | import type { ThemedToken } from '@shikijs/core' 2 | import { getTokenStyleObject, stringifyTokenStyle } from '@shikijs/core' 3 | 4 | export function generateRandomTextStream(text: string, inverval = 50): ReadableStream { 5 | return new ReadableStream({ 6 | async start(controller) { 7 | let index = 0 8 | while (index < text.length) { 9 | await new Promise(r => setTimeout(r, inverval)) 10 | const length = Math.round(Math.random() * 20 + 5) 11 | const chunk = text.slice(index, index + length) 12 | index += length 13 | controller.enqueue(chunk) 14 | } 15 | controller.close() 16 | }, 17 | }) 18 | } 19 | 20 | export function tokenToHtml(token: ThemedToken): string { 21 | return `${escapeHtml(token.content)}` 22 | } 23 | 24 | export function tokensToHtml(tokens: ThemedToken[]): string { 25 | return `
${tokens.map(tokenToHtml).join('')}
` 26 | } 27 | 28 | export function escapeHtml(html: string): string { 29 | return html.replace(//g, '>') 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025-PRESENT Anthony Fu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | // Disable the default formatter, use eslint instead 3 | "prettier.enable": false, 4 | "editor.formatOnSave": false, 5 | 6 | // Auto fix 7 | "editor.codeActionsOnSave": { 8 | "source.fixAll.eslint": "explicit", 9 | "source.organizeImports": "never" 10 | }, 11 | 12 | // Silent the stylistic rules in you IDE, but still auto fix them 13 | "eslint.rules.customizations": [ 14 | { "rule": "style/*", "severity": "off" }, 15 | { "rule": "*-indent", "severity": "off" }, 16 | { "rule": "*-spacing", "severity": "off" }, 17 | { "rule": "*-spaces", "severity": "off" }, 18 | { "rule": "*-order", "severity": "off" }, 19 | { "rule": "*-dangle", "severity": "off" }, 20 | { "rule": "*-newline", "severity": "off" }, 21 | { "rule": "*quotes", "severity": "off" }, 22 | { "rule": "*semi", "severity": "off" } 23 | ], 24 | 25 | // Enable eslint for all supported languages 26 | "eslint.validate": [ 27 | "javascript", 28 | "javascriptreact", 29 | "typescript", 30 | "typescriptreact", 31 | "vue", 32 | "html", 33 | "markdown", 34 | "json", 35 | "jsonc", 36 | "yaml" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | - playgrounds/* 3 | - playground 4 | - docs 5 | - packages/* 6 | - examples/* 7 | catalog: 8 | '@antfu/eslint-config': ^6.2.0 9 | '@antfu/ni': ^27.0.1 10 | '@antfu/utils': ^9.3.0 11 | '@eslint-react/eslint-plugin': ^2.3.1 12 | '@shikijs/core': ^3.0.0 13 | '@shikijs/engine-javascript': ^3.14.0 14 | '@shikijs/langs': ^3.14.0 15 | '@shikijs/themes': ^3.14.0 16 | '@types/node': ^24.10.0 17 | '@types/react': ^19.2.2 18 | '@types/react-dom': ^19.2.2 19 | '@unocss/reset': ^66.5.4 20 | '@vitejs/plugin-react': ^5.1.0 21 | '@vitejs/plugin-vue': ^6.0.1 22 | '@vueuse/core': ^14.0.0 23 | bumpp: ^10.3.1 24 | eslint: ^9.39.1 25 | eslint-plugin-react-hooks: ^7.0.1 26 | lint-staged: ^16.2.6 27 | pnpm: ^10.20.0 28 | react: ^19.2.0 29 | react-dom: ^19.2.0 30 | shiki: ^3.14.0 31 | simple-git-hooks: ^2.13.1 32 | taze: ^19.9.0 33 | tsx: ^4.20.6 34 | typescript: ~5.9.3 35 | unbuild: ^3.6.1 36 | unocss: ^66.5.4 37 | vite: ^7.1.12 38 | vite-plugin-solid: ^2.11.10 39 | vitest: ^4.0.7 40 | vue: ^3.5.22 41 | vue-tsc: ^3.1.3 42 | onlyBuiltDependencies: 43 | - '@swc/core' 44 | - esbuild 45 | - simple-git-hooks 46 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | lint: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: pnpm/action-setup@v4 18 | with: 19 | run_install: false 20 | - uses: actions/setup-node@v4 21 | with: 22 | node-version: lts/* 23 | cache: pnpm 24 | 25 | - run: pnpm i -g @antfu/ni 26 | - run: nci 27 | - run: nr lint 28 | - run: nr typecheck 29 | 30 | test: 31 | runs-on: ${{ matrix.os }} 32 | 33 | strategy: 34 | matrix: 35 | node: [lts/*] 36 | os: [ubuntu-latest, windows-latest, macos-latest] 37 | fail-fast: false 38 | 39 | steps: 40 | - uses: actions/checkout@v4 41 | - uses: pnpm/action-setup@v4 42 | with: 43 | run_install: false 44 | - name: Set node ${{ matrix.node }} 45 | uses: actions/setup-node@v4 46 | with: 47 | node-version: ${{ matrix.node }} 48 | cache: pnpm 49 | 50 | - run: pnpm i -g @antfu/ni 51 | - run: nci 52 | - run: nr build 53 | - run: nr test 54 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import type { CodeToTokensOptions, HighlighterCore, HighlighterGeneric, ThemedToken } from '@shikijs/core' 2 | 3 | /** 4 | * A special token that indicates the number of tokens to be removed from the previous streamed tokens. 5 | * 6 | * Pass `allowRecalls: true` to the `CodeToTokenTransformStream` to enable recall tokens. 7 | */ 8 | export interface RecallToken { 9 | /** 10 | * Number of tokens to be removed from the previous streamed tokens. 11 | */ 12 | recall: number 13 | } 14 | 15 | export type CodeToTokenTransformStreamOptions = ShikiStreamTokenizerOptions & { 16 | /** 17 | * Whether to allow recall tokens to be emitted. 18 | * 19 | * A recall token is a special token that indicates the number of tokens to be removed from the previous streamed tokens. 20 | * 21 | * @default false 22 | */ 23 | allowRecalls?: boolean 24 | } 25 | 26 | export type ShikiStreamTokenizerOptions = CodeToTokensOptions & { 27 | highlighter: HighlighterCore | HighlighterGeneric 28 | } 29 | 30 | export interface ShikiStreamTokenizerEnqueueResult { 31 | /** 32 | * Number of last tokens to be recalled 33 | */ 34 | recall: number 35 | /** 36 | * Stable tokens 37 | */ 38 | stable: ThemedToken[] 39 | /** 40 | * Unstable tokens, they might or might not be recalled 41 | */ 42 | unstable: ThemedToken[] 43 | } 44 | -------------------------------------------------------------------------------- /test/stream.test.ts: -------------------------------------------------------------------------------- 1 | import type { ThemedToken } from 'shiki' 2 | import { createHighlighter } from 'shiki' 3 | import { expect, it } from 'vitest' 4 | import { CodeToTokenTransformStream } from '../src' 5 | import { generateRandomTextStream, tokensToHtml } from './utils' 6 | 7 | const fixture = ` 8 | 13 | 14 | 17 | 18 | 23 | ` 24 | 25 | it('stream transformer', async () => { 26 | const highlighter = await createHighlighter({ 27 | langs: ['vue'], 28 | themes: ['vitesse-dark', 'vitesse-light'], 29 | }) 30 | 31 | const inputStream = generateRandomTextStream(fixture) 32 | const tokensStream = inputStream 33 | .pipeThrough(new CodeToTokenTransformStream( 34 | { 35 | highlighter, 36 | lang: 'vue', 37 | themes: { 38 | dark: 'vitesse-dark', 39 | light: 'vitesse-light', 40 | }, 41 | allowRecalls: true, 42 | }, 43 | )) 44 | 45 | const tokens: ThemedToken[] = [] 46 | const reader = tokensStream.getReader() 47 | while (true) { 48 | const { done, value } = await reader.read() 49 | if (done) 50 | break 51 | if ('recall' in value) 52 | tokens.splice(-value.recall, value.recall) 53 | else 54 | tokens.push(value) 55 | } 56 | const html = tokensToHtml(tokens) 57 | await expect(html) 58 | .toMatchFileSnapshot('output/stream-1.html') 59 | }) 60 | -------------------------------------------------------------------------------- /src/stream.ts: -------------------------------------------------------------------------------- 1 | import type { ThemedToken } from '@shikijs/core' 2 | import type { CodeToTokenTransformStreamOptions, RecallToken } from './types' 3 | import { ShikiStreamTokenizer } from './tokenizer' 4 | 5 | /** 6 | * Create a transform stream that takes code chunks and emits themed tokens. 7 | */ 8 | export class CodeToTokenTransformStream extends TransformStream { 9 | readonly tokenizer: ShikiStreamTokenizer 10 | readonly options: CodeToTokenTransformStreamOptions 11 | 12 | constructor( 13 | options: CodeToTokenTransformStreamOptions, 14 | ) { 15 | const tokenizer = new ShikiStreamTokenizer(options) 16 | const { 17 | allowRecalls = false, 18 | } = options 19 | 20 | super({ 21 | async transform(chunk, controller) { 22 | const { stable, unstable: buffer, recall } = await tokenizer.enqueue(chunk) 23 | if (allowRecalls && recall > 0) { 24 | controller.enqueue({ recall } as any) 25 | } 26 | for (const token of stable) { 27 | controller.enqueue(token) 28 | } 29 | if (allowRecalls) { 30 | for (const token of buffer) { 31 | controller.enqueue(token) 32 | } 33 | } 34 | }, 35 | async flush(controller) { 36 | const { stable } = tokenizer.close() 37 | // if allow recalls, the tokens should already be sent 38 | if (!allowRecalls) { 39 | for (const token of stable) { 40 | controller.enqueue(token) 41 | } 42 | } 43 | }, 44 | }) 45 | 46 | this.tokenizer = tokenizer 47 | this.options = options 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/react/renderer.ts: -------------------------------------------------------------------------------- 1 | import type { ThemedToken } from '@shikijs/core' 2 | import type { JSX } from 'react' 3 | import type { RecallToken } from '..' 4 | import { objectId } from '@antfu/utils' 5 | import { getTokenStyleObject } from '@shikijs/core' 6 | import { createElement as h, useEffect, useState } from 'react' 7 | import { useEffectEvent } from './utils' 8 | 9 | export function ShikiStreamRenderer( 10 | { 11 | stream, 12 | onStreamStart, 13 | onStreamEnd, 14 | }: { 15 | stream: ReadableStream 16 | onStreamStart?: () => void 17 | onStreamEnd?: () => void 18 | }, 19 | ): JSX.Element { 20 | const [tokens, setTokens] = useState([]) 21 | 22 | const _onStreamStart = useEffectEvent(() => onStreamStart?.()) 23 | const _onStreamEnd = useEffectEvent(() => onStreamEnd?.()) 24 | 25 | useEffect(() => { 26 | // eslint-disable-next-line react-hooks-extra/no-direct-set-state-in-use-effect 27 | setTokens(prevTokens => prevTokens.length ? [] : prevTokens) 28 | let started = false 29 | stream.pipeTo(new WritableStream({ 30 | write(token) { 31 | if (!started) { 32 | started = true 33 | _onStreamStart() 34 | } 35 | if ('recall' in token) 36 | setTokens(tokens => tokens.slice(0, -token.recall)) 37 | else 38 | setTokens(tokens => [...tokens, token]) 39 | }, 40 | close: () => _onStreamEnd(), 41 | })) 42 | // eslint-disable-next-line react-hooks/exhaustive-deps 43 | }, [_onStreamEnd, _onStreamStart, stream]) 44 | 45 | return h( 46 | 'pre', 47 | { className: 'shiki shiki-stream' }, 48 | h( 49 | 'code', 50 | {}, 51 | tokens.map(token => h('span', { key: objectId(token), style: token.htmlStyle || getTokenStyleObject(token) }, token.content)), 52 | ), 53 | ) 54 | } 55 | -------------------------------------------------------------------------------- /test/tokenizer.test.ts: -------------------------------------------------------------------------------- 1 | import type { ShikiStreamTokenizerOptions } from '../src' 2 | import { createHighlighter } from 'shiki' 3 | import { expect, it } from 'vitest' 4 | import { ShikiStreamTokenizer } from '../src' 5 | import { tokensToHtml } from './utils' 6 | 7 | const fixture = ` 8 | 13 | 14 | 17 | 18 | 23 | ` 24 | 25 | it('exported', async () => { 26 | const highlighter = await createHighlighter({ 27 | langs: ['vue'], 28 | themes: ['vitesse-dark', 'vitesse-light'], 29 | }) 30 | const options: ShikiStreamTokenizerOptions = { 31 | highlighter, 32 | lang: 'vue', 33 | themes: { 34 | dark: 'vitesse-dark', 35 | light: 'vitesse-light', 36 | }, 37 | } 38 | const streamer = new ShikiStreamTokenizer( 39 | options, 40 | ) 41 | 42 | const cuts = [14, 30, 58, undefined] 43 | const slices = cuts.map((cut, i) => fixture.slice(cuts[i - 1] || 0, cut)) 44 | 45 | let index = 0 46 | for (const slice of slices) { 47 | const { stable, unstable } = await streamer.enqueue(slice) 48 | await expect.soft(tokensToHtml([...stable, ...unstable])) 49 | .toMatchFileSnapshot(`output/slice-${index++}.html`) 50 | } 51 | streamer.close() 52 | 53 | const onepassTokens = highlighter.codeToTokens(fixture, options as any) 54 | .tokens 55 | .flatMap((i, idx) => idx === 0 ? i : [{ content: '\n', offset: 0 }, ...i]) 56 | const onepass = tokensToHtml(onepassTokens) 57 | 58 | const merged = tokensToHtml(streamer.tokensStable) 59 | await expect.soft(merged) 60 | .toMatchFileSnapshot(`output/merged.html`) 61 | 62 | await expect.soft(onepass) 63 | .toMatchFileSnapshot(`output/onepass.html`) 64 | 65 | expect.soft(merged).toBe(onepass) 66 | }) 67 | -------------------------------------------------------------------------------- /src/vue/cached.ts: -------------------------------------------------------------------------------- 1 | import type { HighlighterCore } from '@shikijs/core' 2 | import type { PropType } from 'vue' 3 | import { defineComponent, h, ref, watchEffect } from 'vue' 4 | import { CodeToTokenTransformStream } from '../stream' 5 | import { ShikiStreamRenderer } from './renderer' 6 | 7 | /** 8 | * A simple wrapper around `ShikiStreamRenderer` that caches the code and only re-renders when the code changes. 9 | * 10 | * This component expects `code` prop to only be incrementally updated, and not set to a new value. 11 | * If you need to set the `code` prop to a new value, set a different `key` prop when it happens. 12 | */ 13 | export const ShikiCachedRenderer = defineComponent({ 14 | name: 'ShikiCachedRenderer', 15 | props: { 16 | code: { 17 | type: String, 18 | required: true, 19 | }, 20 | lang: { 21 | type: String, 22 | required: true, 23 | }, 24 | theme: { 25 | type: String, 26 | required: true, 27 | }, 28 | highlighter: { 29 | type: Object as PropType, 30 | required: true, 31 | }, 32 | }, 33 | emits: ['stream-start', 'stream-end'], 34 | setup(props, { emit }) { 35 | const index = ref(0) 36 | let controller: ReadableStreamController | null = null 37 | const textStream = new ReadableStream({ 38 | start(_controller) { 39 | controller = _controller 40 | }, 41 | }) 42 | 43 | watchEffect(() => { 44 | if (props.code.length > index.value) { 45 | controller?.enqueue(props.code.slice(index.value) as any) 46 | index.value = props.code.length 47 | } 48 | }) 49 | 50 | const stream = textStream 51 | .pipeThrough(new CodeToTokenTransformStream({ 52 | highlighter: props.highlighter, 53 | lang: props.lang, 54 | theme: props.theme, 55 | allowRecalls: true, 56 | })) 57 | 58 | return () => h( 59 | ShikiStreamRenderer, 60 | { 61 | stream, 62 | onStreamStart: () => emit('stream-start'), 63 | onStreamEnd: () => emit('stream-end'), 64 | }, 65 | ) 66 | }, 67 | }) 68 | -------------------------------------------------------------------------------- /playground/src/renderer/react.tsx: -------------------------------------------------------------------------------- 1 | import type { JSX } from 'react' 2 | import type { Root } from 'react-dom/client' 3 | /* eslint-disable no-console */ 4 | import type { RendererFactory, RendererFactoryResult } from './types' 5 | import React from 'react' 6 | import ReactDOM from 'react-dom/client' 7 | import { shallowReactive, watch } from 'vue' 8 | import { ShikiStreamRenderer } from '../../../src/react' 9 | 10 | export const createRendererReact: RendererFactory = (options): RendererFactoryResult => { 11 | let app: Root | undefined 12 | 13 | const props = shallowReactive({ 14 | onStreamStart: options.onStart, 15 | onStreamEnd: options.onEnd, 16 | }) 17 | 18 | function App(): JSX.Element { 19 | const [count, setCounter] = React.useState(0) 20 | 21 | React.useEffect(() => { 22 | return watch(props, () => { 23 | // Force React to re-render 24 | setCounter(c => c + 1) 25 | }) 26 | }, []) 27 | 28 | console.log('React rendering', count) 29 | 30 | const [i, setI] = React.useState(0) 31 | 32 | React.useEffect(() => { 33 | const timerId = setInterval(() => { 34 | setI(i => i + 1) 35 | }, 1_000) 36 | return () => clearInterval(timerId) 37 | }, []) 38 | 39 | return ( 40 | { 43 | console.log('onStreamStart', i) 44 | props.onStreamStart?.() 45 | }} 46 | onStreamEnd={() => { 47 | console.log('onStreamEnd', i) 48 | props.onStreamEnd?.() 49 | }} 50 | className={props.class} 51 | /> 52 | ) 53 | } 54 | 55 | return { 56 | mount: (element, payload) => { 57 | Object.assign(props, payload) 58 | app = ReactDOM.createRoot(element) 59 | app.render( 60 | 61 | 62 | , 63 | ) 64 | 65 | console.log('React renderer mounted') 66 | }, 67 | 68 | update: (payload) => { 69 | Object.assign(props, payload) 70 | }, 71 | 72 | dispose: () => { 73 | app?.unmount() 74 | app = undefined 75 | }, 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/vue/renderer.ts: -------------------------------------------------------------------------------- 1 | import type { ThemedToken } from '@shikijs/core' 2 | import type { PropType } from 'vue' 3 | import type { RecallToken } from '..' 4 | import { objectId } from '@antfu/utils' 5 | import { getTokenStyleObject } from '@shikijs/core' 6 | import { defineComponent, h, reactive, renderList, watch } from 'vue' 7 | 8 | export const ShikiStreamRenderer = defineComponent({ 9 | name: 'ShikiStreamRenderer', 10 | props: { 11 | stream: { 12 | type: Object as PropType>, 13 | required: true, 14 | }, 15 | }, 16 | emits: ['stream-start', 'stream-end'], 17 | setup(props, { emit }) { 18 | const tokens = reactive([]) 19 | let currentAbortController: AbortController | null = null 20 | 21 | watch( 22 | () => props.stream, 23 | (newStream) => { 24 | tokens.length = 0 25 | 26 | if (currentAbortController) { 27 | currentAbortController.abort() 28 | } 29 | 30 | currentAbortController = new AbortController() 31 | const signal = currentAbortController.signal 32 | let started = false 33 | 34 | newStream.pipeTo(new WritableStream({ 35 | write(token) { 36 | if (signal.aborted) { 37 | return 38 | } 39 | 40 | if (!started) { 41 | started = true 42 | emit('stream-start') 43 | } 44 | if ('recall' in token) 45 | tokens.length -= token.recall 46 | else 47 | tokens.push(token) 48 | }, 49 | close: () => { 50 | if (!signal.aborted) { 51 | emit('stream-end') 52 | } 53 | }, 54 | abort: () => { 55 | if (!signal.aborted) { 56 | emit('stream-end') 57 | } 58 | }, 59 | }), { signal }).catch((error) => { 60 | if (error.name !== 'AbortError') { 61 | console.error('Stream error:', error) 62 | } 63 | }) 64 | }, 65 | { immediate: true }, 66 | ) 67 | 68 | return () => h( 69 | 'pre', 70 | { class: 'shiki shiki-stream' }, 71 | h( 72 | 'code', 73 | renderList(tokens, token => h('span', { key: objectId(token), style: token.htmlStyle || getTokenStyleObject(token) }, token.content)), 74 | ), 75 | ) 76 | }, 77 | }) 78 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "shiki-stream", 3 | "type": "module", 4 | "version": "0.1.3", 5 | "packageManager": "pnpm@10.20.0", 6 | "description": "Streaming colorization for Shiki", 7 | "author": "Anthony Fu ", 8 | "license": "MIT", 9 | "funding": "https://github.com/sponsors/antfu", 10 | "homepage": "https://github.com/antfu/shiki-stream#readme", 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/antfu/shiki-stream.git" 14 | }, 15 | "bugs": "https://github.com/antfu/shiki-stream/issues", 16 | "keywords": [], 17 | "sideEffects": false, 18 | "exports": { 19 | ".": "./dist/index.mjs", 20 | "./vue": "./dist/vue.mjs", 21 | "./react": "./dist/react.mjs" 22 | }, 23 | "main": "./dist/index.mjs", 24 | "module": "./dist/index.mjs", 25 | "types": "./dist/index.d.mts", 26 | "files": [ 27 | "dist" 28 | ], 29 | "scripts": { 30 | "build": "unbuild", 31 | "dev": "unbuild --stub", 32 | "lint": "eslint .", 33 | "prepublishOnly": "nr build", 34 | "release": "bumpp", 35 | "start": "tsx src/index.ts", 36 | "test": "vitest", 37 | "typecheck": "vue-tsc --noEmit", 38 | "prepare": "simple-git-hooks", 39 | "play": "nr -C playground dev", 40 | "play:build": "nr -C playground build" 41 | }, 42 | "peerDependencies": { 43 | "react": "^19.0.0", 44 | "vue": "^3.2.0" 45 | }, 46 | "peerDependenciesMeta": { 47 | "react": { 48 | "optional": true 49 | }, 50 | "vue": { 51 | "optional": true 52 | } 53 | }, 54 | "dependencies": { 55 | "@shikijs/core": "catalog:" 56 | }, 57 | "devDependencies": { 58 | "@antfu/eslint-config": "catalog:", 59 | "@antfu/ni": "catalog:", 60 | "@antfu/utils": "catalog:", 61 | "@eslint-react/eslint-plugin": "catalog:", 62 | "@shikijs/engine-javascript": "catalog:", 63 | "@shikijs/langs": "catalog:", 64 | "@shikijs/themes": "catalog:", 65 | "@types/node": "catalog:", 66 | "@types/react": "catalog:", 67 | "bumpp": "catalog:", 68 | "eslint": "catalog:", 69 | "eslint-plugin-react-hooks": "catalog:", 70 | "lint-staged": "catalog:", 71 | "pnpm": "catalog:", 72 | "react": "catalog:", 73 | "shiki": "catalog:", 74 | "simple-git-hooks": "catalog:", 75 | "taze": "catalog:", 76 | "tsx": "catalog:", 77 | "typescript": "catalog:", 78 | "unbuild": "catalog:", 79 | "vite": "catalog:", 80 | "vitest": "catalog:", 81 | "vue-tsc": "catalog:" 82 | }, 83 | "simple-git-hooks": { 84 | "pre-commit": "pnpm lint-staged" 85 | }, 86 | "lint-staged": { 87 | "*": "eslint --fix" 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/tokenizer.ts: -------------------------------------------------------------------------------- 1 | import type { GrammarState, ThemedToken } from '@shikijs/core' 2 | import type { ShikiStreamTokenizerEnqueueResult, ShikiStreamTokenizerOptions } from './types' 3 | 4 | export class ShikiStreamTokenizer { 5 | public readonly options: ShikiStreamTokenizerOptions 6 | 7 | public tokensStable: ThemedToken[] = [] 8 | public tokensUnstable: ThemedToken[] = [] 9 | 10 | public lastUnstableCodeChunk: string = '' 11 | public lastStableGrammarState: GrammarState | undefined 12 | 13 | constructor( 14 | options: ShikiStreamTokenizerOptions, 15 | ) { 16 | this.options = options 17 | } 18 | 19 | /** 20 | * Enqueue a chunk of code to the buffer. 21 | */ 22 | async enqueue(chunk: string): Promise { 23 | const chunkLines = (this.lastUnstableCodeChunk + chunk).split('\n') 24 | 25 | const stable: ThemedToken[] = [] 26 | let unstable: ThemedToken[] = [] 27 | const recall = this.tokensUnstable.length 28 | 29 | chunkLines.forEach((line, i) => { 30 | const isLastLine = i === chunkLines.length - 1 31 | 32 | const result = this.options.highlighter.codeToTokens(line, { 33 | ...this.options, 34 | grammarState: this.lastStableGrammarState, 35 | }) 36 | const tokens = result.tokens[0] // only one line 37 | if (!isLastLine) 38 | tokens.push({ content: '\n', offset: 0 }) 39 | 40 | if (!isLastLine) { 41 | this.lastStableGrammarState = result.grammarState 42 | stable.push(...tokens) 43 | } 44 | else { 45 | unstable = tokens 46 | this.lastUnstableCodeChunk = line 47 | } 48 | }) 49 | 50 | this.tokensStable.push(...stable) 51 | this.tokensUnstable = unstable 52 | 53 | return { 54 | recall, 55 | stable, 56 | unstable, 57 | } 58 | } 59 | 60 | close(): { stable: ThemedToken[] } { 61 | const stable = this.tokensUnstable 62 | this.tokensUnstable = [] 63 | this.lastUnstableCodeChunk = '' 64 | this.lastStableGrammarState = undefined 65 | return { 66 | stable, 67 | } 68 | } 69 | 70 | clear(): void { 71 | this.tokensStable = [] 72 | this.tokensUnstable = [] 73 | this.lastUnstableCodeChunk = '' 74 | this.lastStableGrammarState = undefined 75 | } 76 | 77 | clone(): ShikiStreamTokenizer { 78 | const clone = new ShikiStreamTokenizer( 79 | this.options, 80 | ) 81 | clone.lastUnstableCodeChunk = this.lastUnstableCodeChunk 82 | clone.tokensUnstable = this.tokensUnstable 83 | clone.tokensStable = this.tokensStable 84 | clone.lastStableGrammarState = this.lastStableGrammarState 85 | return clone 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /test/output/slice-3.html: -------------------------------------------------------------------------------- 1 |
const count = ref(0)
 2 | </script>
 3 | 
 4 | <template>
 5 |   <button @click="count++">{{ count }}</button>
 6 | </template>
 7 | 
 8 | <style>
 9 | button {
10 |   color: red;
11 | }
12 | </style>
13 | 
-------------------------------------------------------------------------------- /test/output/merged.html: -------------------------------------------------------------------------------- 1 |

 2 | <script setup lang="ts">
 3 | import { ref } from 'vue'
 4 | 
 5 | const count = ref(0)
 6 | </script>
 7 | 
 8 | <template>
 9 |   <button @click="count++">{{ count }}</button>
10 | </template>
11 | 
12 | <style>
13 | button {
14 |   color: red;
15 | }
16 | </style>
17 | 
-------------------------------------------------------------------------------- /test/output/onepass.html: -------------------------------------------------------------------------------- 1 |

 2 | <script setup lang="ts">
 3 | import { ref } from 'vue'
 4 | 
 5 | const count = ref(0)
 6 | </script>
 7 | 
 8 | <template>
 9 |   <button @click="count++">{{ count }}</button>
10 | </template>
11 | 
12 | <style>
13 | button {
14 |   color: red;
15 | }
16 | </style>
17 | 
-------------------------------------------------------------------------------- /test/output/stream-1.html: -------------------------------------------------------------------------------- 1 |

 2 | <script setup lang="ts">
 3 | import { ref } from 'vue'
 4 | 
 5 | const count = ref(0)
 6 | </script>
 7 | 
 8 | <template>
 9 |   <button @click="count++">{{ count }}</button>
10 | </template>
11 | 
12 | <style>
13 | button {
14 |   color: red;
15 | }
16 | </style>
17 | 
-------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shiki-stream 2 | 3 | [![npm version][npm-version-src]][npm-version-href] 4 | [![npm downloads][npm-downloads-src]][npm-downloads-href] 5 | [![bundle][bundle-src]][bundle-href] 6 | [![JSDocs][jsdocs-src]][jsdocs-href] 7 | [![License][license-src]][license-href] 8 | 9 | Streaming highlighting with Shiki. Useful for highlighting text streams like LLM outputs. 10 | 11 | [Live Demo](https://shiki-stream.netlify.app/) 12 | 13 | ## Usage 14 | 15 | Create a transform stream with `CodeToTokenTransformStream` and `.pipeThrough` your text stream: 16 | 17 | ```ts 18 | import { createHighlighter, createJavaScriptRegexEngine } from 'shiki' 19 | import { CodeToTokenTransformStream } from 'shiki-stream' 20 | 21 | // Initialize the Shiki highlighter somewhere in your app 22 | const highlighter = await createHighlighter({ 23 | langs: [/* ... */], 24 | themes: [/* ... */], 25 | engine: createJavaScriptRegexEngine() 26 | }) 27 | 28 | // The ReadableStream you want to highlight 29 | const textStream = getTextStreamFromSomewhere() 30 | 31 | // Pipe the text stream through the token stream 32 | const tokensStream = textStream 33 | .pipeThrough(new CodeToTokenTransformStream({ 34 | highlighter, 35 | lang: 'javascript', 36 | theme: 'nord', 37 | allowRecalls: true, // see explanation below 38 | })) 39 | ``` 40 | 41 | #### `allowRecalls` 42 | 43 | Due fact that the highlighting might be changed based on the context of the code, the themed tokens might be changed as the stream goes on. Because the streams are one-directional, we introduce a special "recall" token to notify the receiver to discard the last tokens that has changed. 44 | 45 | By default, `CodeToTokenTransformStream` only returns stable tokens, no recalls. This also means the tokens are outputted less fine-grained, usually line-by-line. 46 | 47 | For stream consumers that can handle recalls (e.g. our Vue / React components), you can set `allowRecalls: true` to get more fine-grained tokens. 48 | 49 | Typically, recalls should be handled like: 50 | 51 | ```ts 52 | const receivedTokens: ThemedToken[] = [] 53 | 54 | tokensStream.pipeTo(new WritableStream({ 55 | async write(token) { 56 | if ('recall' in token) { 57 | // discard the last `token.recall` tokens 58 | receivedTokens.length -= token.recall 59 | } 60 | else { 61 | receivedTokens.push(token) 62 | } 63 | } 64 | })) 65 | ``` 66 | 67 | ### Consume the Token Stream 68 | 69 | #### Manually 70 | 71 | ```ts 72 | tokensStream.pipeTo(new WritableStream({ 73 | async write(token) { 74 | console.log(token) 75 | } 76 | })) 77 | ``` 78 | 79 | Or in Node.js 80 | 81 | ```ts 82 | for await (const token of tokensStream) { 83 | console.log(token) 84 | } 85 | ``` 86 | 87 | #### Vue 88 | 89 | ```vue 90 | 95 | 96 | 99 | ``` 100 | 101 | #### React 102 | 103 | ```tsx 104 | import { ShikiStreamRenderer } from 'shiki-stream/react' 105 | 106 | export function MyComponent() { 107 | // get the token stream 108 | return 109 | } 110 | ``` 111 | 112 | ## Cached Renderer 113 | 114 | This library also provides a simpfiled renderer API to render incrementally updated code string. 115 | 116 | > [!NOTE] 117 | > Experimental 118 | 119 | ### Vue 120 | 121 | ```vue 122 | 140 | 141 | 149 | ``` 150 | 151 | ### React 152 | 153 | TODO: 154 | 155 | ## Sponsors 156 | 157 |

158 | 159 | 160 | 161 |

162 | 163 | ## License 164 | 165 | [MIT](./LICENSE) License © [Anthony Fu](https://github.com/antfu) 166 | 167 | 168 | 169 | [npm-version-src]: https://img.shields.io/npm/v/shiki-stream?style=flat&colorA=080f12&colorB=1fa669 170 | [npm-version-href]: https://npmjs.com/package/shiki-stream 171 | [npm-downloads-src]: https://img.shields.io/npm/dm/shiki-stream?style=flat&colorA=080f12&colorB=1fa669 172 | [npm-downloads-href]: https://npmjs.com/package/shiki-stream 173 | [bundle-src]: https://img.shields.io/bundlephobia/minzip/shiki-stream?style=flat&colorA=080f12&colorB=1fa669&label=minzip 174 | [bundle-href]: https://bundlephobia.com/result?p=shiki-stream 175 | [license-src]: https://img.shields.io/github/license/antfu/shiki-stream.svg?style=flat&colorA=080f12&colorB=1fa669 176 | [license-href]: https://github.com/antfu/shiki-stream/blob/main/LICENSE 177 | [jsdocs-src]: https://img.shields.io/badge/jsdocs-reference-080f12?style=flat&colorA=080f12&colorB=1fa669 178 | [jsdocs-href]: https://www.jsdocs.io/package/shiki-stream 179 | -------------------------------------------------------------------------------- /playground/src/Playground.vue: -------------------------------------------------------------------------------- 1 | 199 | 200 |