├── src ├── react │ ├── index.ts │ ├── utils.ts │ └── renderer.ts ├── vue │ ├── index.ts │ ├── cached.ts │ └── renderer.ts ├── index.ts ├── types.ts ├── stream.ts └── tokenizer.ts ├── .github ├── FUNDING.yml └── workflows │ ├── release.yml │ └── ci.yml ├── .npmrc ├── CONTRIBUTING.md ├── .gitignore ├── netlify.toml ├── taze.config.ts ├── playground ├── src │ ├── main.ts │ ├── App.vue │ ├── renderer │ │ ├── types.ts │ │ ├── vue.ts │ │ └── react.tsx │ ├── fixture.ts │ └── Playground.vue ├── tsconfig.json ├── index.html ├── package.json └── vite.config.ts ├── eslint.config.js ├── test ├── output │ ├── slice-0.html │ ├── slice-1.html │ ├── slice-2.html │ ├── slice-3.html │ ├── merged.html │ ├── onepass.html │ └── stream-1.html ├── utils.ts ├── stream.test.ts └── tokenizer.test.ts ├── tsconfig.json ├── LICENSE.md ├── .vscode └── settings.json ├── pnpm-workspace.yaml ├── package.json └── README.md /src/react/index.ts: -------------------------------------------------------------------------------- 1 | export * from './renderer' 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [antfu] 2 | opencollective: antfu 3 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | ignore-workspace-root-check=true 2 | shell-emulator=true 3 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Please refer to https://github.com/antfu/contribute 2 | -------------------------------------------------------------------------------- /src/vue/index.ts: -------------------------------------------------------------------------------- 1 | export * from './cached' 2 | export * from './renderer' 3 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './stream' 2 | export * from './tokenizer' 3 | export * from './types' 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .DS_Store 3 | .idea 4 | *.log 5 | *.tgz 6 | coverage 7 | dist 8 | lib-cov 9 | logs 10 | node_modules 11 | temp 12 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | publish = "playground/dist" 3 | command = "pnpm run play:build" 4 | 5 | [build.environment] 6 | NODE_VERSION = "23" 7 | -------------------------------------------------------------------------------- /taze.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'taze' 2 | 3 | export default defineConfig({ 4 | exclude: [ 5 | '@shikijs/core', 6 | ], 7 | }) 8 | -------------------------------------------------------------------------------- /playground/src/main.ts: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import App from './App.vue' 3 | 4 | import '@unocss/reset/tailwind.css' 5 | import 'uno.css' 6 | 7 | createApp(App).mount('#app') 8 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | import antfu from '@antfu/eslint-config' 3 | 4 | export default antfu( 5 | { 6 | type: 'lib', 7 | react: true, 8 | pnpm: true, 9 | }, 10 | ) 11 | -------------------------------------------------------------------------------- /test/output/slice-0.html: -------------------------------------------------------------------------------- 1 |
2 | <script setup-------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | release: 10 | uses: sxzz/workflows/.github/workflows/release.yml@v1 11 | with: 12 | publish: true 13 | permissions: 14 | contents: write 15 | id-token: write 16 | -------------------------------------------------------------------------------- /playground/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "jsx": "preserve", 5 | "lib": ["ESNext", "DOM"], 6 | "module": "ESNext", 7 | "moduleResolution": "Bundler", 8 | "resolveJsonModule": true, 9 | "strict": true, 10 | "strictNullChecks": true, 11 | "esModuleInterop": true, 12 | "skipDefaultLibCheck": true, 13 | "skipLibCheck": true 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /playground/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |
<script setup lang="ts"> 2 | impo-------------------------------------------------------------------------------- /playground/src/fixture.ts: -------------------------------------------------------------------------------- 1 | export const vueBefore = ` 15 | 16 | 17 |
{{ count }} * 2 = {{ doubled }}
18 | 19 | 20 | ` 26 | 27 | export const vueAfter = ` 33 | 34 | 35 |{{ count }} = {{ doubled / 2 }}
36 | 37 | 38 | ` 44 | -------------------------------------------------------------------------------- /test/output/slice-2.html: -------------------------------------------------------------------------------- 1 |import { ref } from 'vue' 2 | 3 | const-------------------------------------------------------------------------------- /src/react/utils.ts: -------------------------------------------------------------------------------- 1 | // source: https://github.com/sanity-io/use-effect-event/blob/main/src/useEffectEvent.ts 2 | import { useCallback, useInsertionEffect, useRef } from 'react' 3 | 4 | /** 5 | * This is a ponyfill of the upcoming `useEffectEvent` hook that'll arrive in React 19. 6 | * https://19.react.dev/learn/separating-events-from-effects#declaring-an-effect-event 7 | * To learn more about the ponyfill itself, see: https://blog.bitsrc.io/a-look-inside-the-useevent-polyfill-from-the-new-react-docs-d1c4739e8072 8 | * @public 9 | */ 10 | export function useEffectEvent< 11 | const T extends ( 12 | ...args: 13 | any[] 14 | ) => void, 15 | >(fn: T): T { 16 | const ref = useRef
${tokens.map(tokenToHtml).join('')}`
26 | }
27 |
28 | export function escapeHtml(html: string): string {
29 | return html.replace(//g, '>')
30 | }
31 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2025-PRESENT Anthony Fu const count = ref(0) 2 | </script> 3 | 4 | <template> 5 | <button @click="count++">{{ count }}</button> 6 | </template> 7 | 8 | <style> 9 | button { 10 | color: red; 11 | } 12 | </style> 13 |-------------------------------------------------------------------------------- /test/output/merged.html: -------------------------------------------------------------------------------- 1 |
2 | <script setup lang="ts"> 3 | import { ref } from 'vue' 4 | 5 | const count = ref(0) 6 | </script> 7 | 8 | <template> 9 | <button @click="count++">{{ count }}</button> 10 | </template> 11 | 12 | <style> 13 | button { 14 | color: red; 15 | } 16 | </style> 17 |-------------------------------------------------------------------------------- /test/output/onepass.html: -------------------------------------------------------------------------------- 1 |
2 | <script setup lang="ts"> 3 | import { ref } from 'vue' 4 | 5 | const count = ref(0) 6 | </script> 7 | 8 | <template> 9 | <button @click="count++">{{ count }}</button> 10 | </template> 11 | 12 | <style> 13 | button { 14 | color: red; 15 | } 16 | </style> 17 |-------------------------------------------------------------------------------- /test/output/stream-1.html: -------------------------------------------------------------------------------- 1 |
2 | <script setup lang="ts"> 3 | import { ref } from 'vue' 4 | 5 | const count = ref(0) 6 | </script> 7 | 8 | <template> 9 | <button @click="count++">{{ count }}</button> 10 | </template> 11 | 12 | <style> 13 | button { 14 | color: red; 15 | } 16 | </style> 17 |-------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shiki-stream 2 | 3 | [![npm version][npm-version-src]][npm-version-href] 4 | [![npm downloads][npm-downloads-src]][npm-downloads-href] 5 | [![bundle][bundle-src]][bundle-href] 6 | [![JSDocs][jsdocs-src]][jsdocs-href] 7 | [![License][license-src]][license-href] 8 | 9 | Streaming highlighting with Shiki. Useful for highlighting text streams like LLM outputs. 10 | 11 | [Live Demo](https://shiki-stream.netlify.app/) 12 | 13 | ## Usage 14 | 15 | Create a transform stream with `CodeToTokenTransformStream` and `.pipeThrough` your text stream: 16 | 17 | ```ts 18 | import { createHighlighter, createJavaScriptRegexEngine } from 'shiki' 19 | import { CodeToTokenTransformStream } from 'shiki-stream' 20 | 21 | // Initialize the Shiki highlighter somewhere in your app 22 | const highlighter = await createHighlighter({ 23 | langs: [/* ... */], 24 | themes: [/* ... */], 25 | engine: createJavaScriptRegexEngine() 26 | }) 27 | 28 | // The ReadableStream
158 |
159 |
160 |
161 |