├── .gitattributes
├── llm-coder
├── src
    ├── noop.js
    ├── Header.js
    ├── utils
    │   ├── crypto.js
    │   ├── csp.js
    │   ├── logging.js
    │   └── math.js
    ├── constants
    │   ├── global.js
    │   └── chat.js
    ├── Home.js
    ├── root.js
    ├── App.js
    ├── LICENSE
    ├── sw.js
    ├── index.html
    ├── hooks
    │   ├── useLLMVision.js
    │   ├── useLLMHtmlGeneration.js
    │   ├── useSpeech.js
    │   ├── useLLMVision.jsold.js
    │   └── useLLMGeneration.js
    ├── playground
    │   └── model_load_test.ts
    └── Chat.js
├── public
    ├── site.gif
    ├── site.webm
    ├── site.webp
    └── site-50.webp
├── Makefile
├── site.apng
├── README.md
├── .github
    └── workflows
    │   └── static.yml
├── .gitignore
└── LICENSE


/.gitattributes:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/llm-coder:
--------------------------------------------------------------------------------
1 | ./src


--------------------------------------------------------------------------------
/src/noop.js:
--------------------------------------------------------------------------------
1 | export default {};
2 | 


--------------------------------------------------------------------------------
/public/site.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pdufour/llm-coder/HEAD/public/site.gif


--------------------------------------------------------------------------------
/public/site.webm:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pdufour/llm-coder/HEAD/public/site.webm


--------------------------------------------------------------------------------
/public/site.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pdufour/llm-coder/HEAD/public/site.webp


--------------------------------------------------------------------------------
/public/site-50.webp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pdufour/llm-coder/HEAD/public/site-50.webp


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | run:
2 | 	static-web-server -p 3000 --cors-allow-origins "*" --cors-allow-headers "*" --cors-expose-headers "*" -d . -e false -z
3 | 


--------------------------------------------------------------------------------
/site.apng:
--------------------------------------------------------------------------------
1 | version https://git-lfs.github.com/spec/v1
2 | oid sha256:fd72f5fddc54105afd8b3bdde7d5821ff38e3e89ef92714f267544b583352cd2
3 | size 39922469
4 | 


--------------------------------------------------------------------------------
/src/Header.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | 
3 | const h = React.createElement;
4 | 
5 | export const Header = () => h('div', { className: 'w-full overflow-hidden' });
6 | 


--------------------------------------------------------------------------------
/src/utils/crypto.js:
--------------------------------------------------------------------------------
1 | export const encodeSHA256 = async (script) =>
2 |   `${btoa(
3 |     String.fromCharCode(
4 |       ...new Uint8Array(
5 |         await crypto.subtle.digest("SHA-256", new TextEncoder().encode(script))
6 |       )
7 |     )
8 |   )}`;
9 | 


--------------------------------------------------------------------------------
/src/constants/global.js:
--------------------------------------------------------------------------------
 1 | export const SESSION_OPTIONS = {
 2 |   executionProviders: ['webgpu'],
 3 |   graphOptimizationLevel: 'all',
 4 |   executionMode: 'sequential',
 5 |   intraOpNumThreads: 4,
 6 |   interOpNumThreads: 4,
 7 |   logSeverityLevel: 1,
 8 |   enableMemoryPattern: true,
 9 | }
10 | 


--------------------------------------------------------------------------------
/src/Home.js:
--------------------------------------------------------------------------------
 1 | import React from "react";
 2 | import { Chat } from "./Chat.js";
 3 | import { Header } from "./Header.js";
 4 | 
 5 | const h = React.createElement;
 6 | 
 7 | export const Home = () => h(React.Fragment, null,
 8 |   h(Header),
 9 |   h('div', { className: 'w-full' },
10 |     h(Chat)
11 |   )
12 | );
13 | 


--------------------------------------------------------------------------------
/src/root.js:
--------------------------------------------------------------------------------
 1 | import React from "react";
 2 | import ReactDOM from "react-dom/client";
 3 | 
 4 | import { App } from "./App.js";
 5 | 
 6 | // await navigator.serviceWorker.ready;
 7 | const rootElement = document.getElementById("root");
 8 | const root = ReactDOM.createRoot(rootElement);
 9 | root.render(React.createElement(App));
10 | 


--------------------------------------------------------------------------------
/src/App.js:
--------------------------------------------------------------------------------
 1 | import { env } from "@huggingface/transformers";
 2 | import React from "react";
 3 | import { Home } from "./Home.js";
 4 | 
 5 | const h = React.createElement;
 6 | 
 7 | async function init() {
 8 |   const wasmPaths = new URL(
 9 |     ".",
10 |     await import.meta.resolve("#onnxruntime-webgpu")
11 |   ).toString();
12 |   env.backends.onnx.wasm.wasmPaths = wasmPaths;
13 | }
14 | 
15 | export const App = () => {
16 |   return h(
17 |     "div",
18 |     { className: "min-h-screen min-w-full" },
19 |     h(
20 |       "div",
21 |       { className: "mx-auto self-center flex flex-col items-center" },
22 |       h(
23 |         "div",
24 |         {
25 |           className:
26 |             "flex flex-col w-full self-center items-center justify-center",
27 |         },
28 |         h(Home)
29 |       )
30 |     )
31 |   );
32 | }
33 | 
34 | init();
35 | 


--------------------------------------------------------------------------------
/src/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2024 Paul Dufour
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # llm-coder
 2 | llm-coder is an open source privacy-first AI website-builder powered by your browser. No API keys required. No server required. See the live demo at [https://pdufour.github.io/llm-coder/](https://pdufour.github.io/llm-coder/).
 3 | 
 4 | ![site](https://github.com/pdufour/llm-coder/raw/main/public/site-50.webp)
 5 | 
 6 | **Privacy:** All AI models run locally in your browser. Your data never leaves your device - no servers involved.
 7 | 
 8 | It supports the following features:
 9 | 1. Describe a site you want - the tool will generate the HTML and CSS for you
10 | 2. Drag and drop an image - the tool will generate the HTML and CSS for you
11 | 
12 | ## Tech Stack
13 | - [onnxruntime-web](https://onnxruntime.ai/docs/tutorials/web/)
14 | 
15 | ## Support
16 | - Text generation (via Qwen2)
17 | - Vision parsing (via Qwen2-VL)
18 | - Audio transcription (Via Moonshine-Base)
19 | 
20 | ## Design Principles
21 | - No backends
22 | - No JS build steps unless the compilation happens directly in the browser.
23 | 
24 | ## Setup
25 | 
26 | 1. Clone the repository
27 | 2. Install [static-web-server](https://static-web-server.net/download-and-install/#macos)
28 | 2. Run `make run`
29 | 3. Visit http://localhost:3000/llm-coder in the browser
30 | 


--------------------------------------------------------------------------------
/src/utils/csp.js:
--------------------------------------------------------------------------------
 1 | import { encodeSHA256 } from "./crypto.js";
 2 | 
 3 | export class CSPBuilder {
 4 |   constructor() {
 5 |     this.policy = {};
 6 |   }
 7 | 
 8 |   addDirective(directive, source) {
 9 |     if (!this.policy[directive]) {
10 |       this.policy[directive] = new Set();
11 |     }
12 |     this.policy[directive].add(source);
13 |     return this; // Enable chaining
14 |   }
15 | 
16 |   async addScriptHash(script) {
17 |     const hash = await encodeSHA256(script);
18 |     this.addDirective("script-src", `'sha256-${hash}'`);
19 |     return this; // Enable chaining
20 |   }
21 | 
22 |   addScriptNonce() {
23 |     if (!this.nonce) {
24 |       this.nonce = this.generateNonce();
25 |     }
26 |     this.addDirective("script-src", `'nonce-${this.nonce}'`);
27 |     return this; // Enable chaining
28 |   }
29 | 
30 |   generateNonce() {
31 |     const array = new Uint8Array(16); // 128-bit nonce
32 |     crypto.getRandomValues(array);
33 |     return btoa(String.fromCharCode(...array));
34 |   }
35 | 
36 |   async build() {
37 |     const cspString = Object.entries(this.policy)
38 |       .map(
39 |         ([directive, sources]) =>
40 |           `${directive} ${Array.from(sources).join(" ")}`
41 |       )
42 |       .join("; ");
43 |     return { cspString, nonce: this.nonce }; // Return both CSP and nonce for use in templates
44 |   }
45 | }
46 | 


--------------------------------------------------------------------------------
/.github/workflows/static.yml:
--------------------------------------------------------------------------------
 1 | # Simple workflow for deploying static content to GitHub Pages
 2 | name: Deploy static content to Pages
 3 | 
 4 | on:
 5 |   # Runs on pushes targeting the default branch
 6 |   push:
 7 |     branches: ["main"]
 8 | 
 9 |   # Allows you to run this workflow manually from the Actions tab
10 |   workflow_dispatch:
11 | 
12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
13 | permissions:
14 |   contents: read
15 |   pages: write
16 |   id-token: write
17 | 
18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
20 | concurrency:
21 |   group: "pages"
22 |   cancel-in-progress: false
23 | 
24 | jobs:
25 |   # Single deploy job since we're just deploying
26 |   deploy:
27 |     environment:
28 |       name: github-pages
29 |       url: ${{ steps.deployment.outputs.page_url }}
30 |     runs-on: ubuntu-latest
31 |     steps:
32 |       - name: Checkout
33 |         uses: actions/checkout@v4
34 |       - name: Setup Pages
35 |         uses: actions/configure-pages@v5
36 |       - name: Upload artifact
37 |         uses: actions/upload-pages-artifact@v3
38 |         with:
39 |           # Upload entire repository
40 |           path: 'src'
41 |       - name: Deploy to GitHub Pages
42 |         id: deployment
43 |         uses: actions/deploy-pages@v4
44 | 


--------------------------------------------------------------------------------
/src/sw.js:
--------------------------------------------------------------------------------
 1 | import { registerRoute } from "https://cdn.jsdelivr.net/npm/workbox-routing/+esm";
 2 | import { NetworkFirst } from "https://cdn.jsdelivr.net/npm/workbox-strategies/+esm";
 3 | 
 4 | import initSwc, {
 5 |   transformSync,
 6 | } from "https://cdn.jsdelivr.net/npm/@swc/wasm-web/+esm";
 7 | import "https://cdn.jsdelivr.net/npm/workbox-sw/+esm";
 8 | 
 9 | workbox.setConfig({
10 |   debug: true,
11 | });
12 | 
13 | self.addEventListener("install", function (event) {
14 |   event.waitUntil(self.skipWaiting());
15 | });
16 | 
17 | self.addEventListener("activate", function (event) {
18 |   event.waitUntil(self.clients.claim());
19 | });
20 | 
21 | let swcInitialized = false;
22 | 
23 | registerRoute(
24 |   ({ url }) => url.pathname.match(/\.(tsx?|jsx)$/),
25 |   async ({ request }) => {
26 |     const response = await fetch(request);
27 | 
28 |     if (!swcInitialized) {
29 |       await initSwc();
30 |       swcInitialized = true;
31 |     }
32 | 
33 |     const source = await response.text();
34 | 
35 |     const { code } = transformSync(source, {
36 |       jsc: {
37 |       "externalHelpers": true,
38 |         loose: true,
39 |         parser: { syntax: "typescript", tsx: true },
40 |         target: "es2022",
41 |         transform: {
42 |           react: {
43 |             development: true,
44 |           },
45 |         },
46 |       },
47 |       module: {
48 |         type: "es6",
49 |         interop: "node"  // Or "swc" or "none"
50 |       },
51 |     });
52 | 
53 |     return new Response(code, {
54 |       headers: {
55 |         "Content-Type": "application/javascript",
56 |         "Cache-Control": "no-cache",
57 |       },
58 |     });
59 |   }
60 | );
61 | 


--------------------------------------------------------------------------------
/src/utils/logging.js:
--------------------------------------------------------------------------------
 1 | import { Float16Array, getFloat16, setFloat16 } from "@petamoriken/float16";
 2 | 
 3 | class Logger {
 4 |   static indentLevel = 0
 5 | 
 6 |   static group(...args) {
 7 |     logger.indentLevel += 1
 8 |     console.group(...args)
 9 |   }
10 | 
11 |   static groupCollapsed(...args) {
12 |     logger.indentLevel += 1
13 |     console.groupCollapsed(...args)
14 |   }
15 | 
16 |   static groupEnd() {
17 |     if (logger.indentLevel === 0) {
18 |       console.error("logGroupEnd() called too many times")
19 |       return
20 |     }
21 |     logger.indentLevel -= 1
22 |     console.groupEnd()
23 |   }
24 | 
25 |   static tensor(name, tensor) {
26 |     logger.groupCollapsed(`[NUMPY] ${name}:`)
27 |     logger.debug(`Shape: (${tensor.dims.join(", ")})`)
28 |     logger.debug(`Dtype: ${tensor.type}`)
29 | 
30 |     let data
31 |     if (tensor.data) {
32 |       if (tensor.type === "float16") {
33 |         const view = new DataView(
34 |           tensor.data.buffer,
35 |           tensor.data.byteOffset,
36 |           tensor.data.byteLength
37 |         )
38 |         const littleEndian = true // Adjust if necessary
39 | 
40 |         const numElements = tensor.data.length
41 |         data = []
42 |         for (let i = 0; i < numElements; i++) {
43 |           const offset = i * 2
44 |           const value = getFloat16(view, offset, littleEndian)
45 |           data.push(value)
46 |         }
47 |       } else {
48 |         data = Array.from(tensor.data).map(val => {
49 |           return val.toString()
50 |         })
51 |       }
52 | 
53 |       if (tensor.dims[0] === 1) {
54 |         logger.debug(`Values: [${data[0]}]`)
55 |       }
56 | 
57 |       logger.debug(`First few values: [${data.slice(0, 5).join(", ")}]`)
58 |       logger.debug(`Last few values: [${data.slice(-5).join(", ")}]`)
59 |     }
60 |     logger.groupEnd()
61 |   }
62 | }
63 | 
64 | export const logger = new Proxy(Logger, {
65 |   get(target, prop) {
66 |     // If the method exists on Logger, use it
67 |     if (prop in target) {
68 |       return target[prop]
69 |     }
70 | 
71 |     // If the method doesn’t exist on Logger, proxy it to console
72 |     if (typeof console[prop] === "function") {
73 |       return console[prop].bind(console)
74 |     }
75 | 
76 |     // If neither Logger nor console has the method, return undefined
77 |     return undefined
78 |   }
79 | })
80 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Logs
  2 | logs
  3 | *.log
  4 | npm-debug.log*
  5 | yarn-debug.log*
  6 | yarn-error.log*
  7 | lerna-debug.log*
  8 | .pnpm-debug.log*
  9 | 
 10 | # Diagnostic reports (https://nodejs.org/api/report.html)
 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 12 | 
 13 | # Runtime data
 14 | pids
 15 | *.pid
 16 | *.seed
 17 | *.pid.lock
 18 | 
 19 | # Directory for instrumented libs generated by jscoverage/JSCover
 20 | lib-cov
 21 | 
 22 | # Coverage directory used by tools like istanbul
 23 | coverage
 24 | *.lcov
 25 | 
 26 | # nyc test coverage
 27 | .nyc_output
 28 | 
 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
 30 | .grunt
 31 | 
 32 | # Bower dependency directory (https://bower.io/)
 33 | bower_components
 34 | 
 35 | # node-waf configuration
 36 | .lock-wscript
 37 | 
 38 | # Compiled binary addons (https://nodejs.org/api/addons.html)
 39 | build/Release
 40 | 
 41 | # Dependency directories
 42 | node_modules/
 43 | jspm_packages/
 44 | 
 45 | # Snowpack dependency directory (https://snowpack.dev/)
 46 | web_modules/
 47 | 
 48 | # TypeScript cache
 49 | *.tsbuildinfo
 50 | 
 51 | # Optional npm cache directory
 52 | .npm
 53 | 
 54 | # Optional eslint cache
 55 | .eslintcache
 56 | 
 57 | # Optional stylelint cache
 58 | .stylelintcache
 59 | 
 60 | # Microbundle cache
 61 | .rpt2_cache/
 62 | .rts2_cache_cjs/
 63 | .rts2_cache_es/
 64 | .rts2_cache_umd/
 65 | 
 66 | # Optional REPL history
 67 | .node_repl_history
 68 | 
 69 | # Output of 'npm pack'
 70 | *.tgz
 71 | 
 72 | # Yarn Integrity file
 73 | .yarn-integrity
 74 | 
 75 | # dotenv environment variable files
 76 | .env
 77 | .env.development.local
 78 | .env.test.local
 79 | .env.production.local
 80 | .env.local
 81 | 
 82 | # parcel-bundler cache (https://parceljs.org/)
 83 | .cache
 84 | .parcel-cache
 85 | 
 86 | # Next.js build output
 87 | .next
 88 | out
 89 | 
 90 | # Nuxt.js build / generate output
 91 | .nuxt
 92 | dist
 93 | 
 94 | # Gatsby files
 95 | .cache/
 96 | # Comment in the public line in if your project uses Gatsby and not Next.js
 97 | # https://nextjs.org/blog/next-9-1#public-directory-support
 98 | # public
 99 | 
100 | # vuepress build output
101 | .vuepress/dist
102 | 
103 | # vuepress v2.x temp and cache directory
104 | .temp
105 | .cache
106 | 
107 | # Docusaurus cache and generated files
108 | .docusaurus
109 | 
110 | # Serverless directories
111 | .serverless/
112 | 
113 | # FuseBox cache
114 | .fusebox/
115 | 
116 | # DynamoDB Local files
117 | .dynamodb/
118 | 
119 | # TernJS port file
120 | .tern-port
121 | 
122 | # Stores VSCode versions used for testing VSCode extensions
123 | .vscode-test
124 | 
125 | # yarn v2
126 | .yarn/cache
127 | .yarn/unplugged
128 | .yarn/build-state.yml
129 | .yarn/install-state.gz
130 | .pnp.*
131 | 
132 | 
133 | .DS_STORE
134 | 


--------------------------------------------------------------------------------
/src/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 |   <head>
 4 |     <meta charset="UTF-8" />
 5 |     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 6 |     <script src="https://cdn.tailwindcss.com"></script>
 7 |     <title>Voice Code Interface</title>
 8 |     <base href="/llm-coder/" />
 9 |   </head>
10 |   <body>
11 |     <div id="root"></div>
12 |     <script type="importmap">
13 |       {
14 |         "imports": {
15 |           "react": "https://esm.sh/react@18.3.1?dev",
16 |           "react/jsx-runtime": "https://esm.sh/react@18.3.1/jsx-runtime?dev",
17 |           "react-dom/client": "https://esm.sh/react-dom@18.3.1/client?dev",
18 |           "lucide-react": "https://esm.run/lucide-react@0.460.0",
19 |           "@huggingface/transformers": "https://cdn.jsdelivr.net/npm/@huggingface/transformers",
20 |           "@huggingface/transformers/utils/hub.js": "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.2/src/utils/hub.js",
21 |           "@huggingface/transformers/src/generation/configuration_utils.js": "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.2/src/src/generation/configuration_utils.js",
22 |           "sharp": "./noop.js",
23 |           "fs": "./noop.js",
24 |           "path": "./noop.js",
25 |           "url": "./noop.js",
26 |           "onnxruntime-node": "/noop.js",
27 |           "@huggingface/jinja": "https://esm.run/@huggingface/jinja",
28 |           "onnxruntime-common": "https://esm.run/onnxruntime-common",
29 |           "#onnxruntime-webgpu": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.20.0/dist/ort.webgpu.mjs",
30 |           "onnxruntime-web/webgpu": "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.20.0/dist/ort.webgpu.mjs",
31 |           "@tensorflow/tfjs": "https://esm.run/@tensorflow/tfjs",
32 |           "@mlc-ai/web-llm": "https://esm.run/@mlc-ai/web-llm",
33 |           "@petamoriken/float16": "https://esm.run/@petamoriken/float16",
34 |           "loglevel": "https://esm.run/loglevel"
35 |         }
36 |       }
37 |     </script>
38 |     <!--<script type="module">
39 |       import { Workbox } from "https://cdn.jsdelivr.net/npm/workbox-window/+esm";
40 | 
41 |       const wb = new Workbox("/sw.js", { type: "module" });
42 | 
43 |       let reg = await navigator.serviceWorker.getRegistration();
44 |       if (reg?.active && !navigator.serviceWorker.controller) {
45 |         // https://github.com/mswjs/msw/issues/98#issuecomment-612118211
46 |         const registrations = await navigator.serviceWorker.getRegistrations();
47 |         await Promise.all(registrations.map((r) => r.unregister()));
48 |         reg = null;
49 |       }
50 | 
51 |       const activationPromise = new Promise((resolve) => {
52 |         if (reg?.active) {
53 |           resolve();
54 |         } else {
55 |           wb.addEventListener(
56 |             "activated",
57 |             async (event) => {
58 |               resolve();
59 |             },
60 |             { once: true }
61 |           );
62 |         }
63 |       });
64 | 
65 |       await wb.register();
66 |       await activationPromise;
67 |       await import("./root.tsx");
68 |     </script>-->
69 |     <script type="module">
70 |       await import("./root.js");
71 |     </script>
72 |   </body>
73 | </html>
74 | 


--------------------------------------------------------------------------------
/src/constants/chat.js:
--------------------------------------------------------------------------------
  1 | import { CSPBuilder } from "../utils/csp.js";
  2 | 
  3 | export const LLM_VISION_MODEL_CONFIG = {
  4 |   modelConfig: {
  5 |     qwen2vl: {
  6 |       modelId: "pdufour/Qwen2-VL-2B-Instruct-ONNX-Q4-F16",
  7 |       generation: {
  8 |         baseModelId: "Qwen/Qwen2-VL-2B-Instruct",
  9 |         max_single_chat_length: 256,
 10 |         max_seq_length: 1024,
 11 |       },
 12 |     }
 13 |   },
 14 | };
 15 | 
 16 | export const LLM_HTML_MODEL_CONFIG = {
 17 |   modelConfig: {
 18 |     webllm: {
 19 |       modelId: "Qwen2.5-Coder-1.5B-Instruct-q4f16_1-MLC",
 20 |       generation: {
 21 |         max_tokens: 500,
 22 |         temperature: 0.3,
 23 |         top_p: 0.9,
 24 |       },
 25 |       warmup: {
 26 |         max_tokens: 1,
 27 |       }
 28 |     },
 29 |     huggingface: {
 30 |       modelId: "Qwen/Qwen2.5-Coder-1.5B-Instruct",
 31 |       options: {
 32 |         device: "webgpu",
 33 |         dtype: "q4f16",
 34 |       },
 35 |       generation: {
 36 |         max_new_tokens: 500,
 37 |         temperature: 0.3,
 38 |         top_p: 0.9,
 39 |         do_sample: true,
 40 |       },
 41 |       warmup: {
 42 |         max_new_tokens: 1,
 43 |       }
 44 |     },
 45 |   },
 46 |   backend: "webllm",
 47 | };
 48 | 
 49 | const { cspString: IFRAME_CSP, nonce: IFRAME_CSP_NONCE } =
 50 |   await new CSPBuilder()
 51 |     .addDirective("default-src", "'none'")
 52 |     .addDirective("img-src", "'none'")
 53 |     .addDirective("style-src", "'unsafe-inline'")
 54 |     .addDirective("script-src", "https://cdn.tailwindcss.com")
 55 |     .addScriptNonce()
 56 |     .build();
 57 | 
 58 | export const IFRAME_POSTMESSAGE_SCRIPT = /*js*/ `
 59 | window.addEventListener('message', function(event) {
 60 |   if (event.origin !== "${window.location.origin}") {
 61 |     return;
 62 |   }
 63 | 
 64 |   if (event.data.type === 'updateContent') {
 65 |     // Update the body content
 66 |     document.body.innerHTML = event.data.content;
 67 | 
 68 |     // Find and execute all script tags in the new content
 69 |     const scripts = document.body.querySelectorAll('script');
 70 |     scripts.forEach(script => {
 71 |       const newScript = document.createElement('script');
 72 |       if (script.src) {
 73 |         // If the script has a src attribute, copy it
 74 |         newScript.src = script.src;
 75 |       } else {
 76 |         // Otherwise, copy the inline script content
 77 |         newScript.textContent = script.textContent;
 78 |       }
 79 |       document.body.appendChild(newScript);
 80 |       // Remove the original script tag
 81 |       script.remove();
 82 |     });
 83 |   }
 84 | });
 85 | `;
 86 | 
 87 | export const IFRAME_TAILWIND_URL = `https://cdn.tailwindcss.com`;
 88 | 
 89 | export const IFRAME_TAILWIND_CONFIG_SCRIPT = /*js*/ `;
 90 | tailwind.config = {
 91 |   theme: {
 92 |     extend: {
 93 |       colors: {
 94 |         primary: "#3B82F6",
 95 |         secondary: "#10B981",
 96 |       },
 97 |     },
 98 |   },
 99 | };
100 | `;
101 | 
102 | export const IFRAME_TEMPLATE = `<!DOCTYPE html>
103 | <html lang="en">
104 | <head>
105 |     <meta charset="UTF-8">
106 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
107 |     <meta http-equiv="Content-Security-Policy" content="${IFRAME_CSP}">
108 |     <script src="${IFRAME_TAILWIND_URL}" nonce="${IFRAME_CSP_NONCE}"></script>
109 |     <style type="text/tailwindcss">
110 |       @layer utilities {
111 |         .content-auto {
112 |           content-visibility: auto;
113 |         }
114 |       }
115 |     </style>
116 |     <script nonce="${IFRAME_CSP_NONCE}">
117 |       ${IFRAME_TAILWIND_CONFIG_SCRIPT}
118 |     </script>
119 | </head>
120 | <body class="bg-gray-50">
121 |   <script nonce="${IFRAME_CSP_NONCE}">${IFRAME_POSTMESSAGE_SCRIPT}</script>
122 | </body>
123 | </html>`;
124 | 


--------------------------------------------------------------------------------
/src/hooks/useLLMVision.js:
--------------------------------------------------------------------------------
  1 | import React from "react";
  2 | import { useLLMGeneration } from "./useLLMGeneration.js";
  3 | export const SYSTEM_PROMPT = `You are a specialized HTML generator that creates complete, production-ready HTML using ONLY Tailwind CSS classes.
  4 | 
  5 | IMPORTANT:
  6 | - Use EXCLUSIVELY Tailwind CSS classes for ALL styling - NO style attributes or style tags
  7 | - Every visual styling must be done through Tailwind classes
  8 | - Never use inline styles, CSS, or style tags
  9 | - If you need a style that seems custom, use Tailwind's arbitrary value syntax like [w-123px]
 10 | - Always write out the full, actual HTML elements with real content
 11 | - Never use Lorem Ipsum - write realistic English content
 12 | - Never include code comments or explanations
 13 | - DO NOT wrap the output in markdown code blocks
 14 | 
 15 | STRICT STYLING RULES:
 16 | - ❌ NO <style> tags
 17 | - ❌ NO style="" attributes
 18 | - ❌ NO CSS classes that aren't Tailwind
 19 | - ✅ ONLY use Tailwind utility classes
 20 | - ✅ Use Tailwind's arbitrary value syntax for custom values
 21 | - ✅ Stack multiple Tailwind classes to achieve desired styling
 22 | 
 23 | Requirements:
 24 | - Generate ONLY the HTML content for inside the <body> tag
 25 | - EXCLUSIVELY use Tailwind CSS classes for ALL styling
 26 | - NO placeholders, NO comments, NO explanations
 27 | - NO server-side code or JavaScript
 28 | - Every element must be complete with real content
 29 | - Write out all actual HTML elements - never indicate "pending" or "todo" sections
 30 | - DO NOT wrap the output in markdown code blocks
 31 | 
 32 | Example of CORRECT styling:
 33 | <div class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
 34 | Instead of:
 35 | <div style="background: blue; color: white;">
 36 | 
 37 | Generate complete, functional HTML using ONLY Tailwind classes for styling.`;
 38 | 
 39 | function cleanHtmlCode(fullHtml) {
 40 |   // First remove markdown code blocks
 41 |   let cleaned = fullHtml
 42 |     .replace(/```html/gi, "")
 43 |     .replace(/```(\s+)?html/gi, "")
 44 |     .replace(/```(\n)?html/gi, "")
 45 |     .replace(/```/g, "")
 46 |     .trim();
 47 | 
 48 |   // If the content is already wrapped in our template, extract just the content
 49 |   const templateContentMatch = cleaned.match(
 50 |     /CONTENT_PLACEHOLDER">\s*([\s\S]*?)\s*<\/body>/i
 51 |   );
 52 |   if (templateContentMatch) {
 53 |     return templateContentMatch[1].trim();
 54 |   }
 55 | 
 56 |   // Try to extract content between body tags
 57 |   const bodyContent = cleaned.match(/<body[^>]*>\s*([\s\S]*?)\s*<\/body>/i);
 58 |   if (bodyContent) {
 59 |     // Further clean the body content of any nested body tags
 60 |     return bodyContent[1]
 61 |       .replace(/<\/?body[^>]*>/gi, "") // Remove any nested body tags
 62 |       .trim();
 63 |   }
 64 | 
 65 |   // If no body tags found, remove any DOCTYPE and html wrapper if present
 66 |   cleaned = cleaned
 67 |     .replace(/<!DOCTYPE[^>]*>/gi, "")
 68 |     .replace(/<\/?html[^>]*>/gi, "")
 69 |     .replace(/<\/?body[^>]*>/gi, "")
 70 |     .replace(/<head>[\s\S]*?<\/head>/gi, "")
 71 |     .trim();
 72 | 
 73 |   return cleaned;
 74 | }
 75 | 
 76 | 
 77 | export function useLLMVisionGeneration({
 78 |   modelConfig,
 79 |   backend = 'qwen2vl',
 80 |   systemPrompt = SYSTEM_PROMPT,
 81 | }) {
 82 |   const [generatedText, setgeneratedText] = React.useState("");
 83 |   const lastgeneratedText = React.useRef("");
 84 | 
 85 |   const { generate, isGenerating, error, partialText } = useLLMGeneration(
 86 |     modelConfig,
 87 |     systemPrompt,
 88 |     backend
 89 |   );
 90 | 
 91 |   React.useEffect(() => {
 92 |     if (partialText) {
 93 |       const cleaned = cleanHtmlCode(partialText);
 94 |       const processedCode = cleaned;
 95 |       setgeneratedText(processedCode);
 96 |       lastgeneratedText.current = cleaned; // Store just the body content
 97 |     }
 98 |   }, [partialText]);
 99 | 
100 |   const generateText = React.useCallback(
101 |     async (prompt) => {
102 |       const fullPrompt = lastgeneratedText.current
103 |         ? `Current HTML: \n${lastgeneratedText.current}\n\nRequest: ${prompt}`
104 |         : `Generate the HTML for: ${prompt}`;
105 | 
106 |       console.log({ fullPrompt });
107 |       await generate(fullPrompt);
108 |     },
109 |     [generate]
110 |   );
111 | 
112 |   return {
113 |     generateText,
114 |     isGenerating,
115 |     error,
116 |     generatedText,
117 |   };
118 | }
119 | 


--------------------------------------------------------------------------------
/src/hooks/useLLMHtmlGeneration.js:
--------------------------------------------------------------------------------
  1 | import React from "react";
  2 | import {
  3 |   useLLMGeneration,
  4 | } from "./useLLMGeneration.js";
  5 | 
  6 | export const SYSTEM_PROMPT = `You are a specialized HTML generator that creates complete, production-ready HTML using ONLY Tailwind CSS classes.
  7 | 
  8 | IMPORTANT:
  9 | - Use EXCLUSIVELY Tailwind CSS classes for ALL styling - NO style attributes or style tags
 10 | - Every visual styling must be done through Tailwind classes
 11 | - Never use inline styles, CSS, or style tags
 12 | - If you need a style that seems custom, use Tailwind's arbitrary value syntax like [w-123px]
 13 | - Always write out the full, actual HTML elements with real content
 14 | - Never use Lorem Ipsum - write realistic English content
 15 | - Never include code comments or explanations
 16 | - DO NOT wrap the output in markdown code blocks
 17 | 
 18 | STRICT STYLING RULES:
 19 | - ❌ NO <style> tags
 20 | - ❌ NO style="" attributes
 21 | - ❌ NO CSS classes that aren't Tailwind
 22 | - ✅ ONLY use Tailwind utility classes
 23 | - ✅ Use Tailwind's arbitrary value syntax for custom values
 24 | - ✅ Stack multiple Tailwind classes to achieve desired styling
 25 | 
 26 | Requirements:
 27 | - Generate ONLY the HTML content for inside the <body> tag
 28 | - EXCLUSIVELY use Tailwind CSS classes for ALL styling
 29 | - NO placeholders, NO comments, NO explanations
 30 | - NO server-side code or JavaScript
 31 | - Every element must be complete with real content
 32 | - Write out all actual HTML elements - never indicate "pending" or "todo" sections
 33 | - DO NOT wrap the output in markdown code blocks
 34 | 
 35 | Example of CORRECT styling:
 36 | <div class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
 37 | Instead of:
 38 | <div style="background: blue; color: white;">
 39 | 
 40 | Generate complete, functional HTML using ONLY Tailwind classes for styling.`;
 41 | 
 42 | function cleanHtmlCode(fullHtml) {
 43 |   // First remove markdown code blocks
 44 |   let cleaned = fullHtml
 45 |     .replace(/```html/gi, "")
 46 |     .replace(/```(\s+)?html/gi, "")
 47 |     .replace(/```(\n)?html/gi, "")
 48 |     .replace(/```/g, "")
 49 |     .trim();
 50 | 
 51 |   // If the content is already wrapped in our template, extract just the content
 52 |   const templateContentMatch = cleaned.match(
 53 |     /CONTENT_PLACEHOLDER">\s*([\s\S]*?)\s*<\/body>/i
 54 |   );
 55 |   if (templateContentMatch) {
 56 |     return templateContentMatch[1].trim();
 57 |   }
 58 | 
 59 |   // Try to extract content between body tags
 60 |   const bodyContent = cleaned.match(/<body[^>]*>\s*([\s\S]*?)\s*<\/body>/i);
 61 |   if (bodyContent) {
 62 |     // Further clean the body content of any nested body tags
 63 |     return bodyContent[1]
 64 |       .replace(/<\/?body[^>]*>/gi, "") // Remove any nested body tags
 65 |       .trim();
 66 |   }
 67 | 
 68 |   // If no body tags found, remove any DOCTYPE and html wrapper if present
 69 |   cleaned = cleaned
 70 |     .replace(/<!DOCTYPE[^>]*>/gi, "")
 71 |     .replace(/<\/?html[^>]*>/gi, "")
 72 |     .replace(/<\/?body[^>]*>/gi, "")
 73 |     .replace(/<head>[\s\S]*?<\/head>/gi, "")
 74 |     .trim();
 75 | 
 76 |   return cleaned;
 77 | }
 78 | 
 79 | export function useLLMHtmlGeneration({
 80 |   modelConfig,
 81 |   backend,
 82 |   systemPrompt = SYSTEM_PROMPT,
 83 | }) {
 84 |   const [generatedCode, setGeneratedCode] = React.useState("");
 85 |   const lastGeneratedCode = React.useRef("");
 86 | 
 87 |   const { generate, warmup, isGenerating, error, partialText } = useLLMGeneration(
 88 |     modelConfig,
 89 |     systemPrompt,
 90 |     backend
 91 |   );
 92 | 
 93 |   React.useEffect(() => {
 94 |     if (partialText) {
 95 |       const cleaned = cleanHtmlCode(partialText);
 96 |       const processedCode = cleaned;
 97 |       setGeneratedCode(processedCode);
 98 |       lastGeneratedCode.current = cleaned; // Store just the body content
 99 |     }
100 |   }, [partialText]);
101 | 
102 |   const generateCode = React.useCallback(
103 |     async (prompt, extras, config) => {
104 |       const fullPrompt = lastGeneratedCode.current
105 |         ? `Current HTML: \n${lastGeneratedCode.current}\n\nRequest: ${prompt}`
106 |         : `Generate the HTML for: ${prompt}`;
107 | 
108 |       await generate(fullPrompt, extras, config);
109 |     },
110 |     [generate]
111 |   );
112 | 
113 |   return {
114 |     generateCode,
115 |     isGenerating,
116 |     error,
117 |     generatedCode,
118 |     warmup,
119 |   };
120 | }
121 | 


--------------------------------------------------------------------------------
/src/hooks/useSpeech.js:
--------------------------------------------------------------------------------
  1 | import { AutoTokenizer } from "@huggingface/transformers";
  2 | import { getModelFile } from "@huggingface/transformers/utils/hub.js";
  3 | import * as ort from "onnxruntime-web/webgpu";
  4 | 
  5 | export async function recognizeSpeechFromAudio(
  6 |   audio: Float32Array
  7 | ): Promise<string> {
  8 |   const modelRepo = "csukuangfj/sherpa-onnx-moonshine-base-en-int8";
  9 |   const tokenizerRepo = "UsefulSensors/moonshine-base";
 10 | 
 11 |   // Load all models first
 12 |   const preprocessModel = await getModelFile(modelRepo, "preprocess.onnx");
 13 |   const encodeModel = await getModelFile(modelRepo, "encode.int8.onnx");
 14 |   const uncachedDecodeModel = await getModelFile(
 15 |     modelRepo,
 16 |     "uncached_decode.int8.onnx"
 17 |   );
 18 |   const cachedDecodeModel = await getModelFile(
 19 |     modelRepo,
 20 |     "cached_decode.int8.onnx"
 21 |   );
 22 | 
 23 |   // Create all sessions
 24 |   const sessionOptions = { executionProviders: ["wasm"] };
 25 | 
 26 |   const preprocess = await ort.InferenceSession.create(
 27 |     preprocessModel,
 28 |     sessionOptions
 29 |   );
 30 |   const encode = await ort.InferenceSession.create(encodeModel, sessionOptions);
 31 |   const uncachedDecode = await ort.InferenceSession.create(
 32 |     uncachedDecodeModel,
 33 |     sessionOptions
 34 |   );
 35 |   const cachedDecode = await ort.InferenceSession.create(
 36 |     cachedDecodeModel,
 37 |     sessionOptions
 38 |   );
 39 | 
 40 |   // Load tokenizer
 41 |   const tokenizer = await AutoTokenizer.from_pretrained(tokenizerRepo);
 42 | 
 43 |   const sos_id = 1; // <s>
 44 |   const eos_id = 2; // </s>
 45 | 
 46 |   // Preprocess audio
 47 |   const audioTensor = new ort.Tensor("float32", audio, [1, audio.length]);
 48 |   const preprocessOutput = await preprocess.run({
 49 |     [preprocess.inputNames[0]]: audioTensor,
 50 |   });
 51 |   const features = preprocessOutput[preprocess.outputNames[0]];
 52 | 
 53 |   // Run encoder
 54 |   const featuresLen = new ort.Tensor(
 55 |     "int32",
 56 |     new Int32Array([features.dims[1]]),
 57 |     [1]
 58 |   );
 59 |   const encodeOutput = await encode.run({
 60 |     [encode.inputNames[0]]: features,
 61 |     [encode.inputNames[1]]: featuresLen,
 62 |   });
 63 |   const encoderOut = encodeOutput[encode.outputNames[0]];
 64 | 
 65 |   // First decode step
 66 |   const tokenTensor = new ort.Tensor("int32", new Int32Array([sos_id]), [1, 1]);
 67 |   const tokenLenTensor = new ort.Tensor("int32", new Int32Array([1]), [1]);
 68 | 
 69 |   const firstDecodeOutput = await uncachedDecode.run({
 70 |     [uncachedDecode.inputNames[0]]: tokenTensor,
 71 |     [uncachedDecode.inputNames[1]]: encoderOut,
 72 |     [uncachedDecode.inputNames[2]]: tokenLenTensor,
 73 |   });
 74 | 
 75 |   let logits = firstDecodeOutput["reversible_embedding"];
 76 |   let states = {};
 77 |   for (const key of Object.keys(firstDecodeOutput)) {
 78 |     if (key !== "reversible_embedding") {
 79 |       states[key] = firstDecodeOutput[key];
 80 |     }
 81 |   }
 82 | 
 83 |   const maxLen = Math.floor((audio.length / 16000) * 6);
 84 |   const generated_tokens: number[] = [];
 85 | 
 86 |   // Decode loop
 87 |   for (let i = 0; i < maxLen; i++) {
 88 |     const logitsArray = new Float32Array(logits.data);
 89 |     const token = Array.from(logitsArray.keys()).reduce(
 90 |       (maxIdx, curr) =>
 91 |         logitsArray[curr] > logitsArray[maxIdx] ? curr : maxIdx,
 92 |       0
 93 |     );
 94 | 
 95 |     if (token === eos_id && generated_tokens.length === 0) {
 96 |       continue;
 97 |     }
 98 | 
 99 |     if (token === eos_id && generated_tokens.length > 0) {
100 |       break;
101 |     }
102 | 
103 |     generated_tokens.push(token);
104 | 
105 |     const nextTokenTensor = new ort.Tensor(
106 |       "int32",
107 |       new Int32Array([token]),
108 |       [1, 1]
109 |     );
110 |     const nextTokenLenTensor = new ort.Tensor(
111 |       "int32",
112 |       new Int32Array([generated_tokens.length + 1]),
113 |       [1]
114 |     );
115 | 
116 |     const feeds = {
117 |       [cachedDecode.inputNames[0]]: nextTokenTensor,
118 |       [cachedDecode.inputNames[1]]: encoderOut,
119 |       [cachedDecode.inputNames[2]]: nextTokenLenTensor,
120 |     };
121 | 
122 |     let stateIdx = 3;
123 |     for (const key of Object.keys(states)) {
124 |       feeds[cachedDecode.inputNames[stateIdx]] = states[key];
125 |       stateIdx++;
126 |     }
127 | 
128 |     const cachedDecodeOutput = await cachedDecode.run(feeds);
129 | 
130 |     logits = cachedDecodeOutput["reversible_embedding"];
131 |     states = {};
132 |     for (const key of Object.keys(cachedDecodeOutput)) {
133 |       if (key !== "reversible_embedding") {
134 |         states[key] = cachedDecodeOutput[key];
135 |       }
136 |     }
137 |   }
138 | 
139 |   // Use the tokenizer's decode method directly
140 |   const text = await tokenizer.decode(generated_tokens, {
141 |     skip_special_tokens: true,
142 |     clean_up_tokenization_spaces: true,
143 |   });
144 | 
145 |   return text;
146 | }
147 | 


--------------------------------------------------------------------------------
/src/utils/math.js:
--------------------------------------------------------------------------------
  1 | export function int64ToFloat16(int64Value) {
  2 |   // Convert BigInt to Number (float64)
  3 |   const float64Value = Number(int64Value)
  4 | 
  5 |   // Handle special cases
  6 |   if (!isFinite(float64Value)) return float64Value > 0 ? 0x7c00 : 0xfc00 // +/- infinity
  7 |   if (float64Value === 0) return 0 // Zero is represented as 0
  8 | 
  9 |   // Get sign, exponent, and mantissa from float64
 10 |   const sign = float64Value < 0 ? 1 : 0
 11 |   const absValue = Math.abs(float64Value)
 12 |   const exponent = Math.floor(Math.log2(absValue))
 13 |   const mantissa = absValue / Math.pow(2, exponent) - 1
 14 | 
 15 |   // Convert exponent and mantissa to float16 format
 16 |   const float16Exponent = exponent + 15 // Offset exponent by 15 (float16 bias)
 17 |   const float16Mantissa = Math.round(mantissa * 1024) // 10-bit mantissa for float16
 18 | 
 19 |   // Handle overflow/underflow
 20 |   if (float16Exponent <= 0) {
 21 |     // Subnormal numbers (exponent <= 0)
 22 |     return (sign << 15) | (float16Mantissa >> 1)
 23 |   } else if (float16Exponent >= 31) {
 24 |     // Overflow, set to infinity
 25 |     return (sign << 15) | 0x7c00
 26 |   } else {
 27 |     // Normalized numbers
 28 |     return (sign << 15) | (float16Exponent << 10) | (float16Mantissa & 0x3ff)
 29 |   }
 30 | }
 31 | 
 32 | export function float16ToInt64(float16Value) {
 33 |   // Extract components from float16
 34 |   const sign = (float16Value & 0x8000) >> 15
 35 |   const exponent = (float16Value & 0x7c00) >> 10
 36 |   const mantissa = float16Value & 0x03ff
 37 | 
 38 |   // Handle special cases
 39 |   if (exponent === 0 && mantissa === 0) return BigInt(0) // Zero
 40 |   if (exponent === 0x1f) return sign ? BigInt("-Infinity") : BigInt("Infinity") // Infinity
 41 | 
 42 |   // Convert back to number
 43 |   let value
 44 |   if (exponent === 0) {
 45 |     // Subnormal numbers
 46 |     value = Math.pow(2, -14) * (mantissa / 1024)
 47 |   } else {
 48 |     // Normalized numbers
 49 |     value = Math.pow(2, exponent - 15) * (1 + mantissa / 1024)
 50 |   }
 51 | 
 52 |   // Apply sign
 53 |   value = sign ? -value : value
 54 | 
 55 |   return BigInt(Math.round(value))
 56 | }
 57 | 
 58 | export function int32ToFloat16(int32Value) {
 59 |   // Convert int32 to float
 60 |   const floatValue = Number(int32Value)
 61 | 
 62 |   // Handle special cases
 63 |   if (!isFinite(floatValue)) return floatValue > 0 ? 0x7c00 : 0xfc00 // +/- infinity
 64 |   if (floatValue === 0) return 0 // Zero is represented as 0
 65 | 
 66 |   // Get sign, exponent, and mantissa from float
 67 |   const sign = floatValue < 0 ? 1 : 0
 68 |   const absValue = Math.abs(floatValue)
 69 |   const exponent = Math.floor(Math.log2(absValue))
 70 |   const mantissa = absValue / Math.pow(2, exponent) - 1
 71 | 
 72 |   // Convert exponent and mantissa to float16 format
 73 |   const float16Exponent = exponent + 15 // Offset exponent by 15 (float16 bias)
 74 |   const float16Mantissa = Math.round(mantissa * 1024) // 10-bit mantissa for float16
 75 | 
 76 |   // Handle overflow/underflow
 77 |   if (float16Exponent <= 0) {
 78 |     // Subnormal numbers (exponent <= 0)
 79 |     return (sign << 15) | (float16Mantissa >> 1)
 80 |   } else if (float16Exponent >= 31) {
 81 |     // Overflow, set to infinity
 82 |     return (sign << 15) | 0x7c00
 83 |   } else {
 84 |     // Normalized numbers
 85 |     return (sign << 15) | (float16Exponent << 10) | (float16Mantissa & 0x3ff)
 86 |   }
 87 | }
 88 | 
 89 | export function float16ToInt32(float16Value) {
 90 |   // Extract components from float16
 91 |   const sign = (float16Value & 0x8000) >> 15
 92 |   const exponent = (float16Value & 0x7c00) >> 10
 93 |   const mantissa = float16Value & 0x03ff
 94 | 
 95 |   // Handle special cases
 96 |   if (exponent === 0 && mantissa === 0) return 0 // Zero
 97 |   if (exponent === 0x1f) return sign ? -2147483648 : 2147483647 // Handle infinity as max/min int32
 98 | 
 99 |   // Convert back to number
100 |   let value
101 |   if (exponent === 0) {
102 |     // Subnormal numbers
103 |     value = Math.pow(2, -14) * (mantissa / 1024)
104 |   } else {
105 |     // Normalized numbers
106 |     value = Math.pow(2, exponent - 15) * (1 + mantissa / 1024)
107 |   }
108 | 
109 |   // Apply sign
110 |   value = sign ? -value : value
111 | 
112 |   // Clamp to int32 range
113 |   value = Math.round(value)
114 |   if (value > 2147483647) return 2147483647
115 |   if (value < -2147483648) return -2147483648
116 | 
117 |   return value
118 | }
119 | 
120 | export function uint16ToFloat16(uint16) {
121 |   // Ensure the input is within the uint16 range
122 |   if (uint16 < 0 || uint16 > 0xffff) {
123 |     throw new RangeError("Input must be a uint16 value (0 to 65535).")
124 |   }
125 | 
126 |   const sign = (uint16 >> 15) & 0x1 // Extract sign bit
127 |   const exponent = (uint16 >> 10) & 0x1f // Extract exponent bits
128 |   const mantissa = uint16 & 0x3ff // Extract mantissa bits
129 | 
130 |   // Handle special cases for float16
131 |   if (exponent === 0 && mantissa === 0) {
132 |     return 0 // Zero
133 |   } else if (exponent === 31) {
134 |     return mantissa === 0 ? Infinity : NaN // Infinity or NaN
135 |   }
136 | 
137 |   // Adjust exponent for float32 representation
138 |   const adjustedExponent = exponent === 0 ? 0 : exponent + (127 - 15)
139 |   const float32 = (sign << 31) | (adjustedExponent << 23) | (mantissa << 13)
140 | 
141 |   // Convert to a Float32 number
142 |   const buffer = new ArrayBuffer(4)
143 |   new Uint32Array(buffer)[0] = float32
144 |   return new Float32Array(buffer)[0]
145 | }
146 | 
147 | function toFloat16(num) {
148 |   const buffer = new ArrayBuffer(4)
149 |   const dataView = new DataView(buffer)
150 |   dataView.setFloat32(0, num, true)
151 |   const float16Bits = dataView.getUint16(0, true)
152 |   return [num, float16Bits]
153 | }
154 | 
155 | function floatToFloat16(value) {
156 |   if (value === 0) return 0
157 |   if (isNaN(value)) return 0x7e00
158 | 
159 |   const sign = value < 0 ? 0x8000 : 0
160 |   value = Math.abs(value)
161 | 
162 |   if (!isFinite(value)) return sign | 0x7c00
163 | 
164 |   let exponent = Math.floor(Math.log2(value))
165 |   let fraction = Math.round((value / Math.pow(2, exponent) - 1) * 1024)
166 | 
167 |   if (exponent <= -15) {
168 |     fraction = Math.round((value / Math.pow(2, -14)) * 1024)
169 |     exponent = 0
170 |   } else {
171 |     exponent += 15
172 |     if (exponent >= 31) return sign | 0x7c00
173 |   }
174 | 
175 |   return sign | (exponent << 10) | fraction
176 | }
177 | 
178 | function float16ToUint16(value) {
179 |   if (value === 0) return 0
180 |   if (isNaN(value)) return 0x7e00
181 |   if (!isFinite(value)) return value < 0 ? 0xfc00 : 0x7c00
182 | 
183 |   const sign = value < 0 ? 0x8000 : 0
184 |   value = Math.abs(value)
185 | 
186 |   let exponent = Math.floor(Math.log2(value))
187 |   let fraction = Math.round((value / Math.pow(2, exponent) - 1) * 1024)
188 | 
189 |   if (exponent <= -15) {
190 |     fraction = Math.round((value / Math.pow(2, -14)) * 1024)
191 |     exponent = 0
192 |   } else {
193 |     exponent += 15
194 |     if (exponent >= 31) return sign | 0x7c00
195 |   }
196 | 
197 |   return sign | (exponent << 10) | fraction
198 | }
199 | 


--------------------------------------------------------------------------------
/src/playground/model_load_test.ts:
--------------------------------------------------------------------------------
  1 | import * as ort from "onnxruntime-web/webgpu";
  2 | 
  3 | export function model_load_test() {
  4 |   async function checkExternalDataExists(baseUrl) {
  5 |     const workerBlob = new Blob(
  6 |       [
  7 |         `
  8 |         self.onmessage = async function(e) {
  9 |           const urls = [e.data + '.onnx.data', e.data + '.onnx_data'];
 10 |           for (const url of urls) {
 11 |             try {
 12 |               const response = await fetch(url, {
 13 |                 method: "HEAD",
 14 |                 cache: 'no-cache'
 15 |               });
 16 |               if (response.ok) {
 17 |                 self.postMessage({ exists: true, url });
 18 |                 return;
 19 |               }
 20 |             } catch {}
 21 |           }
 22 |           self.postMessage({ exists: false, url: null });
 23 |         };
 24 |       `,
 25 |       ],
 26 |       { type: "application/javascript" }
 27 |     );
 28 | 
 29 |     const worker = new Worker(URL.createObjectURL(workerBlob));
 30 | 
 31 |     return new Promise((resolve) => {
 32 |       worker.onmessage = (e) => {
 33 |         worker.terminate();
 34 |         resolve(e.data);
 35 |       };
 36 |       worker.postMessage(baseUrl);
 37 |     });
 38 |   }
 39 | 
 40 |   async function testOnnxModels(baseDir) {
 41 |     // Get saved progress
 42 |     const savedProgress = localStorage.getItem("modelProgress");
 43 |     const savedResults = localStorage.getItem("modelResults");
 44 | 
 45 |     let currentModelIndex = 0;
 46 |     let currentVariantIndex = 0;
 47 |     let results = {
 48 |       success: [],
 49 |       failed: [],
 50 |       withExternalData: [],
 51 |     };
 52 | 
 53 |     if (savedProgress) {
 54 |       const progress = JSON.parse(savedProgress);
 55 |       currentModelIndex = progress.modelIndex;
 56 |       currentVariantIndex = progress.variantIndex;
 57 |     }
 58 | 
 59 |     if (savedResults) {
 60 |       results = JSON.parse(savedResults);
 61 |     }
 62 | 
 63 |     const models = [
 64 |       {
 65 |         name: "QwenVL_A",
 66 |         variants: [
 67 |           "",
 68 |           "_bnb4",
 69 |           "_fp16",
 70 |           "_int8",
 71 |           "_onnxslim",
 72 |           "_onnxslim_bnb4",
 73 |           "_onnxslim_fp16",
 74 |           "_onnxslim_int8",
 75 |           "_onnxslim_q4",
 76 |           "_onnxslim_q4f16",
 77 |           "_onnxslim_quantized",
 78 |           "_onnxslim_uint8",
 79 |           "_q4",
 80 |           "_q4f16",
 81 |           "_quantized",
 82 |           "_uint8",
 83 |         ],
 84 |       },
 85 |       {
 86 |         name: "QwenVL_B",
 87 |         variants: [
 88 |           "",
 89 |           "_bnb4",
 90 |           "_fp16",
 91 |           "_int8",
 92 |           "_onnxslim",
 93 |           "_onnxslim_bnb4",
 94 |           "_onnxslim_fp16",
 95 |           "_onnxslim_int8",
 96 |           "_onnxslim_q4",
 97 |           "_onnxslim_q4f16",
 98 |           "_onnxslim_quantized",
 99 |           "_onnxslim_uint8",
100 |           "_q4",
101 |           "_q4f16",
102 |           "_quantized",
103 |           "_uint8",
104 |         ],
105 |       },
106 |       {
107 |         name: "QwenVL_C",
108 |         variants: [
109 |           "",
110 |           "_bnb4",
111 |           "_fp16",
112 |           "_int8",
113 |           "_onnxslim",
114 |           "_onnxslim_bnb4",
115 |           "_onnxslim_fp16",
116 |           "_onnxslim_int8",
117 |           "_onnxslim_q4",
118 |           "_onnxslim_q4f16",
119 |           "_onnxslim_quantized",
120 |           "_onnxslim_uint8",
121 |           "_q4",
122 |           "_q4f16",
123 |           "_quantized",
124 |           "_uint8",
125 |         ],
126 |       },
127 |       {
128 |         name: "QwenVL_D",
129 |         variants: [
130 |           "",
131 |           "_bnb4",
132 |           "_fp16",
133 |           "_int8",
134 |           "_onnxslim",
135 |           "_onnxslim_bnb4",
136 |           "_onnxslim_fp16",
137 |           "_onnxslim_int8",
138 |           "_onnxslim_q4",
139 |           "_onnxslim_q4f16",
140 |           "_onnxslim_quantized",
141 |           "_onnxslim_uint8",
142 |           "_q4",
143 |           "_q4f16",
144 |           "_quantized",
145 |           "_uint8",
146 |         ],
147 |       },
148 |       {
149 |         name: "QwenVL_E",
150 |         variants: [
151 |           "",
152 |           "_bnb4",
153 |           "_fp16",
154 |           "_int8",
155 |           "_onnxslim",
156 |           "_onnxslim_bnb4",
157 |           "_onnxslim_fp16",
158 |           "_onnxslim_int8",
159 |           "_onnxslim_q4",
160 |           "_onnxslim_q4f16",
161 |           "_onnxslim_quantized",
162 |           "_onnxslim_uint8",
163 |           "_q4",
164 |           "_q4f16",
165 |           "_quantized",
166 |           "_uint8",
167 |         ],
168 |       },
169 |     ];
170 | 
171 |     // Check if we're done
172 |     if (currentModelIndex >= models.length) {
173 |       localStorage.removeItem("modelProgress");
174 |       localStorage.removeItem("modelResults");
175 |       console.log("\nAll tests complete!");
176 |       console.log("Final results:", JSON.stringify(results, null, 2));
177 |       return results;
178 |     }
179 | 
180 |     const model = models[currentModelIndex];
181 |     const variant = model.variants[currentVariantIndex];
182 |     const modelName = `${model.name}${variant}`;
183 |     const modelPath = `${baseDir}/${modelName}.onnx`;
184 |     const externalDataBaseUrl = `http://localhost:3002/${modelName}`;
185 | 
186 |     console.log(`Testing ${modelName}...`);
187 |     console.log(
188 |       `Progress: Model ${currentModelIndex + 1}/${models.length}, Variant ${
189 |         currentVariantIndex + 1
190 |       }/${model.variants.length}`
191 |     );
192 | 
193 |     let session;
194 | 
195 |     try {
196 |       const { exists, url } = await checkExternalDataExists(
197 |         externalDataBaseUrl
198 |       );
199 | 
200 |       if (exists) {
201 |         console.log(`📦 External data found for ${modelName} at ${url}`);
202 |         const dataPath = `./${modelName}.${
203 |           url.endsWith("_data") ? "onnx_data" : "onnx.data"
204 |         }`;
205 |         console.log({ dataPath });
206 |         session = await ort.InferenceSession.create(modelPath, {
207 |           externalData: [
208 |             {
209 |               path: dataPath,
210 |               data: url,
211 |             },
212 |           ],
213 |         });
214 |         results.withExternalData.push(modelName);
215 |         console.log(`✅ ${modelName} loaded successfully with external data`);
216 |       } else {
217 |         session = await ort.InferenceSession.create(modelPath);
218 |         results.success.push(modelName);
219 |         console.log(`✅ ${modelName} loaded successfully`);
220 |       }
221 |     } catch (error) {
222 |       results.failed.push({
223 |         name: modelName,
224 |         error: error.toString(),
225 |       });
226 |       console.error(`❌ ${modelName} failed:`, error.toString());
227 |     } finally {
228 |       if (session) {
229 |         await session.release();
230 |         session = null;
231 |       }
232 | 
233 |       // Calculate next position
234 |       currentVariantIndex++;
235 |       if (currentVariantIndex >= model.variants.length) {
236 |         currentModelIndex++;
237 |         currentVariantIndex = 0;
238 |       }
239 | 
240 |       // Save progress
241 |       localStorage.setItem(
242 |         "modelProgress",
243 |         JSON.stringify({
244 |           modelIndex: currentModelIndex,
245 |           variantIndex: currentVariantIndex,
246 |         })
247 |       );
248 |       localStorage.setItem("modelResults", JSON.stringify(results));
249 | 
250 |       // Wait a moment for cleanup
251 |       await new Promise((resolve) => setTimeout(resolve, 1000));
252 | 
253 |       // Reload the page with a cache-busting parameter
254 |       window.location.href =
255 |         window.location.href.split("?")[0] + "?reload=" + Date.now();
256 |     }
257 |   }
258 | 
259 |   // Start testing
260 |   const baseDir = "http://localhost:3002";
261 |   testOnnxModels(baseDir).catch((error) => {
262 |     console.error("Test execution failed:", error);
263 |   });
264 | }
265 | 
266 | function generateCSV(finalResults) {
267 |   const headers = [
268 |     "Model",
269 |     "Result",
270 |     "Error",
271 |     "With External Data",
272 |     "Quant Type",
273 |     "ONNXSlim",
274 |     "Model Group",
275 |   ];
276 |   const quantTypes = [
277 |     "q4",
278 |     "q4f16",
279 |     "int8",
280 |     "fp16",
281 |     "quantized",
282 |     "bnb4",
283 |     "uint8",
284 |   ];
285 | 
286 |   let csvRows = [headers.join(",")];
287 |   const modelGroups = ["A", "B", "C", "D", "E"];
288 | 
289 |   // Helper function to parse quant type and ONNXSlim
290 |   function parseQuantAndSlim(model) {
291 |     const quantType =
292 |       quantTypes.find((q) => model.toLowerCase().includes(q)) || "none";
293 |     const onnxslim = model.toLowerCase().includes("onnxslim") ? "Yes" : "No";
294 |     return [quantType, onnxslim];
295 |   }
296 | 
297 |   // Helper function to check if model needs external data
298 |   function hasExternalData(model) {
299 |     return finalResults.withExternalData.includes(model) ? "Yes" : "No";
300 |   }
301 | 
302 |   // Iterate through each model group
303 |   modelGroups.forEach((group) => {
304 |     csvRows.push(``); // Add group header
305 |     csvRows.push(`QwenVL_${group},,,,,,${group}`); // Add group header
306 | 
307 |     // Find all models in this group
308 |     const groupModels = finalResults.success
309 |       .concat(finalResults.failed.map((f) => f.name))
310 |       .filter((model) => model.startsWith(`QwenVL_${group}`));
311 | 
312 |     // Add each model entry with details
313 |     groupModels.forEach((model) => {
314 |       const result = finalResults.success.includes(model) ? "✅" : "❌";
315 |       const error =
316 |         finalResults.failed.find((f) => f.name === model)?.error || "";
317 |       const [quantType, onnxslim] = parseQuantAndSlim(model);
318 |       const externalData = hasExternalData(model);
319 | 
320 |       csvRows.push(
321 |         [model, result, error, externalData, quantType, onnxslim, group].join(
322 |           ","
323 |         )
324 |       );
325 |     });
326 |   });
327 | 
328 |   return csvRows.join("\n");
329 | }
330 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/src/Chat.js:
--------------------------------------------------------------------------------
  1 | import { Code, Github, XCircle, Database, MinimizeIcon, Send, Upload, Trash2, Linkedin } from "lucide-react";
  2 | import React, { useEffect, useRef, useState } from "react";
  3 | import { IFRAME_TEMPLATE, LLM_HTML_MODEL_CONFIG, LLM_VISION_MODEL_CONFIG } from "./constants/chat.js";
  4 | import { useLLMHtmlGeneration } from "./hooks/useLLMHtmlGeneration.js";
  5 | import { useLLMVisionGeneration } from "./hooks/useLLMVision.js";
  6 | 
  7 | const h = React.createElement;
  8 | 
  9 | const INTERFACE = 'TEXT';
 10 | 
 11 | export function Chat() {
 12 |   const [messages, setMessages] = useState([]);
 13 |   const [input, setInput] = useState("");
 14 |   const [selectedImage, setSelectedImage] = useState(null);
 15 |   const [selectedImageURL, setSelectedImageURL] = useState(null);
 16 |   const [isCodePanelOpen, setIsCodePanelOpen] = useState(false);
 17 |   const [showWarning, setShowWarning] = useState(true);
 18 |   const iframeRef = useRef(null);
 19 |   const fileInputRef = useRef(null);
 20 |   const [buildStage, setBuildStage] = useState(0);
 21 |   const [hasCache, setHasCache] = useState(false);
 22 |   const [cacheSize, setCacheSize] = useState(0);
 23 | 
 24 |   let generateText;
 25 |   let isGenerating;
 26 |   let error;
 27 |   let generatedText;
 28 |   let warmup;
 29 | 
 30 |   if (INTERFACE === 'IMAGE') {
 31 |     ({ generateText, isGenerating, error, generatedText, warmup } =
 32 |       useLLMVisionGeneration(LLM_VISION_MODEL_CONFIG));
 33 |   } else {
 34 |     ({ generateCode: generateText, isGenerating, error, generatedCode: generatedText, warmup } =
 35 |       useLLMHtmlGeneration(LLM_HTML_MODEL_CONFIG));
 36 |   }
 37 |   const [currentMessageId, setCurrentMessageId] = useState(null);
 38 | 
 39 |   const checkCache = async () => {
 40 |     try {
 41 |       const keys = await caches.keys();
 42 |       if (keys.length > 0) {
 43 |         setHasCache(true);
 44 |         const estimate = await navigator.storage.estimate();
 45 |         const usedBytes = estimate.usage || 0;
 46 |         setCacheSize(Math.round(usedBytes / (1024 * 1024)));
 47 |       }
 48 |     } catch (error) {
 49 |       console.error("Failed to check cache:", error);
 50 |     }
 51 |   };
 52 | 
 53 |   const clearModelCache = async () => {
 54 |     try {
 55 |       const keys = await caches.keys();
 56 |       await Promise.all(keys.map(key => caches.delete(key)));
 57 |       setHasCache(false);
 58 |       setCacheSize(0);
 59 |       window.location.reload();
 60 |     } catch (error) {
 61 |       console.error("Failed to clear cache:", error);
 62 |     }
 63 |   };
 64 | 
 65 |   useEffect(() => {
 66 |     checkCache();
 67 |   }, []);
 68 | 
 69 |   useEffect(() => {
 70 |     if (generatedText) {
 71 |       iframeRef.current?.contentWindow?.postMessage(
 72 |         {
 73 |           type: "updateContent",
 74 |           content: generatedText,
 75 |         },
 76 |         "*"
 77 |       );
 78 | 
 79 |       if (currentMessageId !== null) {
 80 |         setMessages((prev) =>
 81 |           prev.map((msg, idx) =>
 82 |             idx === currentMessageId ? { ...msg, content: generatedText } : msg
 83 |           )
 84 |         );
 85 |       }
 86 |     }
 87 |   }, [generatedText, currentMessageId]);
 88 | 
 89 |   const onChangeInput = (e) => {
 90 |     setInput(e.target.value);
 91 |     if (showWarning) {
 92 |       const proceed = window.confirm("Warning: Using this chat will download AI models larger than 1GB in size. Do you want to continue?");
 93 |       if (!proceed) return;
 94 |       warmup();
 95 |       setShowWarning(false);
 96 |     }
 97 |   }
 98 | 
 99 |   const handleImageUpload = (event) => {
100 |     const file = event.target.files[0];
101 |     if (file) {
102 |       const reader = new FileReader();
103 |       reader.onloadend = (e2) => {
104 |         setSelectedImage(reader.result);
105 |         setSelectedImageURL(e2.target.result);
106 |       };
107 |       reader.readAsDataURL(file);
108 |     }
109 |   };
110 | 
111 |   const handleSubmit = async (e) => {
112 |     e.preventDefault();
113 |     if ((!input.trim() && !selectedImage) || isGenerating) return;
114 | 
115 |     const userMessage = {
116 |       role: "user",
117 |       content: input,
118 |       image: selectedImage
119 |     };
120 |     const assistantMessage = { role: "assistant", content: "" };
121 | 
122 |     setMessages((prev) => [...prev, userMessage, assistantMessage]);
123 |     setCurrentMessageId(messages.length + 1);
124 |     setInput("");
125 |     setSelectedImage(null);
126 |     setIsCodePanelOpen(true);
127 | 
128 |     await generateText(input, { imageURL: selectedImage });
129 |   };
130 | 
131 |   return h(
132 |     "div",
133 |     { className: "min-h-screen bg-gray-950 text-gray-200" },
134 |     h(
135 |       "div",
136 |       { className: "fixed top-4 right-4 z-20 flex items-center gap-2" },
137 |       h(
138 |         "a",
139 |         {
140 |           href: "https://linkedin.com/in/pauldufour",
141 |           target: "_blank",
142 |           rel: "noopener noreferrer",
143 |           className: "bg-gray-900/80 backdrop-blur-sm rounded-full px-4 py-2 text-gray-300 hover:text-white hover:bg-gray-800 transition-colors flex items-center gap-2",
144 |         },
145 |         h(
146 |           "div",
147 |           { className: "bg-white rounded-full p-1" },
148 |           h(Linkedin, { className: "w-4 h-4 text-[#0A66C2]" })
149 |         ),
150 |         "Paul Dufour"
151 |       ),
152 |       h(
153 |         "a",
154 |         {
155 |           href: "https://github.com/pdufour/llm-coder",
156 |           target: "_blank",
157 |           rel: "noopener noreferrer",
158 |           className: "bg-gray-900/80 backdrop-blur-sm rounded-full px-4 py-2 text-gray-300 hover:text-white hover:bg-gray-800 transition-colors flex items-center gap-2",
159 |         },
160 |         h(Github, { className: "w-5 h-5" }),
161 |         "GitHub"
162 |       ), hasCache && h(
163 |         "button",
164 |         {
165 |           onClick: clearModelCache,
166 | 
167 |           className: "bg-gray-900/80 backdrop-blur-sm rounded-full p-2 text-gray-300 hover:text-white hover:bg-gray-800 transition-colors flex items-center gap-2",
168 |           title: `Clear cached models (${cacheSize}MB)`
169 |         },
170 |         h(XCircle, { className: "w-5 h-5" }),
171 |         `Delete Cache ${cacheSize}MB`
172 |       ),
173 |     ),
174 |     !isGenerating && !messages.length
175 |       ? h(
176 |         "div",
177 |         { className: "absolute inset-0 flex items-center justify-center" },
178 |         h(
179 |           "div",
180 |           {
181 |             className:
182 |               "text-5xl font-bold tracking-tighter text-gray-800/20 select-none",
183 |           },
184 |           "Prompt. Create. Deploy."
185 |         )
186 |       )
187 |       : h(
188 |         "div",
189 |         { className: "absolute inset-0 animate-fade-in" },
190 |         h(
191 |           "div",
192 |           {
193 |             className:
194 |               "h-screen w-full bg-white rounded-none overflow-hidden shadow-2xl",
195 |           },
196 |           h("iframe", {
197 |             ref: iframeRef,
198 |             className: "w-full h-full border-0",
199 |             title: "HTML Preview",
200 |             referrerPolicy: "no-referrer",
201 |             sandbox: "allow-forms allow-scripts",
202 |             srcDoc: IFRAME_TEMPLATE,
203 |             loading: "lazy",
204 |           })
205 |         )
206 |       ),
207 |     selectedImage && h(
208 |       "div",
209 |       { className: "fixed bottom-32 left-1/2 -translate-x-1/2 z-10 bg-gray-900/80 backdrop-blur-sm rounded-lg p-2" },
210 |       h(
211 |         "div",
212 |         { className: "relative" },
213 |         h("img", {
214 |           src: selectedImage,
215 |           alt: "Selected",
216 |           className: "max-h-32 rounded"
217 |         }),
218 |         h(
219 |           "button",
220 |           {
221 |             onClick: () => setSelectedImage(null),
222 |             className: "absolute -top-2 -right-2 bg-red-500 rounded-full p-1 text-white",
223 |           },
224 |           "×"
225 |         )
226 |       )
227 |     ),
228 |     isGenerating &&
229 |     h(
230 |       "div",
231 |       { className: "fixed top-4 left-1/2 -translate-x-1/2 z-10" },
232 |       h(
233 |         "div",
234 |         {
235 |           className:
236 |             "bg-gray-900/80 backdrop-blur-sm rounded-full px-4 py-2 text-sm text-gray-300 flex items-center gap-2",
237 |         },
238 |         h("div", {
239 |           className: "w-2 h-2 bg-green-500 rounded-full animate-pulse",
240 |         }),
241 |         "Generating..."
242 |       )
243 |     ),
244 |     h(
245 |       "div",
246 |       {
247 |         className: `fixed bottom-0 left-0 w-full bg-gray-900/95 backdrop-blur-lg border-t border-gray-800 transition-all duration-300 z-10 ${isCodePanelOpen ? "h-1/2" : "h-0"
248 |           }`,
249 |       },
250 |       isCodePanelOpen &&
251 |       h(React.Fragment, null, [
252 |         h(
253 |           "div",
254 |           {
255 |             className:
256 |               "flex justify-between items-center p-2 border-b border-gray-800",
257 |           },
258 |           h(
259 |             "div",
260 |             { className: "flex items-center gap-2 text-sm text-gray-400" },
261 |             h(Code, { className: "w-4 h-4" }),
262 |             "Generated Code"
263 |           ),
264 |           h(
265 |             "button",
266 |             {
267 |               onClick: () => setIsCodePanelOpen(false),
268 |               className: "p-1 hover:bg-gray-800 rounded",
269 |             },
270 |             h(MinimizeIcon, { className: "w-4 h-4" })
271 |           )
272 |         ),
273 |         h(
274 |           "div",
275 |           { className: "overflow-auto p-4 h-[calc(100%-40px)]" },
276 |           messages.map(
277 |             (message, index) =>
278 |               message.role === "assistant" &&
279 |               message.content &&
280 |               h(
281 |                 "pre",
282 |                 {
283 |                   key: index,
284 |                   className:
285 |                     "text-sm font-mono text-gray-300 whitespace-pre-wrap",
286 |                 },
287 |                 message.content
288 |               )
289 |           ),
290 |           error &&
291 |           h(
292 |             "div",
293 |             {
294 |               className: "bg-red-900/50 text-red-300 p-3 rounded-lg mt-4",
295 |             },
296 |             h("strong", null, "Error:"),
297 |             " ",
298 |             error.toString()
299 |           )
300 |         ),
301 |       ])
302 |     ),
303 |     h(
304 |       "div",
305 |       {
306 |         className:
307 |           "fixed bottom-8 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4 z-10",
308 |       },
309 |       h(
310 |         "form",
311 |         { onSubmit: handleSubmit, className: "relative" },
312 |         h(
313 |           "div",
314 |           {
315 |             className:
316 |               "relative flex items-center bg-gray-900/80 backdrop-blur-lg rounded-xl shadow-lg border border-gray-800",
317 |           },
318 |           h("input", {
319 |             type: "text",
320 |             value: input,
321 |             onChange: (e) => onChangeInput(e),
322 |             placeholder: "Describe what you want to create...",
323 |             className:
324 |               "flex-1 bg-transparent px-4 py-3 focus:outline-none placeholder-gray-500",
325 |             disabled: isGenerating,
326 |           }),
327 |           h("input", {
328 |             ref: fileInputRef,
329 |             type: "file",
330 |             accept: "image/*",
331 |             onChange: handleImageUpload,
332 |             className: "hidden",
333 |           }),
334 |           h(
335 |             "div",
336 |             { className: "flex items-center gap-2 pr-3" },
337 |             h(
338 |               "button",
339 |               {
340 |                 type: "button",
341 |                 onClick: () => fileInputRef.current?.click(),
342 |                 className:
343 |                   "p-2 text-gray-400 hover:text-gray-200 transition-colors",
344 |               },
345 |               h(Upload, { className: "w-5 h-5" })
346 |             ),
347 |             !isCodePanelOpen &&
348 |             messages.length > 0 &&
349 |             h(
350 |               "button",
351 |               {
352 |                 type: "button",
353 |                 className:
354 |                   "p-2 text-gray-400 hover:text-gray-200 transition-colors",
355 |                 onClick: () => setIsCodePanelOpen(true),
356 |               },
357 |               h(Code, { className: "w-5 h-5" })
358 |             ),
359 |             h(
360 |               "button",
361 |               {
362 |                 type: "submit",
363 |                 disabled: isGenerating,
364 |                 className:
365 |                   "bg-indigo-500 hover:bg-indigo-600 text-white rounded-lg p-2 transition-colors",
366 |               },
367 |               h(Send, { className: "w-5 h-5" })
368 |             )
369 |           )
370 |         )
371 |       )
372 |     ),
373 |     h("style", { jsx: true, global: true },
374 |       `
375 |         @keyframes fadeIn {
376 |           from {
377 |             opacity: 0;
378 |             transform: scale(0.98);
379 |           }
380 |           to {
381 |             opacity: 1;
382 |             transform: scale(1);
383 |           }
384 |         }
385 |         .animate-fade-in {
386 |           animation: fadeIn 0.5s ease-out forwards;
387 |         }
388 |       `
389 |     )
390 |   );
391 | }
392 | 


--------------------------------------------------------------------------------
/src/hooks/useLLMVision.jsold.js:
--------------------------------------------------------------------------------
  1 | import { AutoTokenizer, RawImage, Tensor } from "@huggingface/transformers"
  2 | import { getModelJSON, getModelFile } from "@huggingface/transformers/utils/hub.js"
  3 | import * as ort from "onnxruntime-web/webgpu"
  4 | import { logger } from "../utils/logging.js"
  5 | import { float16ToInt64, int64ToFloat16 } from "../utils/math.js"
  6 | 
  7 | 
  8 | 
  9 | 
 10 | const INPUT_IMAGE_SIZE = [960, 960]
 11 | const HEIGHT_FACTOR = 10
 12 | const WIDTH_FACTOR = 10
 13 | const IMAGE_EMBED_SIZE = WIDTH_FACTOR * HEIGHT_FACTOR
 14 | const BASE_MODEL = "Qwen/Qwen2-VL-2B-Instruct";
 15 | const ONNX_MODEL = "pdufour/Qwen2-VL-2B-Instruct-ONNX-Q4-F16";
 16 | const QUANTIZATION = "q4f16"
 17 | 
 18 | export async function useLLMVision(imagePath, query, vision = true) {
 19 |   logger.group("[CONFIG] Settings:")
 20 |   logger.log(`  INPUT_IMAGE_SIZE: ${JSON.stringify(INPUT_IMAGE_SIZE)}`)
 21 |   logger.log(`  HEIGHT_FACTOR: ${HEIGHT_FACTOR}`)
 22 |   logger.log(`  WIDTH_FACTOR: ${WIDTH_FACTOR}`)
 23 |   logger.log(`  MAX_SEQ_LENGTH: ${MAX_SEQ_LENGTH}`)
 24 |   logger.groupEnd()
 25 | 
 26 |   const suffix = QUANTIZATION ? `_${QUANTIZATION}` : ""
 27 | 
 28 |   logger.group("[SESSIONS] Loading and analyzing all ONNX sessions...")
 29 |   const startTime = performance.now()
 30 | 
 31 |   let ortSessionA, ortSessionB, ortSessionC, ortSessionD, ortSessionE
 32 | 
 33 |   logger.groupEnd()
 34 | 
 35 |   logger.groupCollapsed("[MODEL] Loading configuration...")
 36 |   const config = await getModelJSON(BASE_MODEL, "config.json")
 37 |   logger.log(`  num_hidden_layers: ${config.num_hidden_layers}`)
 38 |   logger.log(`  num_attention_heads: ${config.num_attention_heads}`)
 39 |   logger.log(`  num_key_value_heads: ${config.num_key_value_heads}`)
 40 |   logger.log(`  hidden_size: ${config.hidden_size}`)
 41 |   logger.groupEnd()
 42 | 
 43 |   const prompt_head_len = new Tensor("int64", new BigInt64Array([5n]), [1])
 44 |   logger.tensor("prompt_head_len", prompt_head_len)
 45 | 
 46 |   let position_ids
 47 |   let num_decode = 0
 48 |   let history_len = new Tensor("int64", new BigInt64Array([0n]), [1])
 49 |   logger.tensor("history_len", history_len)
 50 | 
 51 |   var pos_factor_v = BigInt(1 - IMAGE_EMBED_SIZE + WIDTH_FACTOR)
 52 | 
 53 |   let past_key_states = new ort.Tensor(
 54 |     "float16",
 55 |     new Uint16Array(
 56 |       config.num_hidden_layers *
 57 |       config.num_key_value_heads *
 58 |       MAX_SEQ_LENGTH *
 59 |       (config.hidden_size / config.num_attention_heads)
 60 |     ).fill(0),
 61 |     [
 62 |       config.num_hidden_layers,
 63 |       config.num_key_value_heads,
 64 |       MAX_SEQ_LENGTH,
 65 |       config.hidden_size / config.num_attention_heads
 66 |     ]
 67 |   )
 68 |   logger.tensor("past_key_states", past_key_states)
 69 | 
 70 |   let past_value_states = past_key_states
 71 |   logger.tensor("past_value_states", past_value_states)
 72 | 
 73 |   let attention_mask = new ort.Tensor(
 74 |     "float16", // -65504.0 in float16
 75 |     new Uint16Array([0xfbff]),
 76 |     [1]
 77 |   )
 78 |   logger.tensor("attention_mask", attention_mask)
 79 | 
 80 |   let pos_factor = new Tensor("float16", new Uint16Array([0]), [1])
 81 |   logger.tensor("pos_factor", pos_factor)
 82 | 
 83 |   logger.groupCollapsed("[TOKENIZATION] Processing prompt...")
 84 |   const tokenizer = await AutoTokenizer.from_pretrained(BASE_MODEL)
 85 |   const prompt = `\n<|im_start|>user\n<|vision_start|><|vision_end|>${query}<|im_end|>\n<|im_start|>assistant\n`
 86 |   const token = await tokenizer(prompt, {
 87 |     return_tensors: "pt",
 88 |     add_generation_prompt: false,
 89 |     tokenize: true
 90 |   }).input_ids
 91 |   logger.log("Token shape:", token.dims)
 92 |   logger.log("Token values:", Array.from(token.data))
 93 |   logger.groupEnd()
 94 | 
 95 |   const seq_length = token.dims[1]
 96 |   let ids_len = new Tensor("int64", new BigInt64Array([BigInt(seq_length)]), [
 97 |     1
 98 |   ])
 99 |   logger.tensor("ids_len", ids_len)
100 | 
101 |   let input_ids = new ort.Tensor(
102 |     "int32",
103 |     new Int32Array(MAX_SEQ_LENGTH).fill(0),
104 |     [MAX_SEQ_LENGTH]
105 |   )
106 |   logger.tensor("input_ids (initial)", input_ids)
107 | 
108 |   input_ids.data.set(Array.from(token.data.slice(0, seq_length), Number))
109 |   logger.tensor("input_ids (after set)", input_ids)
110 | 
111 |   const dummy = new ort.Tensor("int32", new Int32Array([0]), [])
112 |   logger.tensor("dummy", dummy)
113 | 
114 |   logger.groupCollapsed("[INFERENCE] Running initial inference...")
115 |   logger.log("Computing hidden states...")
116 |   if (!ortSessionB) {
117 |     ortSessionB = await ort.InferenceSession.create(
118 |       `http://localhost:3004/onnx-dist/QwenVL_B${suffix}.onnx`,
119 |       {
120 |         executionProviders: ["webgpu"],
121 |         logSeverityLevel: 0,
122 |         logVerbosityLevel: 1,
123 |         enableProfiling: true,
124 |         enableCpuMemArena: true,
125 |         graphOptimizationLevel: "all",
126 |         executionMode: "sequential",
127 |         intraOpNumThreads: 0,
128 |         interOpNumThreads: 0
129 |       }
130 |     )
131 |   }
132 |   let { hidden_states } = await ortSessionB.run({
133 |     input_ids: input_ids,
134 |     ids_len: ids_len
135 |   })
136 | 
137 |   logger.tensor("hidden_states (initial)", hidden_states)
138 | 
139 |   logger.groupCollapsed("[POSITION] Computing position IDs...")
140 |   if (!ortSessionC) {
141 |     ortSessionC = await ort.InferenceSession.create(
142 |       `http://localhost:3004/onnx-dist/QwenVL_C${suffix}.onnx`,
143 |       {
144 |         executionProviders: ["webgpu"],
145 |         logSeverityLevel: 2,
146 |         logVerbosityLevel: 1,
147 |         enableProfiling: true,
148 |         enableCpuMemArena: true,
149 |         graphOptimizationLevel: "all",
150 |         executionMode: "sequential",
151 |         intraOpNumThreads: 0,
152 |         interOpNumThreads: 0
153 |       }
154 |     )
155 |   }
156 |   ; ({ position_ids } = await ortSessionC.run({
157 |     dummy: dummy
158 |   }))
159 |   logger.tensor("position_ids (initial)", position_ids)
160 |   logger.groupEnd()
161 |   logger.groupEnd()
162 | 
163 |   // Process image
164 |   if (vision) {
165 |     logger.log("\n[IMAGE] Processing image...")
166 |     let image = await RawImage.fromURL(imagePath)
167 |     logger.log(`Original size: ${image.width}x${image.height}`)
168 |     logger.log(`Original mode: ${image.mode}`)
169 | 
170 |     // image = await image.resize(INPUT_IMAGE_SIZE[0], INPUT_IMAGE_SIZE[1]);
171 |     console.log(`  Resized to: ${image.width}x${image.height}`)
172 | 
173 |     image = image.rgb()
174 | 
175 |     image = image.toTensor("CHW")
176 |     image = image.to("float32")
177 |     image = image.div_(255.0)
178 |     const pixel_values = image.unsqueeze(0)
179 |     logger.tensor("pixel_values", pixel_values)
180 | 
181 |     // Run session A for image embeddings
182 |     if (!ortSessionA) {
183 |       console.log("create session a")
184 |       ortSessionA = await ort.InferenceSession.create(
185 |         `http://localhost:3004/onnx-dist/QwenVL_A${suffix}.onnx`,
186 |         {
187 |           executionProviders: ["webgpu"],
188 |           logSeverityLevel: 0,
189 |           logVerbosityLevel: 0,
190 |           enableProfiling: false,
191 |           enableCpuMemArena: false,
192 |           graphOptimizationLevel: "all",
193 |           executionMode: "sequential",
194 |           intraOpNumThreads: 0,
195 |           interOpNumThreads: 0
196 |         }
197 |       )
198 |     }
199 | 
200 |     logger.log("session a run")
201 |     const { image_embed } = await ortSessionA.run({
202 |       pixel_values: pixel_values
203 |     })
204 |     console.log("done session a")
205 | 
206 |     logger.tensor("image_embed", image_embed)
207 | 
208 |     ids_len = ids_len.add(BigInt(IMAGE_EMBED_SIZE))
209 | 
210 |     const split_factor = new Tensor(
211 |       "int32",
212 |       new Int32Array([
213 |         MAX_SEQ_LENGTH - Number(ids_len.item()) - IMAGE_EMBED_SIZE
214 |       ]),
215 |       [1]
216 |     )
217 | 
218 |     const ids_len_minus = new Tensor(
219 |       "int32",
220 |       new Int32Array([Number(ids_len.item()) - Number(prompt_head_len.item())]),
221 |       [1]
222 |     )
223 | 
224 |     await ortSessionA.release()
225 |     ortSessionA = null
226 | 
227 |     logger.log("session d create")
228 |     ortSessionD = await ort.InferenceSession.create(
229 |       `http://localhost:3004/onnx-dist/QwenVL_D${suffix}.onnx`,
230 |       {
231 |         executionProviders: ["webgpu"],
232 |         logSeverityLevel: 2,
233 |         logVerbosityLevel: 0,
234 |         enableProfiling: false,
235 |         enableCpuMemArena: false,
236 |         graphOptimizationLevel: "all",
237 |         executionMode: "sequential",
238 |         intraOpNumThreads: 0,
239 |         interOpNumThreads: 0
240 |       }
241 |     )
242 | 
243 |     logger.log("session d run")
244 |     logger.tensor("image_embed", image_embed)
245 |     logger.tensor("ids_len", ids_len)
246 |     logger.tensor("ids_len_minus", ids_len_minus)
247 |     logger.tensor("split_factor", split_factor)
248 | 
249 |       ; ({ hidden_states, position_ids } = await ortSessionD.run({
250 |         "hidden_states.1": hidden_states,
251 |         image_embed,
252 |         ids_len,
253 |         ids_len_minus,
254 |         split_factor
255 |       }))
256 | 
257 |     logger.tensor("updated hidden_states", hidden_states)
258 |     logger.tensor("updated position_ids", position_ids)
259 | 
260 |     await ortSessionD.release()
261 |     ortSessionD = null
262 |   }
263 | 
264 |   logger.groupCollapsed("[GENERATION] Starting text generation...")
265 |   const generationStartTime = performance.now()
266 | 
267 |   while (
268 |     num_decode < MAX_SINGLE_CHAT_LENGTH &&
269 |     Number(history_len.data[0]) < MAX_SEQ_LENGTH
270 |   ) {
271 |     let token_id
272 |     logger.groupCollapsed(`Step ${num_decode}`)
273 |     logger.groupCollapsed("Session E inputs:")
274 |     logger.tensor("hidden_states", hidden_states)
275 |     logger.tensor("attention_mask", attention_mask)
276 |     logger.tensor("past_key_states", past_key_states)
277 |     logger.tensor("past_value_states", past_value_states)
278 |     logger.tensor("history_len", history_len)
279 |     logger.tensor("ids_len", ids_len)
280 |     logger.tensor("position_ids", position_ids)
281 |     logger.tensor("pos_factor", pos_factor)
282 |     logger.groupEnd()
283 | 
284 |     if (!ortSessionE) {
285 |       console.log("Create ortSessionE")
286 |       ortSessionE = await ort.InferenceSession.create(
287 |         `http://localhost:3004/onnx-dist/QwenVL_E_q4f16.onnx`,
288 |         {
289 |           executionProviders: ["wasm"],
290 |           logSeverityLevel: 2,
291 |           logVerbosityLevel: 0,
292 |           enableProfiling: false,
293 |           enableCpuMemArena: false,
294 |           graphOptimizationLevel: "all",
295 |           executionMode: "sequential",
296 |           intraOpNumThreads: 0,
297 |           interOpNumThreads: 0
298 |         }
299 |       )
300 |       console.log("outputNames", ortSessionE.outputNames)
301 |     }
302 | 
303 |     // ort.env.debug = true;
304 |     // ort.env.logLevel = "verbose";
305 | 
306 |     ; ({
307 |       max_logit_ids: token_id,
308 |       past_key_states: past_key_states,
309 |       past_value_states: past_value_states
310 |     } = await ortSessionE.run({
311 |       hidden_states,
312 |       attention_mask,
313 |       "past_key_states.1": past_key_states,
314 |       "past_value_states.1": past_value_states,
315 |       history_len,
316 |       ids_len,
317 |       position_ids,
318 |       pos_factor
319 |     }))
320 | 
321 |     if (token_id === 151643 || token_id === 151645) {
322 |       logger.log("Reached stop token")
323 |       logger.groupEnd()
324 |       break
325 |     }
326 | 
327 |     logger.tensor("New token_id", token_id)
328 |     logger.log({ token_id })
329 | 
330 |     num_decode++
331 |     if (num_decode < 2) {
332 |       logger.groupCollapsed("First decode step adjustments:")
333 |       history_len = history_len.add(BigInt(ids_len.data[0]))
334 |       logger.tensor("Updated history_len", history_len)
335 | 
336 |       ids_len = new ort.Tensor("int64", new BigInt64Array([1n]), [1])
337 |       logger.log(`Updated ids_len: ${ids_len.data[0]}`)
338 | 
339 |       attention_mask = new ort.Tensor("float16", new Uint16Array([0]), [1])
340 |       logger.log(`Updated attention_mask: ${attention_mask.data[0]}`)
341 | 
342 |       if (vision) {
343 |         pos_factor = new Tensor(
344 |           "float16",
345 |           new Uint16Array([int64ToFloat16(pos_factor_v + ids_len.data[0])]),
346 |           [1]
347 |         )
348 |       } else {
349 |         pos_factor = new Tensor(
350 |           "float16",
351 |           new Uint16Array([int64ToFloat16(history_len.data[0] + BigInt(1))]),
352 |           [1]
353 |         )
354 |       }
355 | 
356 |       logger.tensor("Updated pos_factor", pos_factor)
357 |       logger.groupEnd()
358 |     } else {
359 |       logger.groupCollapsed(`Regular step ${num_decode} adjustments:`)
360 |       history_len = history_len.add(BigInt(1))
361 |       pos_factor = pos_factor.map(v =>
362 |         int64ToFloat16(float16ToInt64(v) + BigInt(1))
363 |       )
364 |       logger.tensor("Updated history_len", history_len)
365 |       logger.tensor("Updated pos_factor", pos_factor)
366 |       logger.groupEnd()
367 |     }
368 |     input_ids.data[0] = Number(token_id.data[0])
369 |     logger.tensor("Updated input_ids", input_ids)
370 | 
371 |     logger.groupCollapsed("Getting new hidden states...")
372 |     const result_B = await ortSessionB.run({
373 |       input_ids: input_ids,
374 |       ids_len: ids_len
375 |     })
376 |     hidden_states = result_B.hidden_states
377 |     logger.tensor("New hidden_states", hidden_states)
378 |     logger.groupEnd()
379 | 
380 |     if (
381 |       !Number.isInteger(token_id.data[0]) &&
382 |       !["bigint", "number"].includes(typeof token_id.data[0])
383 |     ) {
384 |       throw new Error(`Token ID is not an integer`)
385 |     } else {
386 |       // Decode token
387 |       const decoded = tokenizer.decode([...token_id.data])
388 |       logger.log(`Decoded token: ${decoded}`)
389 |     }
390 | 
391 |     logger.groupEnd()
392 |   }
393 | 
394 |   const generationTime = (performance.now() - generationStartTime) / 1000
395 |   logger.groupCollapsed("[PERFORMANCE] Generation complete:")
396 |   logger.log(`  Total tokens generated: ${num_decode}`)
397 |   logger.log(`  Generation time: ${generationTime.toFixed(2)}s`)
398 |   logger.log(`  Speed: ${(num_decode / generationTime).toFixed(3)} tokens/s`)
399 |   logger.groupEnd()
400 | }
401 | 


--------------------------------------------------------------------------------
/src/hooks/useLLMGeneration.js:
--------------------------------------------------------------------------------
  1 | import {
  2 |   AutoModel,
  3 |   AutoTokenizer,
  4 |   TextStreamer
  5 | } from "@huggingface/transformers";
  6 | import { CreateMLCEngine } from "@mlc-ai/web-llm";
  7 | import React from "react";
  8 | 
  9 | import { RawImage, Tensor } from "@huggingface/transformers";
 10 | import {
 11 |   getModelFile,
 12 |   getModelJSON,
 13 | } from "@huggingface/transformers/utils/hub.js";
 14 | import * as ort from "onnxruntime-web/webgpu";
 15 | import { logger } from "../utils/logging.js";
 16 | import { float16ToInt64, int64ToFloat16 } from "../utils/math.js";
 17 | 
 18 | export const SYSTEM_PROMPT = `You are a specialized HTML generator that creates complete, production-ready HTML using ONLY Tailwind CSS classes.`;
 19 | 
 20 | const INPUT_IMAGE_SIZE = [960, 960];
 21 | const HEIGHT_FACTOR = 10;
 22 | const WIDTH_FACTOR = 10;
 23 | const IMAGE_EMBED_SIZE = WIDTH_FACTOR * HEIGHT_FACTOR;
 24 | const QUANTIZATION = "q4f16";
 25 | const DEFAULT_SESSION_OPTIONS = {
 26 |   executionProviders: ["webgpu"],
 27 |   logSeverityLevel: 2,
 28 |   logVerbosityLevel: 1,
 29 |   // enableProfiling: true,
 30 |   // enableCpuMemArena: true,
 31 |   // graphOptimizationLevel: "all",
 32 |   // executionMode: "sequential",
 33 |   // intraOpNumThreads: 0,
 34 |   // interOpNumThreads: 0,
 35 | };
 36 | 
 37 | class Qwen2VLBackend {
 38 |   constructor(modelId, config) {
 39 |     this.modelId = modelId;
 40 |     this.config = config;
 41 |   }
 42 | 
 43 |   async generate(prompt, systemPrompt, callbacks, extras = {}) {
 44 |     const { imageURL } = extras;
 45 |     const vision = imageURL?.length || 0 > 0;
 46 |     logger.groupCollapsed("[CONFIG] Settings:");
 47 |     logger.log(`  INPUT_IMAGE_SIZE: ${JSON.stringify(INPUT_IMAGE_SIZE)}`);
 48 |     logger.log(`  HEIGHT_FACTOR: ${HEIGHT_FACTOR}`);
 49 |     logger.log(`  WIDTH_FACTOR: ${WIDTH_FACTOR}`);
 50 |     logger.log(`  max_seq_length: ${this.config.max_seq_length}`);
 51 |     logger.groupEnd();
 52 | 
 53 |     let ortSessionA, ortSessionB, ortSessionC, ortSessionD, ortSessionE;
 54 | 
 55 |     logger.groupCollapsed("[MODEL] Loading configuration...");
 56 |     const config = await getModelJSON(this.config.baseModelId, "config.json");
 57 |     logger.log(`  num_hidden_layers: ${config.num_hidden_layers}`);
 58 |     logger.log(`  num_attention_heads: ${config.num_attention_heads}`);
 59 |     logger.log(`  num_key_value_heads: ${config.num_key_value_heads}`);
 60 |     logger.log(`  hidden_size: ${config.hidden_size}`);
 61 |     logger.groupEnd();
 62 | 
 63 |     const prompt_head_len = new Tensor("int64", new BigInt64Array([5n]), [1]);
 64 |     logger.tensor("prompt_head_len", prompt_head_len);
 65 | 
 66 |     let position_ids;
 67 |     let num_decode = 0;
 68 |     let history_len = new Tensor("int64", new BigInt64Array([0n]), [1]);
 69 |     logger.tensor("history_len", history_len);
 70 | 
 71 |     var pos_factor_v = BigInt(1 - IMAGE_EMBED_SIZE + WIDTH_FACTOR);
 72 | 
 73 |     let past_key_states = new ort.Tensor(
 74 |       "float16",
 75 |       new Uint16Array(
 76 |         config.num_hidden_layers *
 77 |         config.num_key_value_heads *
 78 |         this.config.max_seq_length *
 79 |         (config.hidden_size / config.num_attention_heads)
 80 |       ).fill(0),
 81 |       [
 82 |         config.num_hidden_layers,
 83 |         config.num_key_value_heads,
 84 |         this.config.max_seq_length,
 85 |         config.hidden_size / config.num_attention_heads,
 86 |       ]
 87 |     );
 88 |     logger.tensor("past_key_states", past_key_states);
 89 | 
 90 |     let past_value_states = past_key_states;
 91 |     logger.tensor("past_value_states", past_value_states);
 92 | 
 93 |     let attention_mask = new ort.Tensor(
 94 |       "float16", // -65504.0 in float16
 95 |       new Uint16Array([0xfbff]),
 96 |       [1]
 97 |     );
 98 |     logger.tensor("attention_mask", attention_mask);
 99 | 
100 |     let pos_factor = new Tensor("float16", new Uint16Array([0]), [1]);
101 |     logger.tensor("pos_factor", pos_factor);
102 | 
103 |     logger.groupCollapsed("[TOKENIZATION] Processing prompt...");
104 |     const tokenizer = await AutoTokenizer.from_pretrained(this.config.baseModelId);
105 |     const token_prompt = `\n<|im_start|>user\n<|vision_start|><|vision_end|>${prompt}<|im_end|>\n<|im_start|>assistant\n`;
106 |     const token = await tokenizer(token_prompt, {
107 |       return_tensors: "pt",
108 |       add_generation_prompt: false,
109 |       tokenize: true,
110 |     }).input_ids;
111 |     logger.log("Token shape:", token.dims);
112 |     logger.log("Token values:", Array.from(token.data));
113 |     logger.groupEnd();
114 | 
115 |     const seq_length = token.dims[1];
116 |     let ids_len = new Tensor("int64", new BigInt64Array([BigInt(seq_length)]), [
117 |       1,
118 |     ]);
119 |     logger.tensor("ids_len", ids_len);
120 | 
121 |     let input_ids = new ort.Tensor(
122 |       "int32",
123 |       new Int32Array(this.config.max_seq_length).fill(0),
124 |       [this.config.max_seq_length]
125 |     );
126 |     logger.tensor("input_ids (initial)", input_ids);
127 | 
128 |     input_ids.data.set(Array.from(token.data.slice(0, seq_length), Number));
129 |     logger.tensor("input_ids (after set)", input_ids);
130 | 
131 |     const dummy = new ort.Tensor("int32", new Int32Array([0]), []);
132 |     logger.tensor("dummy", dummy);
133 | 
134 |     logger.groupCollapsed("[INFERENCE] Running initial inference...");
135 |     logger.log("Computing hidden states...");
136 |     if (!ortSessionB) {
137 |       ortSessionB = await ort.InferenceSession.create(
138 |         await getModelFile(this.modelId, `onnx/QwenVL_B_${QUANTIZATION}.onnx`),
139 |         DEFAULT_SESSION_OPTIONS,
140 |       );
141 |     }
142 |     let { hidden_states } = await ortSessionB.run({
143 |       input_ids: input_ids,
144 |       ids_len: ids_len,
145 |     });
146 | 
147 |     logger.tensor("hidden_states (initial)", hidden_states);
148 | 
149 |     logger.groupCollapsed("[POSITION] Computing position IDs...");
150 |     if (!ortSessionC) {
151 |       ortSessionC = await ort.InferenceSession.create(
152 |         await getModelFile(this.modelId, `onnx/QwenVL_C_${QUANTIZATION}.onnx`),
153 |         DEFAULT_SESSION_OPTIONS,
154 |       );
155 |     }
156 |     ({ position_ids } = await ortSessionC.run({
157 |       dummy: dummy,
158 |     }));
159 |     logger.tensor("position_ids (initial)", position_ids);
160 |     logger.groupEnd();
161 |     logger.groupEnd();
162 | 
163 |     // Process image
164 |     if (vision) {
165 |       logger.log("\n[IMAGE] Processing image...");
166 |       let image = await RawImage.fromURL(imageURL);
167 |       logger.log(`Original size: ${image.width}x${image.height}`);
168 | 
169 |       image = await image.resize(INPUT_IMAGE_SIZE[0], INPUT_IMAGE_SIZE[1]);
170 |       logger.log(`  Resized to: ${image.width}x${image.height}`);
171 | 
172 |       image = image.rgb();
173 | 
174 |       image = image.toTensor("CHW");
175 |       image = image.to("float32");
176 |       image = image.div_(255.0);
177 |       const pixel_values = image.unsqueeze(0);
178 |       logger.tensor("pixel_values", pixel_values);
179 | 
180 |       // Run session A for image embeddings
181 |       if (!ortSessionA) {
182 |         ortSessionA = await ort.InferenceSession.create(
183 |           await getModelFile(this.modelId, `onnx/QwenVL_A_${QUANTIZATION}.onnx`),
184 |           DEFAULT_SESSION_OPTIONS,
185 |         );
186 |       }
187 | 
188 |       logger.log("session a run");
189 |       const { image_embed } = await ortSessionA.run({
190 |         pixel_values: pixel_values,
191 |       });
192 |       logger.log("done session a");
193 | 
194 |       logger.tensor("image_embed", image_embed);
195 | 
196 |       ids_len = ids_len.add(BigInt(IMAGE_EMBED_SIZE));
197 | 
198 |       const split_factor = new Tensor(
199 |         "int32",
200 |         new Int32Array([
201 |           this.config.max_seq_length - Number(ids_len.item()) - IMAGE_EMBED_SIZE,
202 |         ]),
203 |         [1]
204 |       );
205 | 
206 |       const ids_len_minus = new Tensor(
207 |         "int32",
208 |         new Int32Array([
209 |           Number(ids_len.item()) - Number(prompt_head_len.item()),
210 |         ]),
211 |         [1]
212 |       );
213 | 
214 |       await ortSessionA.release();
215 |       ortSessionA = null;
216 | 
217 |       logger.log("session d create");
218 |       if (!ortSessionD) {
219 |         ortSessionD = await ort.InferenceSession.create(
220 |           await getModelFile(this.modelId, `onnx/QwenVL_D_${QUANTIZATION}.onnx`),
221 |           DEFAULT_SESSION_OPTIONS,
222 |         );
223 |       }
224 | 
225 |       logger.log("session d run");
226 |       logger.tensor("image_embed", image_embed);
227 |       logger.tensor("ids_len", ids_len);
228 |       logger.tensor("ids_len_minus", ids_len_minus);
229 |       logger.tensor("split_factor", split_factor);
230 |       ({ hidden_states, position_ids } = await ortSessionD.run({
231 |         "hidden_states.1": hidden_states,
232 |         image_embed,
233 |         ids_len,
234 |         ids_len_minus,
235 |         split_factor,
236 |       }));
237 | 
238 |       logger.tensor("updated hidden_states", hidden_states);
239 |       logger.tensor("updated position_ids", position_ids);
240 | 
241 |       await ortSessionD.release();
242 |       ortSessionD = null;
243 |     }
244 | 
245 |     logger.groupCollapsed("[GENERATION] Starting text generation...");
246 |     const generationStartTime = performance.now();
247 | 
248 |     let accumulated = "";
249 |     while (
250 |       num_decode < this.config.max_single_chat_length &&
251 |       Number(history_len.data[0]) < this.config.max_seq_length
252 |     ) {
253 |       let token_id;
254 |       logger.groupCollapsed(`Step ${num_decode}`);
255 |       logger.groupCollapsed("Session E inputs:");
256 |       logger.tensor("hidden_states", hidden_states);
257 |       logger.tensor("attention_mask", attention_mask);
258 |       logger.tensor("past_key_states", past_key_states);
259 |       logger.tensor("past_value_states", past_value_states);
260 |       logger.tensor("history_len", history_len);
261 |       logger.tensor("ids_len", ids_len);
262 |       logger.tensor("position_ids", position_ids);
263 |       logger.tensor("pos_factor", pos_factor);
264 |       logger.groupEnd();
265 | 
266 |       if (!ortSessionE) {
267 |         logger.log("Create ortSessionE");
268 |         ortSessionE = await ort.InferenceSession.create(
269 |           await getModelFile(this.modelId, `onnx/QwenVL_E_${QUANTIZATION}.onnx`),
270 |           { ...DEFAULT_SESSION_OPTIONS, executionProviders: ["wasm"] },
271 |         );
272 |       }
273 | 
274 |       ({
275 |         max_logit_ids: token_id,
276 |         past_key_states: past_key_states,
277 |         past_value_states: past_value_states,
278 |       } = await ortSessionE.run({
279 |         hidden_states,
280 |         attention_mask,
281 |         "past_key_states.1": past_key_states,
282 |         "past_value_states.1": past_value_states,
283 |         history_len,
284 |         ids_len,
285 |         position_ids,
286 |         pos_factor,
287 |       }));
288 | 
289 |       if (token_id === 151643 || token_id === 151645) {
290 |         logger.log("Reached stop token");
291 |         logger.groupEnd();
292 |         break;
293 |       }
294 | 
295 |       logger.tensor("New token_id", token_id);
296 |       logger.log({ token_id });
297 | 
298 |       num_decode++;
299 |       if (num_decode < 2) {
300 |         logger.groupCollapsed("First decode step adjustments:");
301 |         history_len = history_len.add(BigInt(ids_len.data[0]));
302 |         logger.tensor("Updated history_len", history_len);
303 | 
304 |         ids_len = new ort.Tensor("int64", new BigInt64Array([1n]), [1]);
305 |         logger.log(`Updated ids_len: ${ids_len.data[0]}`);
306 | 
307 |         attention_mask = new ort.Tensor("float16", new Uint16Array([0]), [1]);
308 |         logger.log(`Updated attention_mask: ${attention_mask.data[0]}`);
309 | 
310 |         if (vision) {
311 |           pos_factor = new Tensor(
312 |             "float16",
313 |             new Uint16Array([int64ToFloat16(pos_factor_v + ids_len.data[0])]),
314 |             [1]
315 |           );
316 |         } else {
317 |           pos_factor = new Tensor(
318 |             "float16",
319 |             new Uint16Array([int64ToFloat16(history_len.data[0] + BigInt(1))]),
320 |             [1]
321 |           );
322 |         }
323 | 
324 |         logger.tensor("Updated pos_factor", pos_factor);
325 |         logger.groupEnd();
326 |       } else {
327 |         logger.groupCollapsed(`Regular step ${num_decode} adjustments:`);
328 |         history_len = history_len.add(BigInt(1));
329 |         pos_factor = pos_factor.map((v) =>
330 |           int64ToFloat16(float16ToInt64(v) + BigInt(1))
331 |         );
332 |         logger.tensor("Updated history_len", history_len);
333 |         logger.tensor("Updated pos_factor", pos_factor);
334 |         logger.groupEnd();
335 |       }
336 |       input_ids.data[0] = Number(token_id.data[0]);
337 |       logger.tensor("Updated input_ids", input_ids);
338 | 
339 |       logger.groupCollapsed("Getting new hidden states...");
340 |       const result_B = await ortSessionB.run({
341 |         input_ids: input_ids,
342 |         ids_len: ids_len,
343 |       });
344 |       hidden_states = result_B.hidden_states;
345 |       logger.tensor("New hidden_states", hidden_states);
346 |       logger.groupEnd();
347 | 
348 |       if (
349 |         !Number.isInteger(token_id.data[0]) &&
350 |         !["bigint", "number"].includes(typeof token_id.data[0])
351 |       ) {
352 |         throw new Error(`Token ID is not an integer`);
353 |       } else {
354 |         // Decode token
355 |         const decoded = tokenizer.decode([...token_id.data]);
356 |         logger.log(`Decoded token: ${decoded}`);
357 |         accumulated += decoded;
358 |         callbacks.onToken(decoded);
359 |         await scheduler?.scheduler?.yield();
360 |       }
361 | 
362 |       logger.groupEnd();
363 |     }
364 | 
365 |     const generationTime = (performance.now() - generationStartTime) / 1000;
366 |     logger.groupCollapsed("[PERFORMANCE] Generation complete:");
367 |     logger.log(`  Total tokens generated: ${num_decode}`);
368 |     logger.log(`  Generation time: ${generationTime.toFixed(2)}s`);
369 |     logger.log(`  Speed: ${(num_decode / generationTime).toFixed(3)} tokens/s`);
370 |     logger.groupEnd();
371 |     callbacks.onComplete(accumulated);
372 |   }
373 | }
374 | 
375 | class WebLLMBackend {
376 |   constructor(modelId, config) {
377 |     this.modelId = modelId;
378 |     this.config = config;
379 |   }
380 | 
381 |   async warmup() {
382 |     if (!this.warmupPromise) {
383 |       this.warmupPromise = (async () => {
384 |         console.log('Creating new engine');
385 |         this.engine = await CreateMLCEngine(this.modelId);
386 |         return this.engine;
387 |       })();
388 |     }
389 |     return this.warmupPromise;
390 |   }
391 | 
392 |   async generate(prompt, systemPrompt, callbacks) {
393 |     const messages = [
394 |       { role: "system", content: systemPrompt },
395 |       { role: "user", content: prompt }
396 |     ]
397 | 
398 |     const asyncChunkGenerator = await this.engine.chat.completions.create({
399 |       messages,
400 |       stream: true,
401 |       ...this.config
402 |     })
403 | 
404 |     let accumulated = ""
405 |     for await (const chunk of asyncChunkGenerator) {
406 |       const newContent = chunk.choices[0]?.delta?.content || ""
407 |       accumulated += newContent
408 |       callbacks.onToken(newContent)
409 |     }
410 | 
411 |     callbacks.onComplete(accumulated)
412 |   }
413 | }
414 | 
415 | class HuggingFaceBackend {
416 |   constructor(modelId, options, config) {
417 |     this.modelId = modelId
418 |     this.options = options
419 |     this.config = config
420 |   }
421 | 
422 |   async generate(prompt, systemPrompt, callbacks) {
423 |     const model = await AutoModel.from_pretrained(this.modelId, this.options)
424 |     const tokenizer = await AutoTokenizer.from_pretrained(this.modelId)
425 | 
426 |     const messages = [
427 |       { role: "system", content: systemPrompt },
428 |       { role: "user", content: prompt }
429 |     ]
430 | 
431 |     const text_inputs = tokenizer.apply_chat_template(messages, {
432 |       tokenize: true,
433 |       add_generation_prompt: true,
434 |       return_dict: true,
435 |       padding: true,
436 |       truncation: true
437 |     })
438 | 
439 |     let accumulated = ""
440 |     const textStreamer = new TextStreamer(tokenizer, {
441 |       skip_prompt: true,
442 |       skip_special_tokens: true
443 |     })
444 | 
445 |     textStreamer.on_finalized_text = text => {
446 |       accumulated += text
447 |       callbacks.onToken(text)
448 |     }
449 | 
450 |     let isError = false
451 | 
452 |     try {
453 |       await model.generate({
454 |         ...text_inputs,
455 |         ...this.config,
456 |         streamer: textStreamer
457 |       })
458 |     } catch (error) {
459 |       isError = true
460 |       callbacks.onError(error);
461 |       console.error(error);
462 |     }
463 | 
464 |     if (!isError) {
465 |       callbacks.onComplete(accumulated)
466 |     }
467 |   }
468 | }
469 | 
470 | export function useLLMGeneration(
471 |   modelConfig,
472 |   systemPrompt,
473 |   backend = "webllm"
474 | ) {
475 |   const [isGenerating, setIsGenerating] = React.useState(false)
476 |   const [error, setError] = React.useState(null)
477 |   const [partialText, setPartialText] = React.useState("")
478 |   const [fullText, setFullText] = React.useState("")
479 | 
480 |   const backendRef = React.useRef(null)
481 | 
482 |   React.useEffect(() => {
483 |     if (backend === "qwen2vl" && modelConfig.qwen2vl) {
484 |       backendRef.current = new Qwen2VLBackend(
485 |         modelConfig.qwen2vl.modelId,
486 |         modelConfig.qwen2vl.generation
487 |       )
488 |     }
489 |     if (backend === "webllm" && modelConfig.webllm) {
490 |       backendRef.current = new WebLLMBackend(
491 |         modelConfig.webllm.modelId,
492 |         modelConfig.webllm.generation
493 |       )
494 |     } else if (backend === "huggingface" && modelConfig.huggingface) {
495 |       backendRef.current = new HuggingFaceBackend(
496 |         modelConfig.huggingface.modelId,
497 |         modelConfig.huggingface.options,
498 |         modelConfig.huggingface.generation
499 |       )
500 |     }
501 |   }, [backend, modelConfig])
502 | 
503 |   const warmup = React.useCallback(async () => {
504 |     const callbacks = {
505 |       onToken: () => { },
506 |       onComplete: () => { },
507 |       onError: () => { }
508 |     };
509 |     await backendRef.current.warmup();
510 |   });
511 | 
512 |   const generate = React.useCallback(
513 |     async (prompt, extras = {}, config = {}) => {
514 |       if (!backendRef.current) {
515 |         throw new Error(`No backend configured for ${backend}`)
516 |       }
517 | 
518 |       setIsGenerating(true)
519 |       setError(null)
520 |       setPartialText("")
521 |       setFullText("")
522 | 
523 |       const callbacks = {
524 |         onToken: token => {
525 |           setPartialText(prev => prev + token)
526 |         },
527 |         onComplete: text => {
528 |           setFullText(text)
529 |           setIsGenerating(false)
530 |         },
531 |         onError: err => {
532 |           setError(err)
533 |           setIsGenerating(false)
534 |         }
535 |       }
536 | 
537 |       backendRef.config = { ...backendRef.config, ...config };
538 | 
539 |       try {
540 |         await backendRef.current.warmup();
541 |         await backendRef.current.generate(prompt, systemPrompt, callbacks, extras)
542 |       } catch (err) {
543 |         callbacks.onError(err)
544 |         logger.error(err);
545 |       }
546 |     },
547 |     [backend, systemPrompt]
548 |   )
549 | 
550 |   return {
551 |     generate,
552 |     warmup,
553 |     isGenerating,
554 |     error,
555 |     partialText,
556 |     fullText
557 |   }
558 | }
559 | 


--------------------------------------------------------------------------------