14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
30 |
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/lib/types.d.ts.map:
--------------------------------------------------------------------------------
1 | {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAEhD;;;GAGG;AACH,MAAM,WAAW,WAAY,SAAQ,OAAO,CAAC,UAAU,CAAC;CAAI;AAE5D;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IAEjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,kBAAkB,KAAK,IAAI,CAAC;AAExE;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;AAEtF;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;;;;;;OAQG;IACH,uBAAuB,EAAE,CAAC,oBAAoB,EAAE,oBAAoB,KAAK,IAAI,CAAC;IAE9E;;;;;;;;OAQG;IACH,MAAM,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,GAAG,QAAQ,KAAK,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC;IAEnI;;;;;;;OAOG;IACH,QAAQ,EAAE,CACR,KAAK,EAAE,MAAM,EACb,gBAAgB,CAAC,EAAE,wBAAwB,EAC3C,cAAc,CAAC,EAAE,MAAM,KACpB,OAAO,CAAC,MAAM,CAAC,CAAC;IAErB;;;OAGG;IACH,gBAAgB,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,iBAAiB,EAAE,MAAM,IAAI,CAAC;IAE9B;;OAEG;IACH,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B;;OAEG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B;;;OAGG;IACH,8BAA8B,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAElD;;;OAGG;IACH,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACjC"}
--------------------------------------------------------------------------------
/examples/chrome-extension/src/background.ts:
--------------------------------------------------------------------------------
1 | import {ChatRestModule} from "@mlc-ai/web-llm";
2 |
3 | // TODO: Surface this as an option to the user
4 | const useWebGPU = true;
5 |
6 | var cm: ChatRestModule;
7 | if (!useWebGPU) {
8 | cm = new ChatRestModule();
9 | }
10 |
11 | // Set reponse callback for chat module
12 | const generateProgressCallback = (_step: number, message: string) => {
13 | // send the answer back to the content script
14 | chrome.runtime.sendMessage({ answer: message });
15 | };
16 |
17 | var context = "";
18 | chrome.runtime.onMessage.addListener(async function (request) {
19 | // check if the request contains a message that the user sent a new message
20 | if (request.input) {
21 | var inp = request.input;
22 | if (context.length > 0) {
23 | inp = "Use only the following context when answering the question at the end. Don't use any other knowledge.\n"+ context + "\n\nQuestion: " + request.input + "\n\nHelpful Answer: ";
24 | }
25 | console.log("Input:", inp);
26 | const response = await cm.generate(inp, generateProgressCallback);
27 | }
28 | if (request.context) {
29 | context = request.context;
30 | console.log("Got context:", context);
31 | }
32 | });
33 |
--------------------------------------------------------------------------------
/examples/simple-chat/src/mlc-local-config.js:
--------------------------------------------------------------------------------
1 | // config used when serving from local mlc-llm/dist
2 | // use web-llm/script/serve_mlc_llm_dist.sh to start the artifact server
3 | export default {
4 | "model_list": [
5 | {
6 | "model_url": "http://localhost:8000/RedPajama-INCITE-Chat-3B-v1-q4f32_1/params/",
7 | "local_id": "RedPajama-INCITE-Chat-3B-v1-q4f32_1",
8 | "model_lib_url": "http://localhost:8000/RedPajama-INCITE-Chat-3B-v1-q4f32_1/RedPajama-INCITE-Chat-3B-v1-q4f32_1-webgpu.wasm",
9 | },
10 | {
11 | "model_url": "http://localhost:8000/Llama-2-7b-chat-hf-q4f32_1/params/",
12 | "local_id": "Llama-2-7b-chat-hf-q4f32_1",
13 | "model_lib_url": "http://localhost:8000/Llama-2-7b-chat-hf-q4f32_1/Llama-2-7b-chat-hf-q4f32_1-webgpu.wasm",
14 | },
15 | // fp16 options are enabled through chrome canary flags
16 | // chrome --enable-dawn-features=enable_unsafe_apis
17 | {
18 | "model_url": "http://localhost:8000/RedPajama-INCITE-Chat-3B-v1-q4f16_1/params/",
19 | "local_id": "RedPajama-INCITE-Chat-3B-v1-q4f16_1",
20 | "model_lib_url": "http://localhost:8000/RedPajama-INCITE-Chat-3B-v1-q4f16_1/RedPajama-INCITE-Chat-3B-v1-q4f16_1-webgpu.wasm",
21 | "required_features": ["shader-f16"]
22 | }
23 | ],
24 | "use_web_worker": true
25 | }
26 |
--------------------------------------------------------------------------------
/lib/conversation.d.ts:
--------------------------------------------------------------------------------
1 | import { ConvTemplateConfig } from "./config";
2 | /**
3 | * Helper to keep track of history conversations.
4 | */
5 | export declare class Conversation {
6 | messages: Array<[string, string | undefined]>;
7 | config: ConvTemplateConfig;
8 | constructor(config: ConvTemplateConfig);
9 | private getPromptArrayInternal;
10 | /**
11 | * Get prompt arrays with the first one as system.
12 | *
13 | * @returns The prompt array.
14 | */
15 | getPromptArray(): Array
25 |
26 |
28 |
29 |
44 |
52 |
53 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
54 |
55 |
56 |
57 |
--------------------------------------------------------------------------------
/lib/config.d.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Conversation template config
3 | */
4 | export interface ConvTemplateConfig {
5 | system: string;
6 | roles: Array
26 |
36 |
37 |
89 | );
90 | };
91 |
92 | export default ChatComponent;
93 |
--------------------------------------------------------------------------------
/examples/next-simple-chat/src/styles/globals.css:
--------------------------------------------------------------------------------
1 | @tailwind base;
2 | @tailwind components;
3 | @tailwind utilities;
4 |
5 | :root {
6 | --foreground-rgb: 0, 0, 0;
7 | --background-start-rgb: 214, 219, 220;
8 | --background-end-rgb: 255, 255, 255;
9 | }
10 |
11 | @media (prefers-color-scheme: dark) {
12 | :root {
13 | --foreground-rgb: 255, 255, 255;
14 | --background-start-rgb: 0, 0, 0;
15 | --background-end-rgb: 0, 0, 0;
16 | }
17 | }
18 |
19 | body {
20 | color: rgb(var(--foreground-rgb));
21 | background: linear-gradient(
22 | to bottom,
23 | transparent,
24 | rgb(var(--background-end-rgb))
25 | )
26 | rgb(var(--background-start-rgb));
27 | }
28 |
29 | a {
30 | color: inherit;
31 | text-decoration: none;
32 | }
33 |
34 | * {
35 | box-sizing: border-box;
36 | }
37 |
38 | chatui-chat {
39 | height: 100;
40 | }
41 |
42 | .chatui {
43 | display: flex;
44 | flex-flow: column wrap;
45 | justify-content: space-between;
46 | width: 100%;
47 | max-width: 867px;
48 | margin: 25px 10px;
49 | height: 600px;
50 | border: 2px solid #ddd;
51 | border-radius: 5px;
52 | box-shadow: 0 15px 15px -5px rgba(0, 0, 0, 0.2);
53 | }
54 |
55 | s .chatui-header {
56 | display: flex;
57 | justify-content: space-between;
58 | padding: 10px;
59 | border-bottom: 2px solid #ddd;
60 | background: #eee;
61 | color: #666;
62 | }
63 |
64 | .chatui-chat {
65 | flex: 1;
66 | overflow-y: auto;
67 | padding: 10px;
68 | }
69 |
70 | .chatui-chat::-webkit-scrollbar {
71 | width: 6px;
72 | }
73 |
74 | .chatui-chat::-webkit-scrollbar-track {
75 | background: #ddd;
76 | }
77 |
78 | .chatui-chat::-webkit-scrollbar-thumb {
79 | background: #bdbdbd;
80 | }
81 |
82 | .msg {
83 | display: flex;
84 | align-items: flex-end;
85 | margin-bottom: 10px;
86 | }
87 |
88 | .msg:last-of-type {
89 | margin: 0;
90 | }
91 |
92 | .msg-bubble {
93 | max-width: 450px;
94 | padding: 15px;
95 | border-radius: 15px;
96 | background: #ececec;
97 | }
98 |
99 | .left-msg .msg-bubble {
100 | border-bottom-left-radius: 0;
101 | }
102 |
103 | .error-msg .msg-bubble {
104 | border-bottom-left-radius: 0;
105 | color: #f15959;
106 | }
107 |
108 | .init-msg .msg-bubble {
109 | border-bottom-left-radius: 0;
110 | }
111 |
112 | .right-msg {
113 | flex-direction: row-reverse;
114 | }
115 |
116 | .right-msg .msg-bubble {
117 | background: #579ffb;
118 | color: #fff;
119 | border-bottom-right-radius: 0;
120 | }
121 |
122 | .chatui-inputarea {
123 | display: flex;
124 | padding: 10px;
125 | border-top: 2px solid #ddd;
126 | background: #eee;
127 | }
128 |
129 | .chatui-inputarea * {
130 | padding: 10px;
131 | border: none;
132 | border-radius: 3px;
133 | font-size: 1em;
134 | }
135 |
136 | .chatui-input {
137 | flex: 1;
138 | background: #ddd;
139 | }
140 |
141 | .chatui-btn {
142 | margin-left: 10px;
143 | background: #579ffb;
144 | color: #fff;
145 | font-weight: bold;
146 | cursor: pointer;
147 | padding: 10px;
148 | }
149 |
150 | .chatui-btn:hover {
151 | background: #577bfb;
152 | }
153 |
154 | .chatui-chat {
155 | background-color: #fcfcfe;
156 | }
157 |
--------------------------------------------------------------------------------
/lib/types.d.ts:
--------------------------------------------------------------------------------
1 | import { AppConfig, ChatConfig } from "./config";
2 | /**
3 | * Custom options that can be used to
4 | * override known config values.
5 | */
6 | export interface ChatOptions extends Partial
38 |
76 |
77 |
39 | {messages.map((value, index) => (
40 |
47 |
48 |
41 |
45 | ))}
46 |
42 |
44 | ${value.text}
43 |
49 | {
55 | if (event.key === "Enter") {
56 | chat_ui
57 | .onGenerate(prompt, updateMessage, setRuntimeStats)
58 | .catch((error) => console.log(error));
59 | }
60 | }}
61 | value={prompt}
62 | onChange={(event) => setPrompt(event.target.value)}
63 | />
64 |
74 |
75 |
78 |
86 |
87 |
88 | 
25 |
26 | We have been seeing amazing progress in generative AI and LLM recently. Thanks to the open-source efforts like LLaMA, Alpaca, Vicuna and Dolly, we start to see an exciting future of building our own open source language models and personal AI assistant.
27 |
28 | These models are usually big and compute-heavy. To build a chat service, we will need a large cluster to run an inference server, while clients send requests to servers and retrieve the inference output. We also usually have to run on a specific type of GPUs where popular deep-learning frameworks are readily available.
29 |
30 | This project is our step to bring more diversity to the ecosystem. Specifically, can we simply bake LLMs directly into the client side and directly run them inside a browser? If that can be realized, we could offer support for client personal AI models with the benefit of cost reduction, enhancement for personalization and privacy protection. The client side is getting pretty powerful.
31 |
32 | Won’t it be even more amazing if we can simply open up a browser and directly bring AI natively to your browser tab? There is some level of readiness in the ecosystem. This project provides an affirmative answer to the question.
33 |
34 |
35 | ## Instructions
36 |
37 | WebGPU just shipped in Chrome 113.
38 |
39 | Select the model you want to try out. Enter your inputs, click “Send” – we are ready to go!
40 | The chat bot will first fetch model parameters into local cache. The download may take a few minutes, only for the first run.
41 | The subsequent refreshes and runs will be faster. We have tested it on Windows and Mac, you will need a GPU with about 6GB memory to run Llama-7B, Vicuna-7B, and about 3GB memory to run RedPajama-3B.
42 |
43 | ## Chat Demo
44 |
45 | The chat demo is based on [Llama 2](https://ai.meta.com/llama/), [Mistral-7B](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) and its variants, and [RedPajama-INCITE-Chat-3B-v1](https://huggingface.co/togethercomputer/RedPajama-INCITE-Chat-3B-v1) model. More model supports are on the way.
46 |
47 |
48 |
49 |
50 |
51 | ## Links
52 |
53 | - [Web LLM GitHub](https://github.com/mlc-ai/web-llm)
54 | - You might also be interested in [Web Stable Diffusion](https://websd.mlc.ai/).
55 |
56 | ## Disclaimer
57 |
58 | This demo site is for research purposes only, subject to the model License of LLaMA, Vicuna and RedPajama. Please contact us if you find any potential violation.
59 |
--------------------------------------------------------------------------------
/web/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 | # TVM WebAssembly Runtime
19 |
20 | This folder contains TVM WebAssembly Runtime.
21 |
22 | ## Installation
23 |
24 | The LLVM main branch support webassembly as a target, we can directly
25 | build TVM with LLVM mainline to generate wasm modules.
26 | Note that, however, we still need emscripten to compile the runtime and provide system library support.
27 |
28 | Note that so far we requires everything to be in the source and setup PYTHONPATH(instead of use setup.py install).
29 |
30 | ### Setup Emscripten
31 |
32 | We use emscripten to compile our runtime wasm library as well as a WASI variant that we can deploy
33 | to the browser environment.
34 |
35 | Follow [Emscripten](https://emscripten.org/) to download emsdk and install emcc on your local environment.
36 |
37 | ### Build TVM Wasm Runtime
38 |
39 | After the emcc is setup correctly. We can build tvm's wasm runtime by typing `make` in the web folder.
40 |
41 | ```bash
42 | make
43 | ```
44 |
45 | This command will create the follow files:
46 | - `dist/wasm/libtvm_runtime.bc` bitcode library `tvm.contrib.emcc` will link into.
47 | - `dist/wasm/tvmjs_runtime.wasm` a standalone wasm runtime for testing purposes.
48 | - `dist/wasm/tvmjs_runtime.wasi.js` a WASI compatible library generated by emscripten that can be fed into runtime.
49 |
50 |
51 | ### Build TVM Wasm JS Frontend
52 |
53 | Type the following command in the web folder.
54 |
55 | ```bash
56 | npm run bundle
57 | ```
58 |
59 | This command will create the tvmjs library that we can use to interface with the wasm runtime.
60 |
61 |
62 | ## Use TVM to Generate Wasm Library and Run it
63 |
64 | Check code snippet in
65 |
66 | - [tests/python/prepare_test_libs.py](https://github.com/apache/tvm/tree/main/web/tests/python/prepare_test_libs.py)
67 | shows how to create a wasm library that links with tvm runtime.
68 | - Note that all wasm libraries have to created using the `--system-lib` option
69 | - emcc.create_wasm will automatically link the runtime library `dist/wasm/libtvm_runtime.bc`
70 | - [tests/web/test_module_load.js](https://github.com/apache/tvm/tree/main/web/tests/node/test_module_load.js) demonstrate
71 | how to run the generated library through tvmjs API.
72 |
73 |
74 | ## Run Wasm Remotely through WebSocket RPC.
75 |
76 | We can now use js side to start an RPC server and connect to it from python side,
77 | making the testing flow easier.
78 |
79 | The following is an example to reproduce this.
80 | - run `python -m tvm.exec.rpc_proxy --example-rpc=1` to start proxy.
81 | - Start the WebSocket RPC
82 | - Browswer version: open https://localhost:8888, click connect to proxy
83 | - NodeJS version: `npm run rpc`
84 | - run `python tests/python/websock_rpc_test.py` to run the rpc test.
85 |
86 |
87 | ## WebGPU Experiments
88 |
89 | Web gpu is still experimental, so apis can change.
90 | Right now we use the SPIRV to generate shaders that can be accepted by Chrome and Firefox.
91 |
92 | - Obtain a browser that support webgpu.
93 | - So far only Chrome Canary on MacOS works
94 | - Firefox should be close pending the support of Fence.
95 | - Download vulkan SDK (1.1 or higher) that supports SPIRV 1.3
96 | - Start the WebSocket RPC
97 | - run `python tests/node/webgpu_rpc_test.py`
98 |
--------------------------------------------------------------------------------
/utils/vram_requirements/src/vram_requirements.ts:
--------------------------------------------------------------------------------
1 | import ModelRecord from "@mlc-ai/web-llm";
2 | import appConfig from "./app-config"; // Modify this to inspect vram requirement for models of choice
3 | import * as tvmjs from "tvmjs";
4 |
5 | function setLabel(id: string, text: string) {
6 | const label = document.getElementById(id);
7 | if (label == null) {
8 | throw Error("Cannot find label " + id);
9 | }
10 | label.innerText = text;
11 | }
12 |
13 | interface AppConfig {
14 | model_list: ArrayTVM WebSocket RPC Server
97 | To use this page 98 |-
99 |
- Run "make" and "npm run bundle" to create the libraries. 100 |
- 101 | run "python -m tvm.exec.rpc_proxy --example-rpc=1" to start proxy. 102 | 103 |
- Click Connect to proxy. 104 |
- run "python tests/python/websock_rpc_test.py" to run the rpc client. 105 |
Options
108 | Proxy URL114 | RPC Server Key
120 | NDArrayCache - 121 | 124 | CacheDevice - 125 | 129 |
130 | 131 | 132 |
133 |
134 | 135 |
136 |
137 |
138 |
139 |
140 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | dist/
2 | params/
3 | *.bak
4 | # Byte-compiled / optimized / DLL files
5 | __pycache__/
6 | *.py[cod]
7 | *$py.class
8 |
9 | *.S
10 | # C extensions
11 | *.so
12 |
13 |
14 | *.ll
15 | .npm
16 | # Distribution / packaging
17 | .Python
18 | env/
19 | build/
20 | build-*/
21 | develop-eggs/
22 | dist/
23 | downloads/
24 | eggs/
25 | .eggs/
26 | web/lib/
27 | lib64/
28 | parts/
29 | sdist/
30 | var/
31 | wheels/
32 | pip-wheel-metadata/
33 | share/python-wheels/
34 | *.egg-info/
35 | .installed.cfg
36 | *.egg
37 | MANIFEST
38 |
39 | .conda/
40 | # PyInstaller
41 | # Usually these files are written by a python script from a template
42 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
43 | *.manifest
44 | *.spec
45 |
46 | # Generated by python/gen_requirements.py
47 | python/requirements/*.txt
48 |
49 | # Installer logs
50 | pip-log.txt
51 | pip-delete-this-directory.txt
52 |
53 | # Unit test / coverage reports
54 | htmlcov/
55 | .tox/
56 | .nox/
57 | .coverage
58 | .coverage.*
59 | .cache
60 | nosetests.xml
61 | coverage.xml
62 | *.cover
63 | *.py,cover
64 | .hypothesis/
65 | .pytest_cache/
66 |
67 | # Translations
68 | *.mo
69 | *.pot
70 |
71 | # Django stuff:
72 | *.log
73 | local_settings.py
74 | db.sqlite3
75 | db.sqlite3-journal
76 |
77 | # Flask stuff:
78 | instance/
79 | .webassets-cache
80 |
81 | # Scrapy stuff:
82 | .scrapy
83 |
84 | # Sphinx documentation
85 | docs/_build/
86 | docs/_staging/
87 |
88 | # PyBuilder
89 | target/
90 | /target/
91 |
92 | # Jupyter Notebook
93 | .ipynb_checkpoints
94 |
95 | # IPython
96 | profile_default/
97 | ipython_config.py
98 |
99 | # pyenv
100 | .python-version
101 |
102 | # pipenv
103 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
104 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
105 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
106 | # install all needed dependencies.
107 | #Pipfile.lock
108 |
109 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
110 | __pypackages__/
111 |
112 | # Celery stuff
113 | celerybeat-schedule
114 | celerybeat.pid
115 |
116 | # SageMath parsed files
117 | *.sage.py
118 |
119 | # Environments
120 | .env
121 | .venv
122 | env/
123 | venv/
124 | ENV/
125 | env.bak/
126 | venv.bak/
127 |
128 | # Spyder project settings
129 | .spyderproject
130 | .spyproject
131 |
132 | # Rope project settings
133 | .ropeproject
134 | *~
135 | *.pyc
136 | *~
137 | config.mk
138 | config.cmake
139 | Win32
140 | *.dir
141 | perf
142 | *.wasm
143 | .emscripten
144 |
145 | ## IOS
146 | DerivedData/
147 |
148 | ## Java
149 | *.class
150 | jvm/*/target/
151 | jvm/*/*/target/
152 | jvm/native/*/generated
153 | jvm/native/src/main/native/org_apache_tvm_native_c_api.h
154 | *.worksheet
155 | *.idea
156 | *.iml
157 | *.classpath
158 | *.project
159 | *.settings
160 | */node_modules/
161 |
162 | ## Various settings
163 | *.pbxuser
164 | !default.pbxuser
165 | *.mode1v3
166 | !default.mode1v3
167 | *.mode2v3
168 | !default.mode2v3
169 | *.perspectivev3
170 | !default.perspectivev3
171 | xcuserdata/
172 | .pkl_memoize_*
173 |
174 | .emscripten*
175 | .m2
176 |
177 | # Compiled Dynamic libraries
178 | *.so
179 | *.dylib
180 | *.dll
181 |
182 | # Compiled Object files
183 | *.slo
184 | *.lo
185 | *.o
186 | *.obj
187 |
188 | # Precompiled Headers
189 | *.gch
190 | *.pch
191 |
192 | # Compiled Static libraries
193 | *.lai
194 | *.la
195 | *.a
196 | *.lib
197 |
198 | # Executables
199 | *.exe
200 | *.out
201 | *.app
202 |
203 | ## Other
204 | *.moved-aside
205 | *.xccheckout
206 | *.xcscmblueprint
207 | .DS_Store
208 | tags
209 | cscope*
210 | *.lock
211 |
212 | # vim temporary files
213 | *.swp
214 | *.swo
215 |
216 | # TVM generated code
217 | perf
218 | .bash_history
219 | # *.json
220 | *.params
221 | *.ro
222 | *.onnx
223 | *.h5
224 | synset.txt
225 | cat.jpg
226 | cat.png
227 | docs.tgz
228 | cat.png
229 | *.mlmodel
230 | tvm_u.*
231 | tvm_t.*
232 | # Mac OS X
233 | .DS_Store
234 |
235 | # Jetbrain
236 | .idea
237 | .ipython
238 | .jupyter
239 | .nv
240 | .pylint.d
241 | .python_history
242 | .pytest_cache
243 | .local
244 | cmake-build-debug
245 |
246 | # Visual Studio
247 | .vs
248 |
249 | # Visual Studio Code
250 | .vscode
251 |
252 | # tmp file
253 | .nfs*
254 |
255 | # keys
256 | *.pem
257 | *.p12
258 | *.pfx
259 | *.cer
260 | *.crt
261 | *.der
262 |
263 | # patch sentinel
264 | patched.txt
265 |
266 | # Python type checking
267 | .mypy_cache/
268 | .pyre/
269 |
270 | # pipenv files
271 | Pipfile
272 | Pipfile.lock
273 |
274 | # conda package artifacts
275 | conda/Dockerfile.cuda*
276 | conda/pkg
277 | .node_repl_history
278 | # nix files
279 | .envrc
280 | *.nix
281 |
282 | # Docker files
283 | .sudo_as_admin_successful
284 |
285 | # Downloaded models/datasets
286 | .tvm_test_data
287 | .dgl
288 | .caffe2
289 |
290 | # Local docs build
291 | _docs/
292 | jvm/target
293 | .config/configstore/
294 | .ci-py-scripts/
295 |
296 | # Generated Hexagon files
297 | src/runtime/hexagon/rpc/hexagon_rpc.h
298 | src/runtime/hexagon/rpc/hexagon_rpc_skel.c
299 | src/runtime/hexagon/rpc/hexagon_rpc_stub.c
300 |
301 | # Local tvm-site checkout
302 | tvm-site/
303 |
304 | # Generated docs files
305 | gallery/how_to/work_with_microtvm/micro_tvmc.py
306 |
307 | # Test sample data files
308 | !tests/python/ci/sample_prs/*.json
309 |
310 | # Used in CI to communicate between Python and Jenkins
311 | .docker-image-names/
312 |
313 | # Printed TIR code on disk
314 | *.tir
315 |
316 | # GDB history file
317 | .gdb_history
318 |
319 | 3rdparty
320 | dist
321 | tvm_home
322 | node_modules
323 | .parcel-cache
324 |
--------------------------------------------------------------------------------
/web/tests/node/test_packed_func.js:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing,
13 | * software distributed under the License is distributed on an
14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | * KIND, either express or implied. See the License for the
16 | * specific language governing permissions and limitations
17 | * under the License.
18 | */
19 | /* eslint-disable no-undef */
20 | const path = require("path");
21 | const fs = require("fs");
22 | const assert = require("assert");
23 | const tvmjs = require("../../dist/tvmjs.bundle")
24 |
25 | const wasmPath = tvmjs.wasmPath();
26 | const wasmSource = fs.readFileSync(path.join(wasmPath, "tvmjs_runtime.wasm"));
27 |
28 | let tvm = new tvmjs.Instance(
29 | new WebAssembly.Module(wasmSource),
30 | tvmjs.createPolyfillWASI()
31 | );
32 |
33 |
34 | test("GetGlobal", () => {
35 | tvm.beginScope();
36 | let flist = tvm.listGlobalFuncNames();
37 | let faddOne = tvm.getGlobalFunc("testing.add_one");
38 | let fecho = tvm.getGlobalFunc("testing.echo");
39 |
40 | assert(faddOne(tvm.scalar(1, "int")) == 2);
41 | assert(faddOne(tvm.scalar(-1, "int")) == 0);
42 |
43 | // check function argument with different types.
44 | assert(fecho(1123) == 1123);
45 | assert(fecho("xyz") == "xyz");
46 |
47 | let bytes = new Uint8Array([1, 2, 3]);
48 | let rbytes = fecho(bytes);
49 | assert(rbytes.length == bytes.length);
50 |
51 | for (let i = 0; i < bytes.length; ++i) {
52 | assert(rbytes[i] == bytes[i]);
53 | }
54 |
55 | assert(fecho(undefined) == undefined);
56 |
57 | tvm.beginScope();
58 |
59 | let arr = tvm.empty([2, 2]).copyFrom([1, 2, 3, 4]);
60 | let arr2 = fecho(arr);
61 | assert(arr.getHandle() == arr2.getHandle());
62 | assert(arr2.toArray().toString() == arr.toArray().toString());
63 |
64 | tvm.moveToParentScope(arr2);
65 | tvm.endScope();
66 | // test move to parent scope and tracking
67 | assert(arr.getHandle(false) == 0);
68 | assert(arr2.handle != 0);
69 |
70 | let mod = tvm.systemLib();
71 | let ret = fecho(mod);
72 | assert(ret.getHandle() == mod.getHandle());
73 | assert(flist.length != 0);
74 | tvm.endScope();
75 |
76 | // assert auto release scope behavior
77 | assert(mod.getHandle(false) == 0);
78 | assert(ret.getHandle(false) == 0);
79 | assert(arr2.getHandle(false) == 0);
80 | assert(fecho._tvmPackedCell.getHandle(false) == 0);
81 | assert(faddOne._tvmPackedCell.getHandle(false) == 0);
82 | });
83 |
84 | test("ReturnFunc", () => {
85 | tvm.beginScope();
86 | function addy(y) {
87 | function add(x, z) {
88 | return x + y + z;
89 | }
90 | return add;
91 | }
92 |
93 | let fecho = tvm.getGlobalFunc("testing.echo");
94 | let myf = tvm.toPackedFunc(addy);
95 | assert(tvm.isPackedFunc(myf));
96 | let myf2 = tvm.toPackedFunc(myf);
97 | assert(myf2._tvmPackedCell.handle === myf._tvmPackedCell.handle);
98 | let f = myf(10);
99 |
100 | assert(tvm.isPackedFunc(f));
101 | assert(f(11, 0) == 21);
102 | assert(f("x", 1) == "x101");
103 | assert(f("x", "yz") == "x10yz");
104 |
105 | fecho.dispose();
106 | myf.dispose();
107 | myf2.dispose();
108 | // test multiple dispose.
109 | f.dispose();
110 | f.dispose();
111 | tvm.endScope();
112 | });
113 |
114 | test("RegisterGlobal", () => {
115 | tvm.beginScope();
116 | tvm.registerFunc("xyz", function (x, y) {
117 | return x + y;
118 | });
119 |
120 | let f = tvm.getGlobalFunc("xyz");
121 | assert(f(1, 2) == 3);
122 | f.dispose();
123 |
124 | let syslib = tvm.systemLib();
125 | syslib.dispose();
126 | tvm.endScope();
127 | });
128 |
129 | test("ExceptionPassing", () => {
130 | tvm.beginScope();
131 | tvm.registerFunc("throw_error", function (msg) {
132 | throw Error(msg);
133 | });
134 | let f = tvm.getGlobalFunc("throw_error");
135 | try {
136 | f("error-xyz");
137 | throw Error("error not caught");
138 | } catch (error) {
139 | assert(error.message.indexOf("error-xyz") != -1);
140 | }
141 | tvm.endScope();
142 | });
143 |
144 | test("NDArrayCbArg", () => {
145 | tvm.beginScope();
146 | let use_count = tvm.getGlobalFunc("testing.object_use_count");
147 | let record = [];
148 |
149 | let fcheck = tvm.toPackedFunc(function (x, retain) {
150 | assert(use_count(x) == 2);
151 | assert(x.handle != 0);
152 | record.push(x);
153 | if (retain) {
154 | tvm.detachFromCurrentScope(x);
155 | }
156 | });
157 |
158 | let x = tvm.empty([2], "float32").copyFrom([1, 2]);
159 | assert(use_count(x) == 1);
160 |
161 | fcheck(x, 0);
162 | // auto-released when it is out of scope.
163 | assert(record[0].getHandle(false) == 0);
164 |
165 | assert(use_count(x) == 1);
166 |
167 | fcheck(x, 1);
168 | assert(use_count(x) == 2);
169 | assert(record[1].handle != 0);
170 | tvm.attachToCurrentScope(record[1]);
171 | tvm.endScope();
172 | assert(record[1].getHandle(false) == 0);
173 | });
174 |
175 | test("Logging", () => {
176 | tvm.beginScope();
177 | const log_info = tvm.getGlobalFunc("testing.log_info_str");
178 | log_info("helow world")
179 | log_info.dispose();
180 | tvm.endScope();
181 | });
182 |
--------------------------------------------------------------------------------
/web/emcc/wasm_runtime.cc:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing,
13 | * software distributed under the License is distributed on an
14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | * KIND, either express or implied. See the License for the
16 | * specific language governing permissions and limitations
17 | * under the License.
18 | */
19 |
20 | /*
21 | * \file wasm_runtime.cc
22 | * \brief TVM wasm runtime library pack.
23 | */
24 |
25 | // configurations for tvm logging
26 | #define TVM_LOG_STACK_TRACE 0
27 | #define TVM_LOG_DEBUG 0
28 | #define TVM_LOG_CUSTOMIZE 1
29 |
30 | #define DMLC_USE_LOGGING_LIBRARY 134 | 135 |
'); 142 | document.getElementById("answer")!.innerHTML = answerWithBreaks; 143 | // Add event listener to copy button 144 | document.getElementById("copyAnswer")!.addEventListener("click", () => { 145 | // Get the answer text 146 | const answerText = answer; 147 | // Copy the answer text to the clipboard 148 | navigator.clipboard.writeText(answerText) 149 | .then(() => console.log("Answer text copied to clipboard")) 150 | .catch((err) => console.error("Could not copy text: ", err)); 151 | }); 152 | const options: Intl.DateTimeFormatOptions = { month: 'short', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit' }; 153 | const time = new Date().toLocaleString('en-US', options); 154 | // Update timestamp 155 | document.getElementById("timestamp")!.innerText = time; 156 | // Hide loading indicator 157 | document.getElementById("loading-indicator")!.style.display = "none"; 158 | } 159 | 160 | function fetchPageContents() { 161 | chrome.tabs.query({ currentWindow: true, active: true }, function (tabs) { 162 | var port = chrome.tabs.connect(tabs[0].id, { name: "channelName" }); 163 | port.postMessage({}); 164 | port.onMessage.addListener(function (msg) { 165 | console.log("Page contents:", msg.contents); 166 | if (useWebGPU) { 167 | context = msg.contents 168 | } else { 169 | chrome.runtime.sendMessage({ context: msg.contents }); 170 | } 171 | }); 172 | }); 173 | } 174 | 175 | // Grab the page contents when the popup is opened 176 | window.onload = function () { 177 | if (!useWebGPU) { 178 | fetchPageContents(); 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /web/src/ctypes.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | * Types for C API. 22 | */ 23 | 24 | /** A pointer to points to the raw address space. */ 25 | export type Pointer = number; 26 | 27 | /** A pointer offset, need to add a base address to get a valid ptr. */ 28 | export type PtrOffset = number; 29 | 30 | // -- TVM runtime C API -- 31 | /** 32 | * const char *TVMGetLastError(); 33 | */ 34 | export type FTVMGetLastError = () => Pointer; 35 | 36 | /** 37 | * void TVMAPISetLastError(const char* msg); 38 | */ 39 | export type FTVMAPISetLastError = (msg: Pointer) => void; 40 | 41 | /** 42 | * int TVMModGetFunction(TVMModuleHandle mod, 43 | * const char* func_name, 44 | * int query_imports, 45 | * TVMFunctionHandle *out); 46 | */ 47 | export type FTVMModGetFunction = ( 48 | mod: Pointer, funcName: Pointer, queryImports: number, out: Pointer) => number; 49 | /** 50 | * int TVMModImport(TVMModuleHandle mod, 51 | * TVMModuleHandle dep); 52 | */ 53 | export type FTVMModImport = (mod: Pointer, dep: Pointer) => number; 54 | 55 | /** 56 | * int TVMModFree(TVMModuleHandle mod); 57 | */ 58 | export type FTVMModFree = (mod: Pointer) => number; 59 | 60 | /** 61 | * int TVMFuncFree(TVMFunctionHandle func); 62 | */ 63 | export type FTVMFuncFree = (func: Pointer) => number; 64 | 65 | /** 66 | * int TVMFuncCall(TVMFunctionHandle func, 67 | * TVMValue* arg_values, 68 | * int* type_codes, 69 | * int num_args, 70 | * TVMValue* ret_val, 71 | * int* ret_type_code); 72 | */ 73 | export type FTVMFuncCall = ( 74 | func: Pointer, argValues: Pointer, typeCode: Pointer, 75 | nargs: number, retValue: Pointer, retCode: Pointer) => number; 76 | 77 | /** 78 | * int TVMCFuncSetReturn(TVMRetValueHandle ret, 79 | * TVMValue* value, 80 | * int* type_code, 81 | * int num_ret); 82 | */ 83 | export type FTVMCFuncSetReturn = ( 84 | ret: Pointer, value: Pointer, typeCode: Pointer, numRet: number) => number; 85 | 86 | /** 87 | * int TVMCbArgToReturn(TVMValue* value, int* code); 88 | */ 89 | export type FTVMCbArgToReturn = (value: Pointer, code: Pointer) => number; 90 | 91 | /** 92 | * int TVMFuncListGlobalNames(int* outSize, const char*** outArray); 93 | */ 94 | export type FTVMFuncListGlobalNames = (outSize: Pointer, outArray: Pointer) => number; 95 | 96 | /** 97 | * int TVMFuncRegisterGlobal( 98 | * const char* name, TVMFunctionHandle f, int override); 99 | */ 100 | export type FTVMFuncRegisterGlobal = ( 101 | name: Pointer, f: Pointer, override: number) => number; 102 | 103 | /** 104 | *int TVMFuncGetGlobal(const char* name, TVMFunctionHandle* out); 105 | */ 106 | export type FTVMFuncGetGlobal = (name: Pointer, out: Pointer) => number; 107 | 108 | /** 109 | * int TVMArrayAlloc(const tvm_index_t* shape, 110 | * int ndim, 111 | * int dtype_code, 112 | * int dtype_bits, 113 | * int dtype_lanes, 114 | * int device_type, 115 | * int device_id, 116 | * TVMArrayHandle* out); 117 | */ 118 | export type FTVMArrayAlloc = ( 119 | shape: Pointer, ndim: number, 120 | dtypeCode: number, dtypeBits: number, 121 | dtypeLanes: number, deviceType: number, deviceId: number, 122 | out: Pointer) => number; 123 | 124 | /** 125 | * int TVMArrayFree(TVMArrayHandle handle); 126 | */ 127 | export type FTVMArrayFree = (handle: Pointer) => number; 128 | 129 | /** 130 | * int TVMArrayCopyFromBytes(TVMArrayHandle handle, 131 | * void* data, 132 | * size_t nbytes); 133 | */ 134 | export type FTVMArrayCopyFromBytes = ( 135 | handle: Pointer, data: Pointer, nbytes: number) => number; 136 | 137 | /** 138 | * int TVMArrayCopyToBytes(TVMArrayHandle handle, 139 | * void* data, 140 | * size_t nbytes); 141 | */ 142 | export type FTVMArrayCopyToBytes = ( 143 | handle: Pointer, data: Pointer, nbytes: number) => number; 144 | 145 | /** 146 | * int TVMArrayCopyFromTo(TVMArrayHandle from, 147 | * TVMArrayHandle to, 148 | * TVMStreamHandle stream); 149 | */ 150 | export type FTVMArrayCopyFromTo = ( 151 | from: Pointer, to: Pointer, stream: Pointer) => number; 152 | 153 | /** 154 | * int TVMSynchronize(int device_type, int device_id, TVMStreamHandle stream); 155 | */ 156 | export type FTVMSynchronize = ( 157 | deviceType: number, deviceId: number, stream: Pointer) => number; 158 | 159 | /** 160 | * typedef int (*TVMBackendPackedCFunc)(TVMValue* args, 161 | * int* type_codes, 162 | * int num_args, 163 | * TVMValue* out_ret_value, 164 | * int* out_ret_tcode); 165 | */ 166 | export type FTVMBackendPackedCFunc = ( 167 | argValues: Pointer, argCodes: Pointer, nargs: number, 168 | outValue: Pointer, outCode: Pointer) => number; 169 | 170 | 171 | /** 172 | * int TVMObjectFree(TVMObjectHandle obj); 173 | */ 174 | export type FTVMObjectFree = (obj: Pointer) => number; 175 | 176 | /** 177 | * int TVMObjectGetTypeIndex(TVMObjectHandle obj, unsigned* out_tindex); 178 | */ 179 | export type FTVMObjectGetTypeIndex = (obj: Pointer, out_tindex: Pointer) => number; 180 | 181 | /** 182 | * int TVMObjectTypeIndex2Key(unsigned tindex, char** out_type_key); 183 | */ 184 | export type FTVMObjectTypeIndex2Key = (type_index: number, out_type_key: Pointer) => number; 185 | 186 | /** 187 | * int TVMObjectTypeKey2Index(const char* type_key, unsigned* out_tindex); 188 | */ 189 | export type FTVMObjectTypeKey2Index = (type_key: Pointer, out_tindex: Pointer) => number; 190 | 191 | // -- TVM Wasm Auxiliary C API -- 192 | 193 | /** void* TVMWasmAllocSpace(int size); */ 194 | export type FTVMWasmAllocSpace = (size: number) => Pointer; 195 | 196 | /** void TVMWasmFreeSpace(void* data); */ 197 | export type FTVMWasmFreeSpace = (ptr: Pointer) => void; 198 | 199 | /** 200 | * int TVMWasmPackedCFunc(TVMValue* args, 201 | * int* type_codes, 202 | * int num_args, 203 | * TVMRetValueHandle ret, 204 | * void* resource_handle); 205 | */ 206 | export type FTVMWasmPackedCFunc = ( 207 | args: Pointer, typeCodes: Pointer, nargs: number, 208 | ret: Pointer, resourceHandle: Pointer) => number; 209 | 210 | /** 211 | * int TVMWasmFuncCreateFromCFunc(void* resource_handle, 212 | * TVMFunctionHandle *out); 213 | */ 214 | export type FTVMWasmFuncCreateFromCFunc = ( 215 | resource: Pointer, out: Pointer) => number; 216 | 217 | /** 218 | * void TVMWasmPackedCFuncFinalizer(void* resource_handle); 219 | */ 220 | export type FTVMWasmPackedCFuncFinalizer = (resourceHandle: Pointer) => void; 221 | 222 | /** 223 | * Size of common data types. 224 | */ 225 | export const enum SizeOf { 226 | U8 = 1, 227 | U16 = 2, 228 | I32 = 4, 229 | I64 = 8, 230 | F32 = 4, 231 | F64 = 8, 232 | TVMValue = 8, 233 | DLDataType = I32, 234 | DLDevice = I32 + I32, 235 | } 236 | 237 | /** 238 | * Argument Type code in TVM FFI. 239 | */ 240 | export const enum ArgTypeCode { 241 | Int = 0, 242 | UInt = 1, 243 | Float = 2, 244 | TVMOpaqueHandle = 3, 245 | Null = 4, 246 | TVMDataType = 5, 247 | DLDevice = 6, 248 | TVMDLTensorHandle = 7, 249 | TVMObjectHandle = 8, 250 | TVMModuleHandle = 9, 251 | TVMPackedFuncHandle = 10, 252 | TVMStr = 11, 253 | TVMBytes = 12, 254 | TVMNDArrayHandle = 13, 255 | TVMObjectRValueRefArg = 14 256 | } 257 | -------------------------------------------------------------------------------- /site/img/logo/catalyst.svg: -------------------------------------------------------------------------------- 1 | 2 | 119 | --------------------------------------------------------------------------------