├── .gitignore ├── GoEmotions.gif ├── LICENSE ├── README.md ├── TrainEmotions.ipynb ├── TrainGoEmotions.ipynb ├── package-lock.json ├── package.json ├── public ├── index.html ├── manifest.json ├── static │ ├── js │ │ ├── ort-wasm-simd-threaded.wasm │ │ ├── ort-wasm-simd.wasm │ │ ├── ort-wasm-threaded.wasm │ │ └── ort-wasm.wasm │ └── vocab.json ├── xtremedistil-int8.onnx └── xtremedistill-go-emotion-int8.onnx └── src ├── App.css ├── App.js ├── App.test.js ├── bert_tokenizer.ts ├── index.css ├── index.js ├── inference.js ├── reportWebVitals.js └── setupProxy.js /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /GoEmotions.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jobergum/browser-ml-inference/fed077a999ddfffd6d2358753d5b010dd91ac566/GoEmotions.gif -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2022] [Jo Kristian Bergum] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Text Emotion Prediction in Browser 2 | 3 | This React App demonstrates ML Inference in the Browser using 4 | 5 | - [Cloudflare Pages](https://pages.cloudflare.com/) to deliver the React app and model via worldwide Content Delivery Network (CDN) 6 | - [ONNX Runtime Web](https://onnxruntime.ai/) for model inference in the Browser 7 | - [Huggingface](https://huggingface.co/bergum/xtremedistil-l6-h384-go-emotion) for NLP model hosting and training API (Transformer library) 8 | - [Google Colab](https://colab.research.google.com/) for model training using GPU instances 9 | 10 | Live demo at [https://aiserv.cloud/](https://aiserv.cloud/). 11 | 12 |

13 | 14 |

15 | 16 | See also my blog post [Moving ML Inference from the Cloud to the Edge](https://bergum.medium.com/moving-ml-inference-from-the-cloud-to-the-edge-d6f98dbdb2e3?source=friends_link&sk=e8183a3a8c10077110952b213ba5bef4) and [Deploy Transformer Models in the Browser with #ONNXRuntime on YouTube](https://www.youtube.com/watch?v=W_lUGPMW_Eg). 17 | 18 | The emotion prediction model is a fine-tuned version of the pre-trained language model 19 | [microsoft/xtremedistil-l6-h384-uncased](https://huggingface.co/microsoft/xtremedistil-l6-h384-uncased). 20 | The model has been fine-tuned on the 21 | [GoEmotions dataset](https://ai.googleblog.com/2021/10/goemotions-dataset-for-fine-grained.html) which is a multi-label 22 | text categorization problem. 23 | 24 | 25 | >GoEmotions, a human-annotated dataset of 58k Reddit comments extracted from popular English-language subreddits and labeled with 27 emotion categories . As the largest fully annotated English language fine-grained emotion dataset to date. In contrast to the basic six emotions, which include only one positive emotion (joy), the taxonomy includes 12 positive, 11 negative, 4 ambiguous emotion categories and 1 “neutral”, making it widely suitable for conversation understanding tasks that require a subtle differentiation between emotion expressions. 26 | 27 | Paper [GoEmotions: A Dataset of Fine-Grained Emotions](https://arxiv.org/pdf/2005.00547.pdf) 28 | 29 | - The fine-tuned model is hosted on [Huggingface:bergum/xtremedistil-l6-h384-go-emotion](https://huggingface.co/bergum/xtremedistil-l6-h384-go-emotion). 30 | - The `go_emotions` dataset is available on [Huggingface dataset hub](https://huggingface.co/datasets/go_emotions). 31 | 32 | See [TrainGoEmotions.ipynb](TrainGoEmotions.ipynb ) for how to train a model on the dataset and export the fine-tuned model to ONNX. 33 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/jobergum/emotion/blob/main/TrainGoEmotions.ipynb) 34 | 35 | ## ONNX-Runtime-web 36 | The model is quantized to `int8` weights and has 22M trainable parameters. 37 | 38 | Inference is multi-threaded. To use 39 | multiple inference threads, specific http headers must be presented by the CDN, see 40 | [Making your website "cross-origin isolated" using COOP and COEP](https://web.dev/coop-coep/). 41 | 42 | Three threads are used for inference. Due to this [bug](https://github.com/microsoft/onnxruntime/issues/11679) 43 | multi-threading and COOP headers had to be disabled as the model would silently fail to initialize on IOS devices. 44 | 45 | For development, the [src/setupProxy.js](src/setupProxy.js) adds the required headers. 46 | See [react issue 10210](https://github.com/facebook/create-react-app/issues/10210) 47 | 48 | ## Code Navigation 49 | - The App frontend logic is in [src/App.js](src/App.js) 50 | - The model inference logic is in [src/inference.js](src/inference.js) 51 | - The tokenizer is in [src/bert_tokenizer.js](src/bert_tokenizer.ts) which is a copy of [Google TFJS](https://raw.githubusercontent.com/tensorflow/tfjs-models/master/qna/src/bert_tokenizer.ts) (Apache 2.0) 52 | - Cloudflare header override for cross-origin coop policy to enable multi threaded inference [public/_header](public/_headers). 53 | 54 | ## Model and Language Biases 55 | The pre-trained language model was trained on text with biases, 56 | see [On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?](https://dl.acm.org/doi/10.1145/3442188.3445922) 57 | for a study on the dangers of pre-trained language models and transfer learning. 58 | 59 | From dataset paper [GoEmotions: A Dataset of Fine-Grained Emotions](https://arxiv.org/pdf/2005.00547.pdf): 60 | >Data Disclaimer: We are aware that the dataset 61 | contains biases and is not representative of global 62 | diversity. We are aware that the dataset contains 63 | potentially problematic content. Potential biases in 64 | the data include: Inherent biases in Reddit and user 65 | base biases, the offensive/vulgar word lists used 66 | for data filtering, inherent or unconscious bias in 67 | assessment of offensive identity labels, annotators 68 | were all native English speakers from India. All 69 | these likely affect labeling, precision, and recall 70 | for a trained model. The emotion pilot model used 71 | for sentiment labeling, was trained on examples 72 | reviewed by the research team. Anyone using this 73 | dataset should be aware of these limitations of the 74 | dataset. 75 | 76 | ## Running this app 77 | Install Node.js/npm, see [Installing Node.js](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) 78 | 79 | In the project directory, you can run: 80 | 81 | ### `npm start` 82 | 83 | Runs the app in the development mode.\ 84 | Open [http://localhost:3000](http://localhost:3000) to view it in your browser. 85 | 86 | The page will reload when you make changes.\ 87 | You may also see any lint errors in the console. 88 | 89 | ### `npm run build` 90 | 91 | Builds the app for production to the `build` folder.\ 92 | It correctly bundles React in production mode and optimizes the build for the best performance. 93 | 94 | ### Deploying app 95 | Clone this repo and use [Cloudflare Pages](https://pages.cloudflare.com/). 96 | 97 | ## TODO 98 | - Fix build to copy wasm files from node_modules to build to avoid having wasm files under source control. 99 | - PR and feedback welcome - create an issue to get in contact. 100 | 101 | -------------------------------------------------------------------------------- /TrainEmotions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "TrainEmotions.ipynb", 7 | "provenance": [] 8 | }, 9 | "kernelspec": { 10 | "name": "python3", 11 | "display_name": "Python 3" 12 | }, 13 | "language_info": { 14 | "name": "python" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": { 23 | "id": "itanOMBN5EWg" 24 | }, 25 | "outputs": [], 26 | "source": [ 27 | "%pip install datasets transformers onnx onnxruntime " 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "source": [ 33 | "We use the small distilled BERT model from Microsoft as our pre-trained model which we fine-tune on the emotion classification task. \n", 34 | "See https://huggingface.co/microsoft/xtremedistil-l6-h256-uncased for details. " 35 | ], 36 | "metadata": { 37 | "id": "HqlT_xzmz2gt" 38 | } 39 | }, 40 | { 41 | "cell_type": "code", 42 | "source": [ 43 | "model_name = 'microsoft/xtremedistil-l6-h256-uncased'" 44 | ], 45 | "metadata": { 46 | "id": "kg76LB_ey0Zi" 47 | }, 48 | "execution_count": 2, 49 | "outputs": [] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "source": [ 54 | "from datasets import load_dataset\n", 55 | "dataset = load_dataset(\"emotion\")\n", 56 | "from transformers import AutoTokenizer\n", 57 | "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", 58 | "def tokenize_function(examples):\n", 59 | " return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True, max_length=128)\n", 60 | "tokenized_datasets = dataset.map(tokenize_function, batched=True)" 61 | ], 62 | "metadata": { 63 | "id": "olHJ-ItY5R6t" 64 | }, 65 | "execution_count": null, 66 | "outputs": [] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "source": [ 71 | "full_train_dataset = tokenized_datasets[\"train\"]\n", 72 | "full_eval_dataset = tokenized_datasets[\"test\"]" 73 | ], 74 | "metadata": { 75 | "id": "LpBYSTs15h27" 76 | }, 77 | "execution_count": 4, 78 | "outputs": [] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "source": [ 83 | "import torch\n", 84 | "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n", 85 | "print(device)" 86 | ], 87 | "metadata": { 88 | "colab": { 89 | "base_uri": "https://localhost:8080/" 90 | }, 91 | "id": "XbT3wxcL6qen", 92 | "outputId": "6f8797ee-e883-4be9-e726-3ad38a62f96a" 93 | }, 94 | "execution_count": 7, 95 | "outputs": [ 96 | { 97 | "output_type": "stream", 98 | "name": "stdout", 99 | "text": [ 100 | "cuda:0\n" 101 | ] 102 | } 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "source": [ 108 | "from transformers import AutoModelForSequenceClassification\n", 109 | "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=6)\n", 110 | "model = model.to(device)" 111 | ], 112 | "metadata": { 113 | "id": "k1pOQHBh5l6l" 114 | }, 115 | "execution_count": null, 116 | "outputs": [] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "source": [ 121 | "import numpy as np\n", 122 | "from datasets import load_metric\n", 123 | "\n", 124 | "metric = load_metric(\"accuracy\")\n", 125 | "def compute_metrics(eval_pred):\n", 126 | " logits, labels = eval_pred\n", 127 | " predictions = np.argmax(logits, axis=-1)\n", 128 | " return metric.compute(predictions=predictions, references=labels)" 129 | ], 130 | "metadata": { 131 | "id": "tBCo1LY17sYS" 132 | }, 133 | "execution_count": 20, 134 | "outputs": [] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "source": [ 139 | "from transformers import TrainingArguments\n", 140 | "training_args = TrainingArguments(\"test_trainer\",\n", 141 | " per_device_train_batch_size=128, \n", 142 | " num_train_epochs=24,learning_rate=3e-05,\n", 143 | " evaluation_strategy=\"epoch\")\n", 144 | "from transformers import Trainer\n", 145 | "trainer = Trainer(\n", 146 | " model=model,\n", 147 | " args=training_args,\n", 148 | " train_dataset=full_train_dataset,\n", 149 | " eval_dataset=full_eval_dataset,\n", 150 | " compute_metrics=compute_metrics,\n", 151 | ")" 152 | ], 153 | "metadata": { 154 | "id": "S8AryVRy6xb5", 155 | "colab": { 156 | "base_uri": "https://localhost:8080/" 157 | }, 158 | "outputId": "8b38f9db-ced7-42e0-9002-dc12205815de" 159 | }, 160 | "execution_count": 21, 161 | "outputs": [ 162 | { 163 | "output_type": "stream", 164 | "name": "stderr", 165 | "text": [ 166 | "PyTorch: setting up devices\n", 167 | "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n" 168 | ] 169 | } 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "source": [ 175 | "trainer.train()" 176 | ], 177 | "metadata": { 178 | "id": "uXHzn8CH53Hh" 179 | }, 180 | "execution_count": null, 181 | "outputs": [] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "source": [ 186 | "trainer.evaluate()" 187 | ], 188 | "metadata": { 189 | "colab": { 190 | "base_uri": "https://localhost:8080/", 191 | "height": 210 192 | }, 193 | "id": "VC_x9_-p8LmO", 194 | "outputId": "ce714416-66f3-4408-9d02-f43eb7513e67" 195 | }, 196 | "execution_count": 23, 197 | "outputs": [ 198 | { 199 | "output_type": "stream", 200 | "name": "stderr", 201 | "text": [ 202 | "The following columns in the evaluation set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: text.\n", 203 | "***** Running Evaluation *****\n", 204 | " Num examples = 2000\n", 205 | " Batch size = 8\n" 206 | ] 207 | }, 208 | { 209 | "output_type": "display_data", 210 | "data": { 211 | "text/html": [ 212 | "\n", 213 | "
\n", 214 | " \n", 215 | " \n", 216 | " [250/250 00:04]\n", 217 | "
\n", 218 | " " 219 | ], 220 | "text/plain": [ 221 | "" 222 | ] 223 | }, 224 | "metadata": {} 225 | }, 226 | { 227 | "output_type": "execute_result", 228 | "data": { 229 | "text/plain": [ 230 | "{'epoch': 24.0,\n", 231 | " 'eval_accuracy': 0.9265,\n", 232 | " 'eval_loss': 0.22581592202186584,\n", 233 | " 'eval_runtime': 5.0402,\n", 234 | " 'eval_samples_per_second': 396.809,\n", 235 | " 'eval_steps_per_second': 49.601}" 236 | ] 237 | }, 238 | "metadata": {}, 239 | "execution_count": 23 240 | } 241 | ] 242 | }, 243 | { 244 | "cell_type": "markdown", 245 | "source": [ 246 | "Export PyTorch model to ONNX format for serving with ONNX Runtime Web " 247 | ], 248 | "metadata": { 249 | "id": "hdTnHS5Ma0Hc" 250 | } 251 | }, 252 | { 253 | "cell_type": "code", 254 | "source": [ 255 | "import transformers\n", 256 | "import transformers.convert_graph_to_onnx as onnx_convert\n", 257 | "from pathlib import Path" 258 | ], 259 | "metadata": { 260 | "id": "gw5w-O0YCpbm" 261 | }, 262 | "execution_count": 24, 263 | "outputs": [] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "source": [ 268 | "pipeline = transformers.pipeline(\"text-classification\",model=model,tokenizer=tokenizer)" 269 | ], 270 | "metadata": { 271 | "id": "e46GMI9FGYV_" 272 | }, 273 | "execution_count": 25, 274 | "outputs": [] 275 | }, 276 | { 277 | "cell_type": "code", 278 | "source": [ 279 | "model = model.to(\"cpu\")" 280 | ], 281 | "metadata": { 282 | "id": "KT-lBe5hHD0U" 283 | }, 284 | "execution_count": 26, 285 | "outputs": [] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "source": [ 290 | "onnx_convert.convert_pytorch(pipeline, opset=11, output=Path(\"classifier.onnx\"), use_external_format=False)" 291 | ], 292 | "metadata": { 293 | "id": "lbYsOZheCwTu" 294 | }, 295 | "execution_count": null, 296 | "outputs": [] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "source": [ 301 | "%pip install onnx onnxruntime " 302 | ], 303 | "metadata": { 304 | "colab": { 305 | "base_uri": "https://localhost:8080/" 306 | }, 307 | "id": "61pSD5KPIgYp", 308 | "outputId": "af9f32ae-d716-42ae-bbda-af2a4e65f9f7" 309 | }, 310 | "execution_count": 71, 311 | "outputs": [ 312 | { 313 | "output_type": "stream", 314 | "name": "stdout", 315 | "text": [ 316 | "Requirement already satisfied: onnx in /usr/local/lib/python3.7/dist-packages (1.10.2)\n", 317 | "Requirement already satisfied: onnxruntime in /usr/local/lib/python3.7/dist-packages (1.10.0)\n", 318 | "Requirement already satisfied: numpy>=1.16.6 in /usr/local/lib/python3.7/dist-packages (from onnx) (1.19.5)\n", 319 | "Requirement already satisfied: typing-extensions>=3.6.2.1 in /usr/local/lib/python3.7/dist-packages (from onnx) (3.10.0.2)\n", 320 | "Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from onnx) (3.17.3)\n", 321 | "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from onnx) (1.15.0)\n", 322 | "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.7/dist-packages (from onnxruntime) (2.0)\n" 323 | ] 324 | } 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "source": [ 330 | "from onnxruntime.quantization import quantize_dynamic, QuantType\n", 331 | "quantize_dynamic(\"classifier.onnx\", \"classifier_int8.onnx\", \n", 332 | " weight_type=QuantType.QUInt8)" 333 | ], 334 | "metadata": { 335 | "id": "IB_nkkDQ7OO2" 336 | }, 337 | "execution_count": 72, 338 | "outputs": [] 339 | }, 340 | { 341 | "cell_type": "markdown", 342 | "source": [ 343 | "Evaluate accuracy using ONNX-Runtime inference - validate PyTorch inference versus ONNX-Runtime " 344 | ], 345 | "metadata": { 346 | "id": "KyLlyMMoa-E9" 347 | } 348 | }, 349 | { 350 | "cell_type": "code", 351 | "source": [ 352 | "import onnxruntime as ort" 353 | ], 354 | "metadata": { 355 | "id": "64GP3FbC3Puz" 356 | }, 357 | "execution_count": 61, 358 | "outputs": [] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "source": [ 363 | "session = ort.InferenceSession(\"classifier.onnx\")\n", 364 | "session_int8 = ort.InferenceSession(\"classifier_int8.onnx\")" 365 | ], 366 | "metadata": { 367 | "id": "ojSj8awa3Rd6" 368 | }, 369 | "execution_count": 73, 370 | "outputs": [] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "source": [ 375 | "import numpy as np" 376 | ], 377 | "metadata": { 378 | "id": "OeLLbPWl36Xt" 379 | }, 380 | "execution_count": 74, 381 | "outputs": [] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "source": [ 386 | "input_feed = {\n", 387 | " \"input_ids\": np.array(full_eval_dataset['input_ids']),\n", 388 | " \"attention_mask\": np.array(full_eval_dataset['attention_mask']),\n", 389 | " \"token_type_ids\": np.array(full_eval_dataset['token_type_ids'])\n", 390 | "}" 391 | ], 392 | "metadata": { 393 | "id": "vRM83eOd3Y7M" 394 | }, 395 | "execution_count": 75, 396 | "outputs": [] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "source": [ 401 | "out = session.run(input_feed=input_feed,output_names=['output_0'])[0]\n", 402 | "out_int8 = session_int8.run(input_feed=input_feed,output_names=['output_0'])[0]" 403 | ], 404 | "metadata": { 405 | "id": "1w5QMJSm4GW1" 406 | }, 407 | "execution_count": 76, 408 | "outputs": [] 409 | }, 410 | { 411 | "cell_type": "code", 412 | "source": [ 413 | "predictions = np.argmax(out, axis=-1)\n", 414 | "predictions_int8 = np.argmax(out_int8, axis=-1)" 415 | ], 416 | "metadata": { 417 | "id": "YC9E5iIu4W4U" 418 | }, 419 | "execution_count": 77, 420 | "outputs": [] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "source": [ 425 | "metric.compute(predictions=predictions, references=full_eval_dataset['label'])" 426 | ], 427 | "metadata": { 428 | "colab": { 429 | "base_uri": "https://localhost:8080/" 430 | }, 431 | "id": "W3LmHcyK4ndB", 432 | "outputId": "72162b8d-01a2-498c-96ca-8360ef26af8c" 433 | }, 434 | "execution_count": 78, 435 | "outputs": [ 436 | { 437 | "output_type": "execute_result", 438 | "data": { 439 | "text/plain": [ 440 | "{'accuracy': 0.9265}" 441 | ] 442 | }, 443 | "metadata": {}, 444 | "execution_count": 78 445 | } 446 | ] 447 | }, 448 | { 449 | "cell_type": "code", 450 | "source": [ 451 | "metric.compute(predictions=predictions_int8, references=full_eval_dataset['label'])" 452 | ], 453 | "metadata": { 454 | "colab": { 455 | "base_uri": "https://localhost:8080/" 456 | }, 457 | "id": "FznKYHhb56Dv", 458 | "outputId": "5cc77516-3a19-4b77-8df5-c36db68c98dc" 459 | }, 460 | "execution_count": 79, 461 | "outputs": [ 462 | { 463 | "output_type": "execute_result", 464 | "data": { 465 | "text/plain": [ 466 | "{'accuracy': 0.8195}" 467 | ] 468 | }, 469 | "metadata": {}, 470 | "execution_count": 79 471 | } 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "source": [ 477 | "from google.colab import files" 478 | ], 479 | "metadata": { 480 | "id": "aCJyzWtzIyad" 481 | }, 482 | "execution_count": 31, 483 | "outputs": [] 484 | }, 485 | { 486 | "cell_type": "code", 487 | "source": [ 488 | "files.download('classifier_int8.onnx') " 489 | ], 490 | "metadata": { 491 | "colab": { 492 | "base_uri": "https://localhost:8080/", 493 | "height": 17 494 | }, 495 | "id": "tIHal-FKI0Zw", 496 | "outputId": "4a60645e-3793-40de-a0d0-b9a5d612bfb2" 497 | }, 498 | "execution_count": 80, 499 | "outputs": [ 500 | { 501 | "output_type": "display_data", 502 | "data": { 503 | "application/javascript": [ 504 | "\n", 505 | " async function download(id, filename, size) {\n", 506 | " if (!google.colab.kernel.accessAllowed) {\n", 507 | " return;\n", 508 | " }\n", 509 | " const div = document.createElement('div');\n", 510 | " const label = document.createElement('label');\n", 511 | " label.textContent = `Downloading \"${filename}\": `;\n", 512 | " div.appendChild(label);\n", 513 | " const progress = document.createElement('progress');\n", 514 | " progress.max = size;\n", 515 | " div.appendChild(progress);\n", 516 | " document.body.appendChild(div);\n", 517 | "\n", 518 | " const buffers = [];\n", 519 | " let downloaded = 0;\n", 520 | "\n", 521 | " const channel = await google.colab.kernel.comms.open(id);\n", 522 | " // Send a message to notify the kernel that we're ready.\n", 523 | " channel.send({})\n", 524 | "\n", 525 | " for await (const message of channel.messages) {\n", 526 | " // Send a message to notify the kernel that we're ready.\n", 527 | " channel.send({})\n", 528 | " if (message.buffers) {\n", 529 | " for (const buffer of message.buffers) {\n", 530 | " buffers.push(buffer);\n", 531 | " downloaded += buffer.byteLength;\n", 532 | " progress.value = downloaded;\n", 533 | " }\n", 534 | " }\n", 535 | " }\n", 536 | " const blob = new Blob(buffers, {type: 'application/binary'});\n", 537 | " const a = document.createElement('a');\n", 538 | " a.href = window.URL.createObjectURL(blob);\n", 539 | " a.download = filename;\n", 540 | " div.appendChild(a);\n", 541 | " a.click();\n", 542 | " div.remove();\n", 543 | " }\n", 544 | " " 545 | ], 546 | "text/plain": [ 547 | "" 548 | ] 549 | }, 550 | "metadata": {} 551 | }, 552 | { 553 | "output_type": "display_data", 554 | "data": { 555 | "application/javascript": [ 556 | "download(\"download_4348d76c-9dbd-4a50-9150-4a7d9a6d97f2\", \"classifier-quantized.onnx\", 13088747)" 557 | ], 558 | "text/plain": [ 559 | "" 560 | ] 561 | }, 562 | "metadata": {} 563 | } 564 | ] 565 | }, 566 | { 567 | "cell_type": "code", 568 | "source": [ 569 | "files.download('classifier.onnx')" 570 | ], 571 | "metadata": { 572 | "colab": { 573 | "base_uri": "https://localhost:8080/", 574 | "height": 17 575 | }, 576 | "id": "BEBqzJAf9oEe", 577 | "outputId": "38a35837-9843-4bc1-f181-9830fa42b81e" 578 | }, 579 | "execution_count": 81, 580 | "outputs": [ 581 | { 582 | "output_type": "display_data", 583 | "data": { 584 | "application/javascript": [ 585 | "\n", 586 | " async function download(id, filename, size) {\n", 587 | " if (!google.colab.kernel.accessAllowed) {\n", 588 | " return;\n", 589 | " }\n", 590 | " const div = document.createElement('div');\n", 591 | " const label = document.createElement('label');\n", 592 | " label.textContent = `Downloading \"${filename}\": `;\n", 593 | " div.appendChild(label);\n", 594 | " const progress = document.createElement('progress');\n", 595 | " progress.max = size;\n", 596 | " div.appendChild(progress);\n", 597 | " document.body.appendChild(div);\n", 598 | "\n", 599 | " const buffers = [];\n", 600 | " let downloaded = 0;\n", 601 | "\n", 602 | " const channel = await google.colab.kernel.comms.open(id);\n", 603 | " // Send a message to notify the kernel that we're ready.\n", 604 | " channel.send({})\n", 605 | "\n", 606 | " for await (const message of channel.messages) {\n", 607 | " // Send a message to notify the kernel that we're ready.\n", 608 | " channel.send({})\n", 609 | " if (message.buffers) {\n", 610 | " for (const buffer of message.buffers) {\n", 611 | " buffers.push(buffer);\n", 612 | " downloaded += buffer.byteLength;\n", 613 | " progress.value = downloaded;\n", 614 | " }\n", 615 | " }\n", 616 | " }\n", 617 | " const blob = new Blob(buffers, {type: 'application/binary'});\n", 618 | " const a = document.createElement('a');\n", 619 | " a.href = window.URL.createObjectURL(blob);\n", 620 | " a.download = filename;\n", 621 | " div.appendChild(a);\n", 622 | " a.click();\n", 623 | " div.remove();\n", 624 | " }\n", 625 | " " 626 | ], 627 | "text/plain": [ 628 | "" 629 | ] 630 | }, 631 | "metadata": {} 632 | }, 633 | { 634 | "output_type": "display_data", 635 | "data": { 636 | "application/javascript": [ 637 | "download(\"download_51111a37-468e-4268-b45f-ed0d63dbd54c\", \"classifier.onnx\", 51044981)" 638 | ], 639 | "text/plain": [ 640 | "" 641 | ] 642 | }, 643 | "metadata": {} 644 | } 645 | ] 646 | } 647 | ] 648 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serv", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@emotion/react": "^11.7.1", 7 | "@emotion/styled": "^11.6.0", 8 | "@mui/material": "^5.2.7", 9 | "@testing-library/jest-dom": "^5.16.1", 10 | "@testing-library/react": "^12.1.2", 11 | "@testing-library/user-event": "^13.5.0", 12 | "onnxruntime-web": "^1.11.0", 13 | "react": "^17.0.2", 14 | "react-dom": "^17.0.2", 15 | "react-google-charts": "^3.0.15", 16 | "react-scripts": "^5.0.1", 17 | "typescript": "^4.5.4", 18 | "wasm-feature-detect": "^1.2.11", 19 | "web-vitals": "^2.1.2" 20 | }, 21 | "scripts": { 22 | "start": "react-scripts start", 23 | "build": "react-scripts build", 24 | "test": "react-scripts test", 25 | "eject": "react-scripts eject" 26 | }, 27 | "eslintConfig": { 28 | "extends": [ 29 | "react-app", 30 | "react-app/jest" 31 | ] 32 | }, 33 | "browserslist": { 34 | "production": [ 35 | ">0.2%", 36 | "not dead", 37 | "not op_mini all" 38 | ], 39 | "development": [ 40 | "last 1 chrome version", 41 | "last 1 firefox version", 42 | "last 1 safari version" 43 | ] 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | ML Inference in the Browser 14 | 15 | 16 | 17 |
18 | 19 | 20 | -------------------------------------------------------------------------------- /public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "Emotion", 3 | "name": "Inference with Transformers in the Browser", 4 | "icons": [], 5 | "start_url": ".", 6 | "display": "standalone", 7 | "theme_color": "#000000", 8 | "background_color": "#ffffff" 9 | } 10 | -------------------------------------------------------------------------------- /public/static/js/ort-wasm-simd-threaded.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jobergum/browser-ml-inference/fed077a999ddfffd6d2358753d5b010dd91ac566/public/static/js/ort-wasm-simd-threaded.wasm -------------------------------------------------------------------------------- /public/static/js/ort-wasm-simd.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jobergum/browser-ml-inference/fed077a999ddfffd6d2358753d5b010dd91ac566/public/static/js/ort-wasm-simd.wasm -------------------------------------------------------------------------------- /public/static/js/ort-wasm-threaded.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jobergum/browser-ml-inference/fed077a999ddfffd6d2358753d5b010dd91ac566/public/static/js/ort-wasm-threaded.wasm -------------------------------------------------------------------------------- /public/static/js/ort-wasm.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jobergum/browser-ml-inference/fed077a999ddfffd6d2358753d5b010dd91ac566/public/static/js/ort-wasm.wasm -------------------------------------------------------------------------------- /public/xtremedistil-int8.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jobergum/browser-ml-inference/fed077a999ddfffd6d2358753d5b010dd91ac566/public/xtremedistil-int8.onnx -------------------------------------------------------------------------------- /public/xtremedistill-go-emotion-int8.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jobergum/browser-ml-inference/fed077a999ddfffd6d2358753d5b010dd91ac566/public/xtremedistill-go-emotion-int8.onnx -------------------------------------------------------------------------------- /src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | .App-textarea { 11 | -webkit-border-radius: 7px; 12 | -moz-border-radius: 7px; 13 | border-radius: 7px; 14 | background-color: #34ebc0; 15 | font-size: calc(10px + 2vmin); 16 | color: #282c34; 17 | } 18 | 19 | .App-header { 20 | background-color: #282c34; 21 | min-height: 100vh; 22 | display: flex; 23 | flex-direction: column; 24 | align-items: center; 25 | justify-content: center; 26 | font-size: calc(10px + 2vmin); 27 | color: white; 28 | } 29 | 30 | .App-link { 31 | color: #61dafb; 32 | } 33 | -------------------------------------------------------------------------------- /src/App.js: -------------------------------------------------------------------------------- 1 | import './App.css'; 2 | 3 | import React from 'react'; 4 | import {inference} from './inference.js'; 5 | import {columnNames} from './inference.js'; 6 | import {modelDownloadInProgress} from './inference.js'; 7 | import Chart from "react-google-charts"; 8 | import Box from '@mui/material/Box'; 9 | import LinearProgress from '@mui/material/LinearProgress'; 10 | 11 | class TextInputArea extends React.Component { 12 | constructor(props) { 13 | super(props); 14 | this.state = { 15 | text: 'Enter text to classify emotion, model trained on English text.', 16 | data:columnNames, 17 | latency:0.0, 18 | downloading:modelDownloadInProgress() 19 | }; 20 | this.handleChange = this.handleChange.bind(this); 21 | } 22 | 23 | componentDidMount() { 24 | this.timerID = setInterval( 25 | () => this.checkModelStatus(), 26 | 1000 27 | ); 28 | } 29 | 30 | checkModelStatus() { 31 | this.setState({ 32 | downloading: modelDownloadInProgress(), 33 | }); 34 | if (!this.state.downloading) { 35 | this.timerID = setInterval( 36 | () => this.checkModelStatus, 37 | 5000000 38 | ); 39 | } 40 | } 41 | 42 | handleChange(event) { 43 | inference(event.target.value).then( result => { 44 | this.setState({ 45 | text : event.target.value, 46 | data:result[1], 47 | latency:result[0], 48 | }); 49 | }); 50 | } 51 | 52 | render() { 53 | return ( 54 |
55 |
56 | In-Browser Transformer Inference 57 | 85 | 86 | {this.state.downloading && 87 |
Downloading model from CDN to browser.. 88 | 89 | 90 | 91 |

92 |
93 | } 94 | 97 |
Inference Latency {this.state.latency} ms
98 | 99 |
Model was trained on the GoEmotions dataset.
100 |
101 |
102 | ); 103 | } 104 | } 105 | export default TextInputArea; 106 | -------------------------------------------------------------------------------- /src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import App from './App'; 3 | 4 | test('renders learn react link', () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /src/bert_tokenizer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This tokenizer is a copy of 3 | * https://raw.githubusercontent.com/tensorflow/tfjs-models/master/qna/src/bert_tokenizer.ts 4 | * with minor modifications (Removed all tfjs dependencies) 5 | * 6 | * Copyright 2020 Google LLC. All Rights Reserved. 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | * ============================================================================= 19 | */ 20 | 21 | const SEPERATOR = '\u2581'; 22 | export const UNK_INDEX = 100; 23 | export const CLS_INDEX = 101; 24 | export const CLS_TOKEN = '[CLS]'; 25 | export const SEP_INDEX = 102; 26 | export const SEP_TOKEN = '[SEP]'; 27 | export const NFKC_TOKEN = 'NFKC'; 28 | export const VOCAB_URL = './static/vocab.json'; 29 | 30 | /** 31 | * Class for represent node for token parsing Trie data structure. 32 | */ 33 | class TrieNode { 34 | parent: TrieNode; 35 | children: {[key: string]: TrieNode} = {}; 36 | end = false; 37 | score: number; 38 | index: number; 39 | constructor(private key: string) {} 40 | 41 | getWord(): [string[], number, number] { 42 | const output: string[] = []; 43 | let node: TrieNode = this; 44 | 45 | while (node != null) { 46 | if (node.key != null) { 47 | output.unshift(node.key); 48 | } 49 | node = node.parent; 50 | } 51 | 52 | return [output, this.score, this.index]; 53 | } 54 | } 55 | 56 | class Trie { 57 | private root = new TrieNode(null); 58 | 59 | /** 60 | * Insert the bert vacabulary word into the trie. 61 | * @param word word to be inserted. 62 | * @param score word score. 63 | * @param index index of word in the bert vocabulary file. 64 | */ 65 | insert(word: string, score: number, index: number) { 66 | let node = this.root; 67 | 68 | const symbols = []; 69 | for (const symbol of word) { 70 | symbols.push(symbol); 71 | } 72 | 73 | for (let i = 0; i < symbols.length; i++) { 74 | if (node.children[symbols[i]] == null) { 75 | node.children[symbols[i]] = new TrieNode(symbols[i]); 76 | node.children[symbols[i]].parent = node; 77 | } 78 | 79 | node = node.children[symbols[i]]; 80 | 81 | if (i === symbols.length - 1) { 82 | node.end = true; 83 | node.score = score; 84 | node.index = index; 85 | } 86 | } 87 | } 88 | 89 | /** 90 | * Find the Trie node for the given token, it will return the first node that 91 | * matches the subtoken from the beginning of the token. 92 | * @param token string, input string to be searched. 93 | */ 94 | find(token: string): TrieNode { 95 | let node = this.root; 96 | let iter = 0; 97 | 98 | while (iter < token.length && node != null) { 99 | node = node.children[token[iter]]; 100 | iter++; 101 | } 102 | 103 | return node; 104 | } 105 | } 106 | 107 | function isWhitespace(ch: string): boolean { 108 | return /\s/.test(ch); 109 | } 110 | 111 | function isInvalid(ch: string): boolean { 112 | return (ch.charCodeAt(0) === 0 || ch.charCodeAt(0) === 0xfffd); 113 | } 114 | 115 | const punctuations = '[~`!@#$%^&*(){}[];:"\'<,.>?/\\|-_+='; 116 | 117 | /** To judge whether it's a punctuation. */ 118 | function isPunctuation(ch: string): boolean { 119 | return punctuations.indexOf(ch) !== -1; 120 | } 121 | 122 | export interface Token { 123 | text: string; 124 | index: number; 125 | } 126 | /** 127 | * Tokenizer for Bert. 128 | */ 129 | export class BertTokenizer { 130 | private vocab: string[]; 131 | private trie: Trie; 132 | 133 | /** 134 | * Load the vacabulary file and initialize the Trie for lookup. 135 | */ 136 | async load() { 137 | this.vocab = await this.loadVocab(); 138 | 139 | this.trie = new Trie(); 140 | // Actual tokens start at 999. 141 | for (let vocabIndex = 999; vocabIndex < this.vocab.length; vocabIndex++) { 142 | const word = this.vocab[vocabIndex]; 143 | this.trie.insert(word, 1, vocabIndex); 144 | } 145 | } 146 | 147 | private async loadVocab(): Promise<[]> { 148 | return fetch(VOCAB_URL).then(d => d.json()); 149 | } 150 | 151 | processInput(text: string): Token[] { 152 | const charOriginalIndex: number[] = []; 153 | const cleanedText = this.cleanText(text, charOriginalIndex); 154 | const origTokens = cleanedText.split(' '); 155 | 156 | let charCount = 0; 157 | const tokens = origTokens.map((token) => { 158 | token = token.toLowerCase(); 159 | const tokens = this.runSplitOnPunc(token, charCount, charOriginalIndex); 160 | charCount += token.length + 1; 161 | return tokens; 162 | }); 163 | 164 | let flattenTokens: Token[] = []; 165 | for (let index = 0; index < tokens.length; index++) { 166 | flattenTokens = flattenTokens.concat(tokens[index]); 167 | } 168 | return flattenTokens; 169 | } 170 | 171 | /* Performs invalid character removal and whitespace cleanup on text. */ 172 | private cleanText(text: string, charOriginalIndex: number[]): string { 173 | const stringBuilder: string[] = []; 174 | let originalCharIndex = 0, newCharIndex = 0; 175 | for (const ch of text) { 176 | // Skip the characters that cannot be used. 177 | if (isInvalid(ch)) { 178 | originalCharIndex += ch.length; 179 | continue; 180 | } 181 | if (isWhitespace(ch)) { 182 | if (stringBuilder.length > 0 && 183 | stringBuilder[stringBuilder.length - 1] !== ' ') { 184 | stringBuilder.push(' '); 185 | charOriginalIndex[newCharIndex] = originalCharIndex; 186 | originalCharIndex += ch.length; 187 | } else { 188 | originalCharIndex += ch.length; 189 | continue; 190 | } 191 | } else { 192 | stringBuilder.push(ch); 193 | charOriginalIndex[newCharIndex] = originalCharIndex; 194 | originalCharIndex += ch.length; 195 | } 196 | newCharIndex++; 197 | } 198 | return stringBuilder.join(''); 199 | } 200 | 201 | /* Splits punctuation on a piece of text. */ 202 | private runSplitOnPunc( 203 | text: string, count: number, 204 | charOriginalIndex: number[]): Token[] { 205 | const tokens: Token[] = []; 206 | let startNewWord = true; 207 | for (const ch of text) { 208 | if (isPunctuation(ch)) { 209 | tokens.push({text: ch, index: charOriginalIndex[count]}); 210 | count += ch.length; 211 | startNewWord = true; 212 | } else { 213 | if (startNewWord) { 214 | tokens.push({text: '', index: charOriginalIndex[count]}); 215 | startNewWord = false; 216 | } 217 | tokens[tokens.length - 1].text += ch; 218 | count += ch.length; 219 | } 220 | } 221 | return tokens; 222 | } 223 | 224 | /** 225 | * Generate tokens for the given vocalbuary. 226 | * @param text text to be tokenized. 227 | */ 228 | tokenize(text: string): number[] { 229 | // Source: 230 | // https://github.com/google-research/bert/blob/88a817c37f788702a363ff935fd173b6dc6ac0d6/tokenization.py#L311 231 | 232 | let outputTokens: number[] = []; 233 | 234 | const words = this.processInput(text); 235 | words.forEach(word => { 236 | if (word.text !== CLS_TOKEN && word.text !== SEP_TOKEN) { 237 | word.text = `${SEPERATOR}${word.text.normalize(NFKC_TOKEN)}`; 238 | } 239 | }); 240 | 241 | for (let i = 0; i < words.length; i++) { 242 | const chars = []; 243 | for (const symbol of words[i].text) { 244 | chars.push(symbol); 245 | } 246 | 247 | let isUnknown = false; 248 | let start = 0; 249 | const subTokens: number[] = []; 250 | 251 | const charsLength = chars.length; 252 | 253 | while (start < charsLength) { 254 | let end = charsLength; 255 | let currIndex; 256 | 257 | while (start < end) { 258 | const substr = chars.slice(start, end).join(''); 259 | 260 | const match = this.trie.find(substr); 261 | if (match != null && match.end != null) { 262 | currIndex = match.getWord()[2]; 263 | break; 264 | } 265 | 266 | end = end - 1; 267 | } 268 | 269 | if (currIndex == null) { 270 | isUnknown = true; 271 | break; 272 | } 273 | 274 | subTokens.push(currIndex); 275 | start = end; 276 | } 277 | 278 | if (isUnknown) { 279 | outputTokens.push(UNK_INDEX); 280 | } else { 281 | outputTokens = outputTokens.concat(subTokens); 282 | } 283 | } 284 | 285 | return outputTokens; 286 | } 287 | } 288 | 289 | export async function loadTokenizer(): Promise { 290 | const tokenizer = new BertTokenizer(); 291 | await tokenizer.load(); 292 | return tokenizer; 293 | } 294 | -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import './index.css'; 4 | import TextInputArea from './App'; 5 | 6 | ReactDOM.render( 7 | 8 | 9 | , 10 | document.getElementById('root') 11 | ); -------------------------------------------------------------------------------- /src/inference.js: -------------------------------------------------------------------------------- 1 | /** */ 2 | /*global BigInt */ 3 | /*global BigInt64Array */ 4 | 5 | import { loadTokenizer } from './bert_tokenizer.ts'; 6 | import * as wasmFeatureDetect from 'wasm-feature-detect'; 7 | 8 | //Setup onnxruntime 9 | const ort = require('onnxruntime-web'); 10 | 11 | //requires Cross-Origin-*-policy headers https://web.dev/coop-coep/ 12 | /** 13 | const simdResolver = wasmFeatureDetect.simd().then(simdSupported => { 14 | console.log("simd is supported? "+ simdSupported); 15 | if (simdSupported) { 16 | ort.env.wasm.numThreads = 3; 17 | ort.env.wasm.simd = true; 18 | } else { 19 | ort.env.wasm.numThreads = 1; 20 | ort.env.wasm.simd = false; 21 | } 22 | }); 23 | */ 24 | 25 | const options = { 26 | executionProviders: ['wasm'], 27 | graphOptimizationLevel: 'all' 28 | }; 29 | 30 | var downLoadingModel = true; 31 | const model = "./xtremedistill-go-emotion-int8.onnx"; 32 | 33 | const session = ort.InferenceSession.create(model, options); 34 | session.then(t => { 35 | downLoadingModel = false; 36 | //warmup the VM 37 | for(var i = 0; i < 10; i++) { 38 | console.log("Inference warmup " + i); 39 | lm_inference("this is a warmup inference"); 40 | } 41 | }); 42 | 43 | const tokenizer = loadTokenizer() 44 | 45 | const EMOJI_DEFAULT_DISPLAY = [ 46 | ["Emotion", "Score"], 47 | ['admiration 👏',0], 48 | ['amusement 😂', 0], 49 | ['neutral 😐',0], 50 | ['approval 👍',0], 51 | ['joy 😃',0], 52 | ['gratitude 🙏',0], 53 | ]; 54 | 55 | const EMOJIS = [ 56 | 'admiration 👏', 57 | 'amusement 😂', 58 | 'anger 😡', 59 | 'annoyance 😒', 60 | 'approval 👍', 61 | 'caring 🤗', 62 | 'confusion 😕', 63 | 'curiosity 🤔', 64 | 'desire 😍', 65 | 'disappointment 😞', 66 | 'disapproval 👎', 67 | 'disgust 🤮', 68 | 'embarrassment 😳', 69 | 'excitement 🤩', 70 | 'fear 😨', 71 | 'gratitude 🙏', 72 | 'grief 😢', 73 | 'joy 😃', 74 | 'love ❤️', 75 | 'nervousness 😬', 76 | 'optimism 🤞', 77 | 'pride 😌', 78 | 'realization 💡', 79 | 'relief😅', 80 | 'remorse 😞', 81 | 'sadness 😞', 82 | 'surprise 😲', 83 | 'neutral 😐' 84 | ]; 85 | 86 | function isDownloading() { 87 | return downLoadingModel; 88 | } 89 | 90 | function sortResult(a, b) { 91 | if (a[1] === b[1]) { 92 | return 0; 93 | } 94 | else { 95 | return (a[1] < b[1]) ? 1 : -1; 96 | } 97 | } 98 | 99 | function sigmoid(t) { 100 | return 1/(1+Math.pow(Math.E, -t)); 101 | } 102 | 103 | function create_model_input(encoded) { 104 | var input_ids = new Array(encoded.length+2); 105 | var attention_mask = new Array(encoded.length+2); 106 | var token_type_ids = new Array(encoded.length+2); 107 | input_ids[0] = BigInt(101); 108 | attention_mask[0] = BigInt(1); 109 | token_type_ids[0] = BigInt(0); 110 | var i = 0; 111 | for(; i < encoded.length; i++) { 112 | input_ids[i+1] = BigInt(encoded[i]); 113 | attention_mask[i+1] = BigInt(1); 114 | token_type_ids[i+1] = BigInt(0); 115 | } 116 | input_ids[i+1] = BigInt(102); 117 | attention_mask[i+1] = BigInt(1); 118 | token_type_ids[i+1] = BigInt(0); 119 | const sequence_length = input_ids.length; 120 | input_ids = new ort.Tensor('int64', BigInt64Array.from(input_ids), [1,sequence_length]); 121 | attention_mask = new ort.Tensor('int64', BigInt64Array.from(attention_mask), [1,sequence_length]); 122 | token_type_ids = new ort.Tensor('int64', BigInt64Array.from(token_type_ids), [1,sequence_length]); 123 | return { 124 | input_ids: input_ids, 125 | attention_mask: attention_mask, 126 | token_type_ids:token_type_ids 127 | } 128 | } 129 | 130 | async function lm_inference(text) { 131 | try { 132 | const encoded_ids = await tokenizer.then(t => { 133 | return t.tokenize(text); 134 | }); 135 | if(encoded_ids.length === 0) { 136 | return [0.0, EMOJI_DEFAULT_DISPLAY]; 137 | } 138 | const start = performance.now(); 139 | const model_input = create_model_input(encoded_ids); 140 | const output = await session.then(s => { return s.run(model_input,['output_0'])}); 141 | const duration = (performance.now() - start).toFixed(1); 142 | const probs = output['output_0'].data.map(sigmoid).map( t => Math.floor(t*100)); 143 | 144 | const result = []; 145 | for(var i = 0; i < EMOJIS.length;i++) { 146 | const t = [EMOJIS[i], probs[i]]; 147 | result[i] = t; 148 | } 149 | result.sort(sortResult); 150 | 151 | const result_list = []; 152 | result_list[0] = ["Emotion", "Score"]; 153 | for(i = 0; i < 6; i++) { 154 | result_list[i+1] = result[i]; 155 | } 156 | return [duration,result_list]; 157 | } catch (e) { 158 | return [0.0,EMOJI_DEFAULT_DISPLAY]; 159 | } 160 | } 161 | 162 | export let inference = lm_inference 163 | export let columnNames = EMOJI_DEFAULT_DISPLAY 164 | export let modelDownloadInProgress = isDownloading 165 | -------------------------------------------------------------------------------- /src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = onPerfEntry => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /src/setupProxy.js: -------------------------------------------------------------------------------- 1 | module.exports = function (app) { 2 | app.use(function (req, res, next) { 3 | res.setHeader("Cross-Origin-Opener-Policy", "same-origin"); 4 | res.setHeader("Cross-Origin-Embedder-Policy", "require-corp"); 5 | next(); 6 | }); 7 | }; 8 | --------------------------------------------------------------------------------