├── .gitignore ├── README.md ├── embeddings ├── clustering │ ├── index.html │ ├── p5.txt │ ├── sketch.js │ └── umap-js.js └── comparison-matrix │ ├── index.html │ └── sketch.js ├── llms ├── chat-conversation │ ├── index.html │ ├── sketch.js │ └── tf-helper.js ├── chat-prompt │ ├── index.html │ ├── sketch.js │ └── tf-helper.js └── text-completion │ ├── index.html │ ├── sketch.js │ └── tf-helper.js ├── p5-sketches ├── depth_estimation_image │ ├── dog_cat.jpg │ ├── index.html │ ├── sketch.js │ ├── style.css │ └── tf-helper.js ├── depth_estimation_video │ ├── index.html │ ├── sketch.js │ ├── style.css │ └── tf-helper.js ├── object_detection_image │ ├── dog_cat.jpg │ ├── index.html │ ├── sketch.js │ ├── style.css │ └── tf-helper.js ├── object_detection_video │ ├── index.html │ ├── sketch.js │ ├── style.css │ └── tf-helper.js ├── sentiment_analysis │ ├── index.html │ ├── sketch.js │ ├── style.css │ └── tf-helper.js └── text_generation │ ├── index.html │ ├── sketch.js │ ├── style.css │ └── tf-helper.js └── whisper-demo ├── index.html └── sketch.js /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | models/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Transformers.js Examples 2 | 3 | ## Overview 4 | 5 | This repository contains several demonstrations of running machine learning NLP models directly in the browser using transformers.js. 6 | 7 | ### 1. Embeddings 8 | 9 | Embeds sentences using `Xenova/all-MiniLM-L6-v2` and visualizes the results in two different contexts: 10 | 11 | - **Comparison Matrix**: Visualizes the cosine similarity between different sentences using p5.js to render a matrix. 12 | - **Clustering**: Plots sentence embeddings in a two-dimensional space with umap-js for dimensionality reduction. 13 | 14 | - Feature Extraction (embeddings): `Xenova/all-MiniLM-L6-v2` 15 | 16 | ### 2. Language Models 17 | 18 | A set of demos using language models to generate or complete text interactively in the browser. 19 | 20 | - **Chatbot Demo**: Implements a simple chatbot using `onnx-community/Qwen2.5-0.5B-Instruct`. 21 | - **Text Generation**: Generate text using various models including `onnx-community/Llama-3.2-1B-Instruct-q4f16` and `HuggingFaceTB/SmolLM-135M`. 22 | 23 | ### 3. Vision Models 24 | 25 | - **Depth Estimation**: Estimates depth from an image using `onnx-community/depth-anything-v2-small` and visualizes it using p5.js. 26 | - **Object Detection**: Detects objects in images and video in real-time using `Xenova/detr-resnet-50` with p5.js for visualization. 27 | 28 | ### 4. Whisper 29 | 30 | Implements a real-time audio transcription system using `Xenova/whisper-tiny.en`. 31 | 32 | ### Example Code 33 | 34 | ```javascript 35 | import { pipeline } from '@xenova/transformers'; 36 | 37 | // Allocate a pipeline for sentiment-analysis 38 | let pipe = await pipeline('sentiment-analysis'); 39 | 40 | let out = await pipe('I love transformers!'); 41 | // [{'label': 'POSITIVE', 'score': 0.999817686}] 42 | ``` 43 | 44 | ```javascript 45 | import { pipeline } from '@xenova/transformers'; 46 | 47 | // Allocate a pipeline for feature extraction 48 | let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2'); 49 | 50 | let out = await extractor('Your sentence here.'); 51 | // Example output: Array of feature embeddings 52 | ``` 53 | 54 | ## About Transformers.js 55 | 56 | Transformers.js allows for the direct execution of 🤗 transformer models in the browser with no server required. It is designed to be functionally equivalent to the 🤗 [Python transformers library](https://github.com/huggingface/transformers), supporting a range of tasks in NLP, computer vision, audio, and multimodal applications. 57 | 58 | - Run pretrained models for tasks such as text summarization, classification, and generation, image classification, and speech recognition. 59 | - Functionally similar to Hugging Face's Python library with the pipeline API. 60 | 61 | For further information and a comprehensive list of supported tasks and models, refer to the [Transformers.js Documentation](https://huggingface.co/docs/transformers.js/index). 62 | 63 | The [Transformers.js Model Hub](https://huggingface.co/models?library=transformers.js) lists models compatible with the library, searchable by tasks. 64 | 65 | ## p5.js Integration and Usage 66 | 67 | To integrate transformers.js with p5.js, instance mode can be used to enable compatibility with ES6 import statements required by transformers.js. However, dynamic import statements can also be used directly with p5.js as follows. 68 | 69 | ### Example of Loading Transformers.js 70 | 71 | ```javascript 72 | // Function to load transformers.js dynamically 73 | async function loadTransformers() { 74 | console.log('Loading transformers.js...'); 75 | const module = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers'); 76 | const { pipeline } = module; 77 | console.log('Transformers.js loaded successfully.'); 78 | return pipeline; 79 | } 80 | ``` 81 | 82 | ## What is UMAP? 83 | 84 | UMAP (Uniform Manifold Approximation and Projection) is a dimension reduction technique used to visualize high-dimensional data, like sentence embeddings from transformers.js. 85 | 86 | For [more about UMAP in JavaScript, here's the umap-js repo](https://github.com/PAIR-code/umap-js). This accompanying [Understanding UMAP](https://pair-code.github.io/understanding-umap/) article is also a fantastic read! 87 | -------------------------------------------------------------------------------- /embeddings/clustering/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Text Transformer Demo 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /embeddings/clustering/p5.txt: -------------------------------------------------------------------------------- 1 | 3D 2 | Color 3 | Constants 4 | DOM 5 | Data 6 | Environment 7 | Events 8 | Foundation 9 | IO 10 | Image 11 | Math 12 | Rendering 13 | Shape 14 | Structure 15 | Transform 16 | Typography 17 | Environment 18 | describe() 19 | describeElement() 20 | textOutput() 21 | gridOutput() 22 | print() 23 | frameCount 24 | deltaTime 25 | focused 26 | cursor() 27 | frameRate() 28 | getTargetFrameRate() 29 | noCursor() 30 | webglVersion 31 | displayWidth 32 | displayHeight 33 | windowWidth 34 | windowHeight 35 | windowResized() 36 | width 37 | height 38 | fullscreen() 39 | pixelDensity() 40 | displayDensity() 41 | getURL() 42 | getURLPath() 43 | getURLParams() 44 | Color 45 | Creating & Reading 46 | alpha() 47 | blue() 48 | brightness() 49 | color() 50 | green() 51 | hue() 52 | lerpColor() 53 | lightness() 54 | red() 55 | saturation() 56 | p5.Color 57 | Setting 58 | beginClip() 59 | endClip() 60 | clip() 61 | background() 62 | clear() 63 | colorMode() 64 | fill() 65 | noFill() 66 | noStroke() 67 | stroke() 68 | erase() 69 | noErase() 70 | Shape 71 | 2D Primitives 72 | arc() 73 | ellipse() 74 | circle() 75 | line() 76 | point() 77 | quad() 78 | rect() 79 | square() 80 | triangle() 81 | Attributes 82 | ellipseMode() 83 | noSmooth() 84 | rectMode() 85 | smooth() 86 | strokeCap() 87 | strokeJoin() 88 | strokeWeight() 89 | Curves 90 | bezier() 91 | bezierDetail() 92 | bezierPoint() 93 | bezierTangent() 94 | curve() 95 | curveDetail() 96 | curveTightness() 97 | curvePoint() 98 | curveTangent() 99 | Vertex 100 | beginContour() 101 | beginShape() 102 | bezierVertex() 103 | curveVertex() 104 | endContour() 105 | endShape() 106 | quadraticVertex() 107 | vertex() 108 | normal() 109 | 3D Primitives 110 | beginGeometry() 111 | endGeometry() 112 | buildGeometry() 113 | freeGeometry() 114 | plane() 115 | box() 116 | sphere() 117 | cylinder() 118 | cone() 119 | ellipsoid() 120 | torus() 121 | p5.Geometry 122 | 3D Models 123 | loadModel() 124 | model() 125 | vertexNormal() 126 | Constants 127 | HALF_PI 128 | PI 129 | QUARTER_PI 130 | TAU 131 | TWO_PI 132 | DEGREES 133 | RADIANS 134 | Structure 135 | preload() 136 | setup() 137 | draw() 138 | remove() 139 | disableFriendlyErrors 140 | noLoop() 141 | loop() 142 | isLooping() 143 | push() 144 | pop() 145 | redraw() 146 | p5() 147 | DOM 148 | p5.Element 149 | select() 150 | selectAll() 151 | removeElements() 152 | changed() 153 | input() 154 | createDiv() 155 | createP() 156 | createSpan() 157 | createImg() 158 | createA() 159 | createSlider() 160 | createButton() 161 | createCheckbox() 162 | createSelect() 163 | createRadio() 164 | createColorPicker() 165 | createInput() 166 | createFileInput() 167 | createVideo() 168 | createAudio() 169 | createCapture() 170 | createElement() 171 | p5.MediaElement 172 | p5.File 173 | Rendering 174 | p5.Graphics 175 | createCanvas() 176 | resizeCanvas() 177 | noCanvas() 178 | createGraphics() 179 | createFramebuffer() 180 | blendMode() 181 | drawingContext 182 | setAttributes() 183 | p5.Framebuffer 184 | Foundation 185 | let 186 | const 187 | === 188 | > 189 | >= 190 | < 191 | <= 192 | if-else 193 | function 194 | return 195 | boolean 196 | string 197 | number 198 | object 199 | class 200 | for 201 | while 202 | JSON 203 | console 204 | Transform 205 | applyMatrix() 206 | resetMatrix() 207 | rotate() 208 | rotateX() 209 | rotateY() 210 | rotateZ() 211 | scale() 212 | shearX() 213 | shearY() 214 | translate() 215 | Data 216 | LocalStorage 217 | storeItem() 218 | getItem() 219 | clearStorage() 220 | removeItem() 221 | Dictionary 222 | createStringDict() 223 | createNumberDict() 224 | p5.TypedDict 225 | p5.NumberDict 226 | Array Functions 227 | append() 228 | arrayCopy() 229 | concat() 230 | reverse() 231 | shorten() 232 | shuffle() 233 | sort() 234 | splice() 235 | subset() 236 | Conversion 237 | float() 238 | int() 239 | str() 240 | boolean() 241 | byte() 242 | char() 243 | unchar() 244 | hex() 245 | unhex() 246 | String Functions 247 | join() 248 | match() 249 | matchAll() 250 | nf() 251 | nfc() 252 | nfp() 253 | nfs() 254 | split() 255 | splitTokens() 256 | trim() 257 | Events 258 | Acceleration 259 | deviceOrientation 260 | accelerationX 261 | accelerationY 262 | accelerationZ 263 | pAccelerationX 264 | pAccelerationY 265 | pAccelerationZ 266 | rotationX 267 | rotationY 268 | rotationZ 269 | pRotationX 270 | pRotationY 271 | pRotationZ 272 | turnAxis 273 | setMoveThreshold() 274 | setShakeThreshold() 275 | deviceMoved() 276 | deviceTurned() 277 | deviceShaken() 278 | Keyboard 279 | keyIsPressed 280 | key 281 | keyCode 282 | keyPressed() 283 | keyReleased() 284 | keyTyped() 285 | keyIsDown() 286 | Mouse 287 | movedX 288 | movedY 289 | mouseX 290 | mouseY 291 | pmouseX 292 | pmouseY 293 | winMouseX 294 | winMouseY 295 | pwinMouseX 296 | pwinMouseY 297 | mouseButton 298 | mouseIsPressed 299 | mouseMoved() 300 | mouseDragged() 301 | mousePressed() 302 | mouseReleased() 303 | mouseClicked() 304 | doubleClicked() 305 | mouseWheel() 306 | requestPointerLock() 307 | exitPointerLock() 308 | Touch 309 | touches 310 | touchStarted() 311 | touchMoved() 312 | touchEnded() 313 | Image 314 | createImage() 315 | saveCanvas() 316 | saveFrames() 317 | p5.Image 318 | Loading & Displaying 319 | loadImage() 320 | saveGif() 321 | image() 322 | tint() 323 | noTint() 324 | imageMode() 325 | Pixels 326 | pixels 327 | blend() 328 | copy() 329 | filter() 330 | get() 331 | loadPixels() 332 | set() 333 | updatePixels() 334 | IO 335 | Input 336 | loadJSON() 337 | loadStrings() 338 | loadTable() 339 | loadXML() 340 | loadBytes() 341 | httpGet() 342 | httpPost() 343 | httpDo() 344 | p5.XML 345 | Output 346 | createWriter() 347 | p5.PrintWriter 348 | save() 349 | saveJSON() 350 | saveStrings() 351 | saveTable() 352 | Table 353 | p5.Table 354 | p5.TableRow 355 | Time & Date 356 | day() 357 | hour() 358 | minute() 359 | millis() 360 | month() 361 | second() 362 | year() 363 | Math 364 | Calculation 365 | abs() 366 | ceil() 367 | constrain() 368 | dist() 369 | exp() 370 | floor() 371 | lerp() 372 | log() 373 | mag() 374 | map() 375 | max() 376 | min() 377 | norm() 378 | pow() 379 | round() 380 | sq() 381 | sqrt() 382 | fract() 383 | Vector 384 | createVector() 385 | p5.Vector 386 | Noise 387 | noise() 388 | noiseDetail() 389 | noiseSeed() 390 | Random 391 | randomSeed() 392 | random() 393 | randomGaussian() 394 | Trigonometry 395 | acos() 396 | asin() 397 | atan() 398 | atan2() 399 | cos() 400 | sin() 401 | tan() 402 | degrees() 403 | radians() 404 | angleMode() 405 | Typography 406 | Attributes 407 | textAlign() 408 | textLeading() 409 | textSize() 410 | textStyle() 411 | textWidth() 412 | textAscent() 413 | textDescent() 414 | textWrap() 415 | Loading & Displaying 416 | loadFont() 417 | text() 418 | textFont() 419 | p5.Font 420 | 3D 421 | Interaction 422 | orbitControl() 423 | debugMode() 424 | noDebugMode() 425 | Lights 426 | ambientLight() 427 | specularColor() 428 | directionalLight() 429 | pointLight() 430 | imageLight() 431 | lights() 432 | lightFalloff() 433 | spotLight() 434 | noLights() 435 | Material 436 | loadShader() 437 | createShader() 438 | createFilterShader() 439 | shader() 440 | resetShader() 441 | texture() 442 | textureMode() 443 | textureWrap() 444 | normalMaterial() 445 | ambientMaterial() 446 | emissiveMaterial() 447 | specularMaterial() 448 | shininess() 449 | p5.Shader 450 | Camera 451 | camera() 452 | perspective() 453 | ortho() 454 | frustum() 455 | createCamera() 456 | p5.Camera 457 | setCamera() 458 | Credits 459 | Forum 460 | GitHub 461 | Twitter 462 | Instagram 463 | Discord 464 | Home 465 | Editor 466 | Download 467 | Donate 468 | Get Started 469 | Reference 470 | Libraries 471 | Learn 472 | Teach 473 | Examples 474 | Contribute 475 | Books 476 | Community 477 | Showcase -------------------------------------------------------------------------------- /embeddings/clustering/sketch.js: -------------------------------------------------------------------------------- 1 | // Programming A to Z 2 | // https://github.com/Programming-from-A-to-Z/A2Z-F24 3 | 4 | // Function to load transformers.js dynamically 5 | async function loadTransformers() { 6 | try { 7 | console.log('Loading transformers.js...'); 8 | const module = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers'); 9 | const { pipeline } = module; 10 | console.log('Transformers.js loaded successfully.'); 11 | return pipeline; 12 | } catch (error) { 13 | console.error('Failed to load transformers.js', error); 14 | } 15 | } 16 | 17 | // Function to load transformers.js dynamically 18 | async function loadTransformers() { 19 | console.log('Loading transformers.js...'); 20 | const module = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers'); 21 | const { pipeline } = module; 22 | console.log('Transformers.js loaded successfully.'); 23 | return pipeline; 24 | } 25 | 26 | // Dot class for representing and visualizing each data point 27 | class Dot { 28 | constructor(x, y, sentence, embedding) { 29 | this.x = x; 30 | this.y = y; 31 | this.sentence = sentence; 32 | this.embedding = embedding; 33 | this.r = 4; 34 | } 35 | 36 | // Display the dot on canvas 37 | show() { 38 | fill(175); 39 | stroke(255); 40 | circle(this.x, this.y, this.r * 2); 41 | } 42 | 43 | // Display the associated text of the dot 44 | showText() { 45 | fill(255); 46 | noStroke(); 47 | textSize(24); 48 | text(this.sentence, 10, height - 10); 49 | } 50 | 51 | // Check if a point (x, y) is over this dot 52 | over(x, y) { 53 | let d = dist(x, y, this.x, this.y); 54 | return d < this.r; 55 | } 56 | } 57 | 58 | let dots = []; 59 | let extractor; 60 | 61 | async function setup() { 62 | createCanvas(800, 800); 63 | console.log('Fetching sentences from p5.txt...'); 64 | let raw = await fetch('p5.txt'); 65 | let terms = await raw.text(); 66 | let sentences = terms.split(/\n+/); 67 | console.log('Sentences loaded:', sentences); 68 | 69 | let umap = new UMAP({ nNeighbors: 15, minDist: 0.1, nComponents: 2 }); 70 | let pipeline = await loadTransformers(); 71 | extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2'); 72 | console.log('Extracting embeddings for sentences one by one...'); 73 | 74 | let embeddings = []; 75 | for (let sentence of sentences) { 76 | let embeddingResult = await embedding(sentence); 77 | embeddings.push(embeddingResult); 78 | } 79 | 80 | let umapResults = umap.fit(embeddings); 81 | console.log('UMAP results:', umapResults); 82 | 83 | // Mapping UMAP results to pixel space for visualization 84 | let [maxW, minW, maxH, minH] = mapUMAPToPixelSpace(umapResults); 85 | 86 | // Creating Dot objects from UMAP results and sentences 87 | for (let i = 0; i < umapResults.length; i++) { 88 | let x = map(umapResults[i][0], minW, maxW, 10, width - 10); 89 | let y = map(umapResults[i][1], minH, maxH, 10, height - 10); 90 | let dot = new Dot(x, y, sentences[i], embeddings[i]); 91 | dots.push(dot); 92 | } 93 | console.log('Dots created:', dots); 94 | } 95 | 96 | function draw() { 97 | if (dots.length > 0) { 98 | background(0); 99 | for (let dot of dots) { 100 | dot.show(); 101 | } 102 | for (let dot of dots) { 103 | if (dot.over(mouseX, mouseY)) { 104 | dot.showText(); 105 | return; 106 | } 107 | } 108 | } 109 | } 110 | 111 | // Function to extract embedding from a sentence using transformers.js 112 | async function embedding(sentence) { 113 | console.log('Extracting embedding for sentence:', sentence); 114 | let output = await extractor(sentence, { pooling: 'mean', normalize: true }); 115 | return output.data; 116 | } 117 | 118 | // Function to map UMAP results to pixel space for visualization 119 | function mapUMAPToPixelSpace(umapResults) { 120 | // Initialize variables to track the maximum and minimum values in width and height 121 | let maxW = 0; 122 | let minW = Infinity; 123 | let maxH = 0; 124 | let minH = Infinity; 125 | 126 | // Iterate over each UMAP result to find the extreme values 127 | for (let i = 0; i < umapResults.length; i++) { 128 | // Update maxW and minW with the maximum and minimum x-coordinates 129 | maxW = Math.max(maxW, umapResults[i][0]); 130 | minW = Math.min(minW, umapResults[i][0]); 131 | 132 | // Update maxH and minH with the maximum and minimum y-coordinates 133 | maxH = Math.max(maxH, umapResults[i][1]); 134 | minH = Math.min(minH, umapResults[i][1]); 135 | } 136 | 137 | // Return the extreme values which define the bounding box for UMAP results 138 | return [maxW, minW, maxH, minH]; 139 | } 140 | -------------------------------------------------------------------------------- /embeddings/comparison-matrix/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Text Transformer Demo 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /embeddings/comparison-matrix/sketch.js: -------------------------------------------------------------------------------- 1 | // Programming A to Z 2 | // https://github.com/Programming-from-A-to-Z/A2Z-F24 3 | 4 | // Function to load transformers.js dynamically 5 | async function loadTransformers() { 6 | try { 7 | console.log('Loading transformers.js...'); 8 | const module = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers'); 9 | const { pipeline } = module; 10 | console.log('Transformers.js loaded successfully.'); 11 | return pipeline; 12 | } catch (error) { 13 | console.error('Failed to load transformers.js', error); 14 | } 15 | } 16 | 17 | // Function to get embeddings for a list of sentences using the transformers.js library 18 | async function getEmbeddings(sentences) { 19 | let pipeline = await loadTransformers(); 20 | const extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2'); 21 | 22 | // Store embeddings for each sentence 23 | let embeddings = []; 24 | for (let sentence of sentences) { 25 | let output = await extractor(sentence, { pooling: 'mean', normalize: true }); 26 | embeddings.push(output.data); 27 | } 28 | return embeddings; 29 | } 30 | 31 | // Store the similarity comparisons between sentences 32 | let comparison; 33 | 34 | // List of sentences to compare 35 | const sentences = [ 36 | 'What color is the sky?', 37 | 'What is an apple?', 38 | 'The sky is blue.', 39 | 'What does the fox say?', 40 | 'An apple is a fruit.', 41 | 'I have no idea.', 42 | ]; 43 | 44 | // p5.js sketch for visualizing the sentence similarities 45 | let cellSize; 46 | let cols = sentences.length; 47 | let rows = sentences.length; 48 | let whichsentences; 49 | 50 | async function setup() { 51 | createCanvas(400, 400); 52 | // Retrieve embeddings for the sentences 53 | const embeddings = await getEmbeddings(sentences); 54 | 55 | // Initialize and fill the comparison matrix with cosine similarity values 56 | comparison = []; 57 | for (let i = 0; i < sentences.length; i++) { 58 | comparison[i] = []; 59 | for (let j = 0; j < sentences.length; j++) { 60 | comparison[i][j] = cosineSimilarity(embeddings[i], embeddings[j]); 61 | } 62 | } 63 | console.log(comparison); 64 | cellSize = width / sentences.length; 65 | background(0); 66 | whichsentences = createP(''); 67 | } 68 | 69 | // Draw the similarity matrix and handle mouse interactions 70 | function draw() { 71 | background(0); 72 | if (comparison) { 73 | for (let i = 0; i < cols; i++) { 74 | for (let j = 0; j < rows; j++) { 75 | let colorValue = comparison[i][j] * 255; 76 | fill(colorValue); 77 | rect(j * cellSize, i * cellSize, cellSize, cellSize); 78 | } 79 | } 80 | 81 | // Display sentences and similarity score based on mouse position 82 | let row = Math.floor(mouseY / cellSize); 83 | let col = Math.floor(mouseX / cellSize); 84 | if (row >= 0 && row < rows && col >= 0 && col < cols) { 85 | whichsentences.html( 86 | `[${col}]: "${sentences[col]}"
[${row}]: "${sentences[row]}"
${comparison[col][row]}` 87 | ); 88 | } 89 | } 90 | } 91 | 92 | // Function to calculate dot product of two vectors 93 | function dotProduct(vecA, vecB) { 94 | return vecA.reduce((sum, val, i) => sum + val * vecB[i], 0); 95 | } 96 | 97 | // Function to calculate the magnitude of a vector 98 | function magnitude(vec) { 99 | return Math.sqrt(vec.reduce((sum, val) => sum + val * val, 0)); 100 | } 101 | 102 | // Function to calculate cosine similarity between two vectors 103 | function cosineSimilarity(vecA, vecB) { 104 | return dotProduct(vecA, vecB) / (magnitude(vecA) * magnitude(vecB)); 105 | } 106 | -------------------------------------------------------------------------------- /llms/chat-conversation/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /llms/chat-conversation/sketch.js: -------------------------------------------------------------------------------- 1 | // Programming A to Z, Fall 2024 2 | // https://github.com/Programming-from-A-to-Z/A2Z-F24 3 | 4 | let conversationHistory = []; 5 | let inputBox; 6 | let chatLog = ''; 7 | let chatP; 8 | let generator; 9 | 10 | async function setup() { 11 | noCanvas(); 12 | inputBox = createInput(); 13 | inputBox.size(300); 14 | let sendButton = createButton('Send'); 15 | sendButton.mousePressed(sendMessage); 16 | chatP = createP(); 17 | conversationHistory.push({ 18 | role: 'system', 19 | content: 'You are a helpful assistant.', 20 | // content: 21 | // 'You are a frog that only ever says ribbit. No matter what anyone else says you only say Ribbit.', 22 | }); 23 | 24 | // Load the Transformers.js model pipeline 25 | let pipeline = await loadTransformers(); 26 | generator = await pipeline('text-generation', 'onnx-community/Llama-3.2-1B-Instruct-q4f16', { 27 | dtype: 'q4f16', 28 | device: 'webgpu', 29 | progress_callback: (x) => { 30 | console.log(x); 31 | }, 32 | }); 33 | } 34 | 35 | async function sendMessage() { 36 | let userInput = inputBox.value(); 37 | conversationHistory.push({ role: 'user', content: userInput }); 38 | chatLog = `You: ${userInput}

` + chatLog; 39 | chatP.html(chatLog); 40 | 41 | if (generator) { 42 | try { 43 | // Generate a response based on the input prompt 44 | const output = await generator(conversationHistory, { max_new_tokens: 128 }); 45 | // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/at 46 | const reply = output[0].generated_text.at(-1).content; 47 | conversationHistory.push({ role: 'assistant', content: reply }); 48 | chatLog = `Chatbot: ${reply}

` + chatLog; 49 | chatP.html(chatLog); 50 | } catch (error) { 51 | console.error('Error communicating with Transformers.js:', error); 52 | chatLog += 'Error: Unable to communicate with the chatbot'; 53 | } 54 | } else { 55 | console.log('Model not loaded yet, try again in a minute.'); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /llms/chat-conversation/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /llms/chat-prompt/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /llms/chat-prompt/sketch.js: -------------------------------------------------------------------------------- 1 | // Programming A to Z 2 | // https://github.com/Programming-from-A-to-Z/A2Z-F24 3 | 4 | let generator; 5 | let inputText; 6 | let outputText; 7 | 8 | async function setup() { 9 | // Create a canvas and text input field 10 | createCanvas(400, 200); 11 | inputText = createInput('Type a prompt here...'); 12 | 13 | // Load the Transformers.js model pipeline with async/await 14 | let pipeline = await loadTransformers(); 15 | 16 | // Create a text generation pipeline with specific model and options 17 | generator = await pipeline('text-generation', 'onnx-community/Llama-3.2-1B-Instruct-q4f16', { 18 | dtype: 'q4f16', 19 | device: 'webgpu', 20 | progress_callback: (x) => { 21 | console.log(x); 22 | }, 23 | }); 24 | 25 | // Create a button after model is loaded 26 | let button = createButton('Generate Text'); 27 | button.mousePressed(generateText); 28 | } 29 | 30 | // Asynchronous function to generate text based on user input 31 | async function generateText() { 32 | // Ensure the model is loaded 33 | if (generator) { 34 | // Define the prompt structure for the text generation model 35 | const messages = [ 36 | { role: 'system', content: 'You are a helpful assistant.' }, 37 | { role: 'user', content: inputText.value() }, 38 | ]; 39 | 40 | // Generate a response based on the input prompt 41 | const output = await generator(messages, { max_new_tokens: 128 }); 42 | console.log(output); 43 | 44 | // Extract and display the generated text 45 | // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/at 46 | let outputText = output[0].generated_text.at(-1).content; 47 | background(240); 48 | text(outputText, 10, 10, width - 20, height - 20); 49 | } else { 50 | // Log a message if the model is not yet loaded 51 | console.log('Model not loaded yet, try again in a minute.'); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /llms/chat-prompt/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /llms/text-completion/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /llms/text-completion/sketch.js: -------------------------------------------------------------------------------- 1 | // Programming A to Z 2 | // https://github.com/Programming-from-A-to-Z/A2Z-F24 3 | 4 | let generator; 5 | let inputText; 6 | let outputText; 7 | 8 | async function setup() { 9 | // Create a canvas and text input field 10 | createCanvas(400, 200); 11 | inputText = createInput('Type a prompt here...'); 12 | 13 | // Load the Transformers.js model pipeline with async/await 14 | let pipeline = await loadTransformers(); 15 | 16 | // Try 17 | // https://huggingface.co/HuggingFaceTB/SmolLM-135M 18 | // https://huggingface.co/HuggingFaceTB/SmolLM-360M 19 | 20 | // Create a text generation pipeline with specific model and options 21 | generator = await pipeline('text-generation', 'HuggingFaceTB/SmolLM-135M', { 22 | dtype: 'q4', 23 | device: 'webgpu', 24 | progress_callback: (x) => { 25 | console.log(x); 26 | }, 27 | }); 28 | 29 | // Create a button after model is loaded 30 | let button = createButton('Generate Text'); 31 | button.mousePressed(generateText); 32 | } 33 | 34 | // Asynchronous function to generate text based on user input 35 | async function generateText() { 36 | // Ensure the model is loaded 37 | if (generator) { 38 | // Complete the user's text 39 | const output = await generator(inputText.value(), { max_new_tokens: 128 }); 40 | console.log(output); 41 | // Extract and display the generated text 42 | let outputText = output[0].generated_text; 43 | background(240); 44 | text(outputText, 10, 10, width - 20, height - 20); 45 | } else { 46 | // Log a message if the model is not yet loaded 47 | console.log('Model not loaded yet, try again in a minute.'); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /llms/text-completion/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_image/dog_cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Programming-from-A-to-Z/transformers-js-examples/0ca4f42e63427e70ccb0f90e09bb97af0778b6f5/p5-sketches/depth_estimation_image/dog_cat.jpg -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_image/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |
10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_image/sketch.js: -------------------------------------------------------------------------------- 1 | // Introduction to Machine Learning for the Arts 2 | // https://github.com/ml5js/Intro-ML-Arts-IMA-F24 3 | 4 | let depthEstimation; 5 | let img; 6 | let depthImg; 7 | 8 | function preload() { 9 | // Load an image of a dog and cat before the sketch starts 10 | img = loadImage('dog_cat.jpg'); 11 | } 12 | 13 | async function setup() { 14 | // Create a canvas for displaying the image and depth map 15 | createCanvas(640, 380); 16 | 17 | // Load the Transformers.js model pipeline with async/await 18 | let pipeline = await loadTransformers(); 19 | 20 | // Initialize the depth estimation model with specified options 21 | depthEstimation = await pipeline('depth-estimation', 'onnx-community/depth-anything-v2-small', { 22 | device: 'webgpu', 23 | }); 24 | 25 | // Create an image to hold the depth map, matching the original image size 26 | depthImg = createImage(img.width, img.height); 27 | 28 | // Estimate depth and display it on the canvas 29 | await estimateDepth(); 30 | image(depthImg, 0, 0); 31 | } 32 | 33 | // Asynchronous function to estimate depth in the image 34 | async function estimateDepth() { 35 | // Convert image to data URL and run depth estimation 36 | let results = await depthEstimation(img.canvas.toDataURL()); 37 | 38 | // Extract the depth map from the results 39 | let { depth } = results; 40 | 41 | // Load pixels of the depth image for manipulation 42 | depthImg.loadPixels(); 43 | 44 | // Loop through each row of the depth map 45 | for (let y = 0; y < depth.height; y++) { 46 | // Loop through each column of the depth map 47 | for (let x = 0; x < depth.width; x++) { 48 | // Calculate the 1D array index from 2D coordinates 49 | let index = x + y * depth.width; 50 | 51 | // Get the depth value for the current pixel 52 | let depthValue = depth.data[index]; 53 | 54 | // Calculate the corresponding pixel index in the depth image 55 | let pixelIndex = index * 4; 56 | 57 | // Set the RGB values to the depth value for a grayscale effect 58 | depthImg.pixels[pixelIndex] = depthValue; 59 | depthImg.pixels[pixelIndex + 1] = depthValue; 60 | depthImg.pixels[pixelIndex + 2] = depthValue; 61 | 62 | // Set the alpha value to fully opaque 63 | depthImg.pixels[pixelIndex + 3] = 255; 64 | } 65 | } 66 | 67 | // Update the pixels of the depth image 68 | depthImg.updatePixels(); 69 | } 70 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_image/style.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | canvas { 6 | display: block; 7 | } 8 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_image/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_video/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |
11 |
12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_video/sketch.js: -------------------------------------------------------------------------------- 1 | // Introduction to Machine Learning for the Arts 2 | // https://github.com/ml5js/Intro-ML-Arts-IMA-F24 3 | 4 | let video; 5 | let depthResult; 6 | let depthEstimation; 7 | let results; 8 | 9 | async function setup() { 10 | // Create canvas and set up video capture with constraints 11 | createCanvas(640, 480); 12 | video = createCapture(); 13 | video.size(320, 240); 14 | 15 | // Load the Transformers.js model pipeline with async/await 16 | let pipeline = await loadTransformers(); 17 | 18 | // Initialize the depth estimation model 19 | depthEstimation = await pipeline( 20 | "depth-estimation", 21 | "onnx-community/depth-anything-v2-small", 22 | { device: "webgpu" } 23 | ); 24 | 25 | // Start processing the video for depth estimation 26 | processVideo(); 27 | } 28 | 29 | function draw() { 30 | // Draw the video on the canvas 31 | image(video, 0, 0); 32 | 33 | // If depth results are available, visualize them using pixel manipulation 34 | if (results) { 35 | const { depth } = results; 36 | 37 | // Create an image to store the depth visualization 38 | let depthImg = createImage(depth.width, depth.height); 39 | 40 | // Load pixels of the depth image for manipulation 41 | depthImg.loadPixels(); 42 | 43 | // Loop through each row of the depth map 44 | for (let y = 0; y < depth.height; y++) { 45 | // Loop through each column of the depth map 46 | for (let x = 0; x < depth.width; x++) { 47 | // Calculate the 1D array index from 2D coordinates 48 | let index = x + y * depth.width; 49 | 50 | // Get the depth value for the current pixel 51 | let depthValue = depth.data[index]; 52 | 53 | // Calculate the corresponding pixel index in the depth image 54 | let pixelIndex = index * 4; 55 | 56 | // Set the RGB values to the depth value for a grayscale effect 57 | depthImg.pixels[pixelIndex] = depthValue; 58 | depthImg.pixels[pixelIndex + 1] = depthValue; 59 | depthImg.pixels[pixelIndex + 2] = depthValue; 60 | 61 | // Set the alpha value to fully opaque 62 | depthImg.pixels[pixelIndex + 3] = 255; 63 | } 64 | } 65 | 66 | // Update the pixels of the depth image 67 | depthImg.updatePixels(); 68 | 69 | // Draw the depth image on the canvas 70 | image(depthImg, 0, 0, width, height); 71 | } 72 | } 73 | 74 | // Asynchronous function to continuously process video frames 75 | async function processVideo() { 76 | // Convert video frame to data URL and run depth estimation 77 | results = await depthEstimation(video.canvas.toDataURL()); 78 | 79 | // Recursively call processVideo() to keep processing frames 80 | processVideo(); 81 | } 82 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_video/style.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | canvas { 6 | display: block; 7 | } 8 | -------------------------------------------------------------------------------- /p5-sketches/depth_estimation_video/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_image/dog_cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Programming-from-A-to-Z/transformers-js-examples/0ca4f42e63427e70ccb0f90e09bb97af0778b6f5/p5-sketches/object_detection_image/dog_cat.jpg -------------------------------------------------------------------------------- /p5-sketches/object_detection_image/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |
11 |
12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_image/sketch.js: -------------------------------------------------------------------------------- 1 | // Introduction to Machine Learning for the Arts 2 | // https://github.com/ml5js/Intro-ML-Arts-IMA-F24 3 | 4 | let img; 5 | let objectDetection; 6 | 7 | async function preload() { 8 | // Load the image of a dog and cat before the sketch starts 9 | img = loadImage('dog_cat.jpg'); 10 | } 11 | 12 | async function setup() { 13 | // Create a canvas and display the loaded image 14 | createCanvas(640, 480); 15 | image(img, 0, 0); 16 | 17 | // Load the Transformers.js model pipeline with async/await 18 | let pipeline = await loadTransformers(); 19 | 20 | objectDetection = await pipeline('object-detection', 'Xenova/detr-resnet-50', { 21 | device: 'webgpu', 22 | }); 23 | 24 | // Start object detection 25 | detectObjects(); 26 | } 27 | 28 | // Asynchronous function to detect objects in the image 29 | async function detectObjects() { 30 | // Convert image to data URL and run detection 31 | const results = await objectDetection(img.canvas.toDataURL()); 32 | 33 | // Log results to the console for inspection 34 | console.log(results); 35 | 36 | // Loop through detected objects and display them 37 | for (const result of results) { 38 | // Extract label, bounding box, and score using destructuring 39 | const { label, box, score } = result; 40 | const { xmin, ymin, xmax, ymax } = box; 41 | 42 | // Draw bounding box around detected object 43 | stroke(255, 0, 255); 44 | fill(255, 0, 255, 50); 45 | rectMode(CORNERS); 46 | rect(xmin, ymin, xmax, ymax); 47 | 48 | // Display label and formatted score above the bounding box 49 | noStroke(); 50 | fill(255); 51 | textSize(12); 52 | text(`${label} (${nf(score, 1, 4)})`, xmin, ymin - 5); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_image/style.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | canvas { 6 | display: block; 7 | } 8 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_image/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_video/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |
11 |
12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_video/sketch.js: -------------------------------------------------------------------------------- 1 | // Introduction to Machine Learning for the Arts 2 | // https://github.com/ml5js/Intro-ML-Arts-IMA-F24 3 | 4 | // Asynchronous video capture and object detection setup 5 | let video; 6 | let objectDetector; 7 | let results; 8 | 9 | async function setup() { 10 | // Create canvas and initialize video capture 11 | createCanvas(640, 480); 12 | video = createCapture(VIDEO); 13 | video.size(640, 480); 14 | video.hide(); 15 | 16 | // Load the Transformers.js model pipeline with async/await 17 | let pipeline = await loadTransformers(); 18 | objectDetector = await pipeline("object-detection", "Xenova/detr-resnet-50", { 19 | device: "webgpu", 20 | }); 21 | 22 | // Start object detection loop 23 | detectObjects(); 24 | 25 | // Confirm model loading 26 | console.log("model loaded"); 27 | } 28 | 29 | // Asynchronous function to detect objects in video frames 30 | async function detectObjects() { 31 | // Convert video frame to data URL and run detection 32 | results = await objectDetector(video.canvas.toDataURL()); 33 | 34 | // Continue detection loop 35 | detectObjects(); 36 | } 37 | 38 | function draw() { 39 | // Display the video frame on canvas 40 | image(video, 0, 0); 41 | 42 | // If results exist, display detected objects 43 | if (results) { 44 | for (let i = 0; i < results.length; i++) { 45 | // Extract label, score, and bounding box using destructuring 46 | let { label, score, box } = results[i]; 47 | let { xmin, ymin, xmax, ymax } = box; 48 | 49 | // Display label and score above bounding box 50 | fill(255); 51 | noStroke(); 52 | textSize(16); 53 | text(label, xmin, ymin - 16); 54 | text(score, xmin, ymin); 55 | 56 | // Draw bounding box around detected object 57 | noFill(); 58 | stroke(0); 59 | strokeWeight(2); 60 | rect(xmin, ymin, xmax - xmin, ymax - ymin); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_video/style.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | canvas { 6 | display: block; 7 | } 8 | -------------------------------------------------------------------------------- /p5-sketches/object_detection_video/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /p5-sketches/sentiment_analysis/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |
11 |
12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /p5-sketches/sentiment_analysis/sketch.js: -------------------------------------------------------------------------------- 1 | // Introduction to Machine Learning for the Arts 2 | // https://github.com/ml5js/Intro-ML-Arts-IMA-F24 3 | 4 | let sentimentModel; 5 | let userInput; 6 | 7 | async function setup() { 8 | // Create a canvas and a text input field 9 | createCanvas(200, 200); 10 | userInput = createElement('textArea', 'Write your text here.'); 11 | 12 | // Load the Transformers.js model pipeline with async/await 13 | let pipeline = await loadTransformers(); 14 | 15 | // Initialize the sentiment analysis model 16 | sentimentModel = await pipeline('sentiment-analysis'); 17 | 18 | // Create a button to trigger sentiment analysis (after model is loaded) 19 | let button = createButton('analyze'); 20 | button.mousePressed(analyzeText); 21 | } 22 | 23 | // Asynchronous function to analyze user input text 24 | async function analyzeText() { 25 | // Analyze the sentiment of the input text 26 | let results = await sentimentModel(userInput.value()); 27 | console.log(results); 28 | 29 | // Extract label and score using destructuring 30 | let { label, score } = results[0]; 31 | 32 | // Set background color based on sentiment label and confidence score 33 | if (label == 'POSITIVE') { 34 | background(0, 255 * score, 0); 35 | } else { 36 | background(255 * score, 0, 0); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /p5-sketches/sentiment_analysis/style.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | canvas { 6 | display: block; 7 | } 8 | -------------------------------------------------------------------------------- /p5-sketches/sentiment_analysis/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /p5-sketches/text_generation/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |
11 |
12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /p5-sketches/text_generation/sketch.js: -------------------------------------------------------------------------------- 1 | // Introduction to Machine Learning for the Arts 2 | // https://github.com/ml5js/Intro-ML-Arts-IMA-F24 3 | 4 | let generator; 5 | let inputText; 6 | let outputText; 7 | 8 | async function setup() { 9 | // Create a canvas and text input field 10 | createCanvas(400, 200); 11 | inputText = createInput('Type a prompt here...'); 12 | 13 | // Load the Transformers.js model pipeline with async/await 14 | let pipeline = await loadTransformers(); 15 | 16 | // Create a text generation pipeline with specific model and options 17 | generator = await pipeline('text-generation', 'onnx-community/Qwen2.5-0.5B-Instruct', { 18 | dtype: 'q4', 19 | device: 'webgpu', 20 | }); 21 | 22 | // Create a button after model is loaded 23 | let button = createButton('Generate Text'); 24 | button.mousePressed(generateText); 25 | } 26 | 27 | // Asynchronous function to generate text based on user input 28 | async function generateText() { 29 | // Ensure the model is loaded 30 | if (generator) { 31 | // Define the prompt structure for the text generation model 32 | const messages = [ 33 | { role: 'system', content: 'You are a helpful assistant.' }, 34 | { role: 'user', content: inputText.value() }, 35 | ]; 36 | 37 | // Generate a response based on the input prompt 38 | const output = await generator(messages, { max_new_tokens: 128 }); 39 | 40 | // Extract and display the generated text 41 | let outputText = output[0].generated_text.at(-1).content; 42 | background(240); 43 | text(outputText, 10, 10, width - 20, height - 20); 44 | } else { 45 | // Log a message if the model is not yet loaded 46 | console.log('Model not loaded yet, try again in a minute.'); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /p5-sketches/text_generation/style.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | canvas { 6 | display: block; 7 | } 8 | -------------------------------------------------------------------------------- /p5-sketches/text_generation/tf-helper.js: -------------------------------------------------------------------------------- 1 | async function loadTransformers() { 2 | try { 3 | const module = await import( 4 | "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0" 5 | ); 6 | const { pipeline } = module; 7 | return pipeline; 8 | } catch (error) { 9 | console.error("Failed to load transformers.js", error); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /whisper-demo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Whisper Audio Transcription 7 | 15 | 16 | 17 |

Whisper Transcription

18 | 19 |
transcript
20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /whisper-demo/sketch.js: -------------------------------------------------------------------------------- 1 | // Programming A to Z 2 | // https://github.com/Programming-from-A-to-Z/A2Z-F24 3 | 4 | // Import the Transformers.js pipeline for speech recognition 5 | import { pipeline } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2'; 6 | 7 | let mediaRecorder; 8 | let audioChunks = []; 9 | let isRecording = false; 10 | let transcriptDiv = document.getElementById('transcript'); 11 | let recordButton = document.getElementById('recordButton'); 12 | 13 | // Function to downsample audio buffer to 16000Hz for Whisper model 14 | function downsampleAudioBuffer(buffer, targetSampleRate) { 15 | const sampleRate = buffer.sampleRate; 16 | 17 | // If sample rate matches target, return original buffer 18 | if (sampleRate === targetSampleRate) { 19 | return buffer; 20 | } 21 | 22 | // Calculate downsample ratio and new buffer length 23 | const ratio = sampleRate / targetSampleRate; 24 | const newLength = Math.round(buffer.length / ratio); 25 | const newBuffer = new Float32Array(newLength); 26 | 27 | // Populate new buffer with downsampled audio data 28 | for (let i = 0; i < newLength; i++) { 29 | newBuffer[i] = buffer.getChannelData(0)[Math.round(i * ratio)]; 30 | } 31 | return newBuffer; 32 | } 33 | 34 | // Asynchronous function to transcribe audio using Whisper 35 | async function transcribeAudio(blob) { 36 | // Show transcribing status 37 | transcriptDiv.textContent = 'transcribing...'; 38 | 39 | // Load Whisper model from Transformers.js 40 | const model = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en'); 41 | 42 | // Convert Blob to ArrayBuffer for audio decoding 43 | const arrayBuffer = await blob.arrayBuffer(); 44 | 45 | // Decode ArrayBuffer to audio data using Web Audio API 46 | const audioContext = new (window.AudioContext || window.webkitAudioContext)(); 47 | const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); 48 | 49 | // Downsample the audio to 16000Hz as required by the model 50 | const downsampledAudio = downsampleAudioBuffer(audioBuffer, 16000); 51 | 52 | // Perform transcription with Whisper model 53 | const result = await model(downsampledAudio); 54 | 55 | // Display transcription result in the DOM 56 | transcriptDiv.textContent = result.text; 57 | } 58 | 59 | // Function to start audio recording 60 | function startRecording() { 61 | // Request microphone access from the user 62 | navigator.mediaDevices 63 | .getUserMedia({ audio: true }) 64 | .then((stream) => { 65 | // Initialize MediaRecorder with the audio stream 66 | mediaRecorder = new MediaRecorder(stream); 67 | mediaRecorder.start(); 68 | isRecording = true; 69 | recordButton.textContent = 'stop recording'; 70 | 71 | // Collect audio data while recording 72 | mediaRecorder.ondataavailable = (event) => { 73 | audioChunks.push(event.data); 74 | }; 75 | 76 | // Stop recording and start transcription 77 | mediaRecorder.onstop = () => { 78 | const audioBlob = new Blob(audioChunks, { type: 'audio/wav' }); 79 | transcribeAudio(audioBlob); 80 | audioChunks = []; 81 | }; 82 | }) 83 | .catch((error) => { 84 | // Handle errors when accessing the microphone 85 | console.error('Error accessing microphone: ', error); 86 | }); 87 | } 88 | 89 | // Function to stop audio recording 90 | function stopRecording() { 91 | mediaRecorder.stop(); 92 | isRecording = false; 93 | recordButton.textContent = 'start recording'; 94 | } 95 | 96 | // Event listener to toggle recording state 97 | recordButton.addEventListener('click', () => { 98 | if (isRecording) { 99 | stopRecording(); 100 | } else { 101 | startRecording(); 102 | } 103 | }); 104 | --------------------------------------------------------------------------------