├── .gitignore ├── LICENSE ├── README.md ├── assets └── react_sam_demo.gif ├── dist ├── SegmentAnything.d.ts ├── SegmentAnything.d.ts.map ├── SegmentAnything.js ├── SegmentAnything.js.map ├── Tool.d.ts ├── Tool.d.ts.map ├── Tool.js ├── Tool.js.map ├── helpers │ ├── Interfaces.d.ts │ ├── Interfaces.d.ts.map │ ├── Interfaces.js │ ├── Interfaces.js.map │ ├── maskUtils.d.ts │ ├── maskUtils.d.ts.map │ ├── maskUtils.js │ ├── maskUtils.js.map │ ├── onnxModelAPI.d.ts │ ├── onnxModelAPI.d.ts.map │ ├── onnxModelAPI.js │ ├── onnxModelAPI.js.map │ ├── scaleHelper.d.ts │ ├── scaleHelper.d.ts.map │ ├── scaleHelper.js │ └── scaleHelper.js.map ├── index.d.ts ├── index.d.ts.map ├── index.es.js ├── index.es.js.map ├── index.js ├── index.js.map ├── index.min.js ├── index.min.js.map └── types.d.ts ├── example ├── README.md ├── package-lock.json ├── package.json ├── public │ ├── favicon.ico │ ├── groceries.jpg │ ├── groceries_embedding.npy │ ├── index.html │ ├── logo192.png │ ├── manifest.json │ ├── sam_onnx_quantized_example.onnx │ └── static │ │ └── js │ │ ├── ort-wasm-simd-threaded.wasm │ │ ├── ort-wasm-simd.wasm │ │ ├── ort-wasm-threaded.wasm │ │ └── ort-wasm.wasm ├── src │ ├── DemoApp.tsx │ ├── helpers │ │ ├── Interfaces.tsx │ │ ├── maskUtils.tsx │ │ ├── onnxModelAPI.tsx │ │ └── scaleHelper.tsx │ └── index.tsx └── tsconfig.json ├── package-lock.json ├── package.json ├── rollup.config.js ├── src ├── Tool.tsx ├── helpers │ ├── Interfaces.tsx │ ├── maskUtils.tsx │ ├── onnxModelAPI.tsx │ └── scaleHelper.tsx └── index.tsx ├── tsconfig.json └── types ├── SegmentAnything.d.ts ├── Tool.d.ts ├── helpers ├── Interfaces.d.ts ├── maskUtils.d.ts ├── onnxModelAPI.d.ts └── scaleHelper.d.ts └── index.d.ts /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_STORE 2 | node_modules 3 | scripts/flow/*/.flowconfig 4 | .flowconfig 5 | *~ 6 | *.pyc 7 | .grunt 8 | _SpecRunner.html 9 | __benchmarks__ 10 | build/ 11 | remote-repo/ 12 | coverage/ 13 | .module-cache 14 | fixtures/dom/public/react-dom.js 15 | fixtures/dom/public/react.js 16 | test/the-files-to-test.generated.js 17 | *.log* 18 | chrome-user-data 19 | *.sublime-project 20 | *.sublime-workspace 21 | .idea 22 | *.iml 23 | .vscode 24 | *.swp 25 | *.swo 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Michael Murray 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # React Segment Anything 2 | 3 | React component for interfacing with Meta's Segment Anything Model (SAM) 4 | 5 | [![license](https://img.shields.io/badge/license-MIT-blue.svg)](hhttps://github.com/mmurray/react-segment-anything/blob/HEAD/LICENSE) 6 | [![npm latest package](https://img.shields.io/npm/v/react-segment-anything/latest.svg)](https://www.npmjs.com/package/react-segment-anything) 7 | 8 | ![Demonstration of react-segment-anything component](assets/react_sam_demo.gif?raw=true) 9 | 10 | 11 | ## Getting Started 12 | 13 | ### Installation 14 | 15 | 1. Ensure that you have React 17 or later installed (MUI V5 requires React 17 or 18) 16 | 17 | 2. Install Peer Dependencies (Material UI V5) 18 | 19 | ```bash 20 | npm install @mui/material @mui/icons-material 21 | ``` 22 | 23 | 3. Install react-segment-anything 24 | 25 | ```bash 26 | npm install react-segment-anything 27 | ``` 28 | 29 | ### Usage 30 | 31 | > See usage example [here](./example) 32 | 33 | ```jsx 34 | import React, { useState, useEffect } from 'react'; 35 | import { Tensor } from "onnxruntime-web"; 36 | 37 | /* @ts-ignore */ 38 | import npyjs from "npyjs"; 39 | 40 | import Container from '@mui/material/Container'; 41 | import { SegmentAnything } from 'react-segment-anything'; 42 | 43 | const ort = require("onnxruntime-web"); 44 | 45 | const IMAGE_EMBEDDING = "/groceries_embedding.npy"; 46 | const IMAGE_PATH = "/groceries.jpg"; 47 | const MODEL_URL = "/sam_onnx_quantized_example.onnx"; 48 | 49 | const DemoApp = () => { 50 | 51 | const [image, setImage] = useState(undefined); 52 | const loadImage = async (imageFile: string) => { 53 | const img = new Image(); 54 | img.src = imageFile; 55 | img.onload = () => setImage(img); 56 | }; 57 | 58 | const [tensor, setTensor] = useState(null); 59 | const loadNpyTensor = async (tensorFile: string, dType: string) => { 60 | let npLoader = new npyjs(); 61 | const npArray = await npLoader.load(tensorFile); 62 | const tensor = new ort.Tensor(dType, npArray.data, npArray.shape); 63 | return tensor; 64 | }; 65 | 66 | useEffect(() => { 67 | Promise.resolve(loadNpyTensor(IMAGE_EMBEDDING, "float32")).then( 68 | (embedding) => setTensor(embedding) 69 | ); 70 | Promise.resolve(loadImage(IMAGE_PATH)); 71 | }, []); 72 | 73 | if (!image || !tensor) return
Loading...
; 74 | 75 | return ( 76 | 77 | {}} 79 | image={image} 80 | embedding={tensor} 81 | modelUrl=MODEL_URL 82 | /> 83 | 84 | ); 85 | } 86 | 87 | ``` 88 | -------------------------------------------------------------------------------- /assets/react_sam_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/assets/react_sam_demo.gif -------------------------------------------------------------------------------- /dist/SegmentAnything.d.ts: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { SegmentAnythingProps } from './helpers/Interfaces'; 3 | declare const SegmentAnything: ({ image, embedding, modelUrl, handleMaskSaved }: SegmentAnythingProps) => React.JSX.Element; 4 | export default SegmentAnything; 5 | //# sourceMappingURL=SegmentAnything.d.ts.map -------------------------------------------------------------------------------- /dist/SegmentAnything.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"SegmentAnything.d.ts","sourceRoot":"","sources":["../src/SegmentAnything.tsx"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;AAkBzB,OAAO,EAAE,oBAAoB,EAAgC,MAAM,sBAAsB,CAAC;AAQ1F,QAAA,MAAM,eAAe,oDAAmD,oBAAoB,sBA6I3F,CAAA;AAED,eAAe,eAAe,CAAC"} -------------------------------------------------------------------------------- /dist/SegmentAnything.js: -------------------------------------------------------------------------------- 1 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 2 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 3 | return new (P || (P = Promise))(function (resolve, reject) { 4 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 5 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 6 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 7 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 8 | }); 9 | }; 10 | var __generator = (this && this.__generator) || function (thisArg, body) { 11 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 12 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 13 | function verb(n) { return function (v) { return step([n, v]); }; } 14 | function step(op) { 15 | if (f) throw new TypeError("Generator is already executing."); 16 | while (g && (g = 0, op[0] && (_ = 0)), _) try { 17 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 18 | if (y = 0, t) op = [op[0] & 2, t.value]; 19 | switch (op[0]) { 20 | case 0: case 1: t = op; break; 21 | case 4: _.label++; return { value: op[1], done: false }; 22 | case 5: _.label++; y = op[1]; op = [0]; continue; 23 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 24 | default: 25 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 26 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 27 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 28 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 29 | if (t[2]) _.ops.pop(); 30 | _.trys.pop(); continue; 31 | } 32 | op = body.call(thisArg, _); 33 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 34 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 35 | } 36 | }; 37 | import React from 'react'; 38 | import { Button, Stack, Card, CardContent, Box, Typography, Grid } from "@mui/material"; 39 | import BoundingBoxIcon from '@mui/icons-material/HighlightAlt'; 40 | import PointIcon from '@mui/icons-material/AdsClick'; 41 | import CheckIcon from '@mui/icons-material/Check'; 42 | import { modelData } from './helpers/onnxModelAPI'; 43 | import { onnxMaskToImage } from "./helpers/maskUtils"; 44 | import { ToolMode } from './helpers/Interfaces'; 45 | import Tool from './Tool'; 46 | import { useState, useEffect } from 'react'; 47 | import { InferenceSession } from "onnxruntime-web"; 48 | /* @ts-ignore */ 49 | import * as _ from "underscore"; 50 | var SegmentAnything = function (_a) { 51 | var image = _a.image, embedding = _a.embedding, modelUrl = _a.modelUrl, handleMaskSaved = _a.handleMaskSaved; 52 | // ONNX model 53 | var _b = useState(null), model = _b[0], setModel = _b[1]; 54 | // Tool mode 55 | var _c = useState(ToolMode.Default), mode = _c[0], setMode = _c[1]; // Tool mode 56 | // Model parameters (clicks and bounding box) 57 | var _d = useState([]), clicks = _d[0], setClicks = _d[1]; 58 | var _e = useState(undefined), bbox = _e[0], setBbox = _e[1]; 59 | // Mask image 60 | var _f = useState(undefined), maskImage = _f[0], setMaskImage = _f[1]; 61 | // Handler for the clear all clicks button 62 | var clearAllClicks = function () { 63 | setClicks([]); 64 | setMode(ToolMode.Default); 65 | }; 66 | // Handler for the clear all bbox button 67 | var clearBBox = function () { 68 | setBbox(undefined); 69 | setMode(ToolMode.Default); 70 | }; 71 | // Input images to SAM must be resized so the longest side is 1024 72 | var LONG_SIDE_LENGTH = 1024; 73 | var w = image.naturalWidth; 74 | var h = image.naturalHeight; 75 | var samScale = LONG_SIDE_LENGTH / Math.max(h, w); 76 | var modelScale = { width: w, height: h, samScale: samScale }; 77 | // Initialize the ONNX model. load the image, and load the SAM 78 | // pre-computed image embedding 79 | useEffect(function () { 80 | // Initialize the ONNX model 81 | var initModel = function () { return __awaiter(void 0, void 0, void 0, function () { 82 | var model_1, e_1; 83 | return __generator(this, function (_a) { 84 | switch (_a.label) { 85 | case 0: 86 | _a.trys.push([0, 2, , 3]); 87 | return [4 /*yield*/, InferenceSession.create(modelUrl)]; 88 | case 1: 89 | model_1 = _a.sent(); 90 | setModel(model_1); 91 | return [3 /*break*/, 3]; 92 | case 2: 93 | e_1 = _a.sent(); 94 | console.log(e_1); 95 | return [3 /*break*/, 3]; 96 | case 3: return [2 /*return*/]; 97 | } 98 | }); 99 | }); }; 100 | initModel(); 101 | }, []); 102 | // Run the ONNX model and generate a mask image 103 | var runONNX = function () { return __awaiter(void 0, void 0, void 0, function () { 104 | var feeds, results, output, e_2; 105 | return __generator(this, function (_a) { 106 | switch (_a.label) { 107 | case 0: 108 | _a.trys.push([0, 2, , 3]); 109 | if (model === null || clicks === null) 110 | return [2 /*return*/]; 111 | feeds = modelData({ 112 | clicks: clicks, 113 | bbox: bbox, 114 | embedding: embedding, 115 | modelScale: modelScale, 116 | }); 117 | if (feeds === undefined) 118 | return [2 /*return*/]; 119 | return [4 /*yield*/, model.run(feeds)]; 120 | case 1: 121 | results = _a.sent(); 122 | output = results[model.outputNames[0]]; 123 | // The predicted mask returned from the ONNX model is an array which is 124 | // rendered as an HTML image using onnxMaskToImage() from maskUtils.tsx. 125 | setMaskImage(onnxMaskToImage(output.data, output.dims[2], output.dims[3])); 126 | return [3 /*break*/, 3]; 127 | case 2: 128 | e_2 = _a.sent(); 129 | console.log(e_2); 130 | return [3 /*break*/, 3]; 131 | case 3: return [2 /*return*/]; 132 | } 133 | }); 134 | }); }; 135 | var throttledRunONNX = _.throttle(runONNX, 15); 136 | // Run the ONNX model every time clicks or bbox have changed 137 | useEffect(function () { 138 | runONNX(); 139 | }, [clicks]); 140 | useEffect(function () { 141 | throttledRunONNX(); 142 | }, [bbox]); 143 | return (React.createElement("div", { className: "react-sam" }, 144 | React.createElement(Box, { sx: { flexGrow: 1 } }, 145 | React.createElement(Grid, { container: true, spacing: 2 }, 146 | React.createElement(Grid, { item: true, xs: 3 }, 147 | React.createElement(Stack, { spacing: 2, className: "react-sam-toolbar" }, 148 | React.createElement(Card, null, 149 | React.createElement(CardContent, null, 150 | React.createElement(Typography, { className: "react-sam-toolbar-item-title", sx: { mb: 2 } }, 151 | React.createElement(BoundingBoxIcon, null), 152 | " Bounding Box"), 153 | React.createElement(Stack, null, 154 | React.createElement(Typography, { sx: { mb: 1 } }, 155 | React.createElement(Button, { onClick: function () { return setMode(ToolMode.BoundingBox); }, variant: "contained" }, "Select")), 156 | React.createElement(Typography, { sx: { mb: 1 } }, 157 | React.createElement(Button, { onClick: clearBBox, variant: "contained", color: "error" }, "Clear All"))))), 158 | React.createElement(Card, null, 159 | React.createElement(CardContent, null, 160 | React.createElement(Typography, { className: "react-sam-toolbar-item-title", sx: { mb: 2 } }, 161 | React.createElement(PointIcon, null), 162 | " Points"), 163 | React.createElement(Stack, null, 164 | React.createElement(Typography, { sx: { mb: 1 } }, 165 | React.createElement(Button, { onClick: function () { return setMode(ToolMode.PositivePoints); }, variant: "contained" }, "Add Positive Points")), 166 | React.createElement(Typography, { sx: { mb: 1 } }, 167 | React.createElement(Button, { onClick: function () { return setMode(ToolMode.NegativePoints); }, variant: "contained" }, "Add Negative Points")), 168 | React.createElement(Typography, { sx: { mb: 1 } }, 169 | React.createElement(Button, { onClick: clearAllClicks, variant: "contained", color: "error" }, "Clear All"))))), 170 | React.createElement(Button, { disabled: !maskImage, onClick: function () { return handleMaskSaved(maskImage, image); }, sx: { mt: 2 }, className: 'react-sam-toolbar-save', startIcon: React.createElement(CheckIcon, null), size: "large", color: "success", variant: "contained" }, "Save Mask"))), 171 | React.createElement(Grid, { item: true, xs: 9 }, 172 | React.createElement(Tool, { image: image, maskImage: maskImage, mode: mode, clicks: clicks, bbox: bbox, handleClicksChange: function (newClicks) { return setClicks(newClicks); }, handleBoundingBoxChange: function (newBbox) { return setBbox(newBbox); } })))))); 173 | }; 174 | export default SegmentAnything; 175 | //# sourceMappingURL=SegmentAnything.js.map -------------------------------------------------------------------------------- /dist/SegmentAnything.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"SegmentAnything.js","sourceRoot":"","sources":["../src/SegmentAnything.tsx"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;AAEzB,OAAO,EACH,MAAM,EACN,KAAK,EACL,IAAI,EACJ,WAAW,EACX,GAAG,EACH,UAAU,EACV,IAAI,EACP,MAAM,eAAe,CAAC;AAEvB,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAC/D,OAAO,SAAS,MAAM,8BAA8B,CAAC;AACrD,OAAO,SAAS,MAAM,2BAA2B,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAwB,QAAQ,EAAsB,MAAM,sBAAsB,CAAC;AAC1F,OAAO,IAAI,MAAM,QAAQ,CAAC;AAE1B,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,gBAAgB;AAChB,OAAO,KAAK,CAAC,MAAM,YAAY,CAAC;AAEhC,IAAM,eAAe,GAAG,UAAC,EAAmE;QAAlE,KAAK,WAAA,EAAE,SAAS,eAAA,EAAE,QAAQ,cAAA,EAAE,eAAe,qBAAA;IACjE,aAAa;IACP,IAAA,KAAoB,QAAQ,CAA0B,IAAI,CAAC,EAA1D,KAAK,QAAA,EAAE,QAAQ,QAA2C,CAAC;IAElE,YAAY;IACN,IAAA,KAAkB,QAAQ,CAAW,QAAQ,CAAC,OAAO,CAAC,EAArD,IAAI,QAAA,EAAE,OAAO,QAAwC,CAAC,CAAC,YAAY;IAE1E,6CAA6C;IACvC,IAAA,KAAsB,QAAQ,CAAU,EAAE,CAAC,EAA1C,MAAM,QAAA,EAAE,SAAS,QAAyB,CAAC;IAC5C,IAAA,KAAkB,QAAQ,CAA0B,SAAS,CAAC,EAA7D,IAAI,QAAA,EAAE,OAAO,QAAgD,CAAC;IAErE,aAAa;IACP,IAAA,KAA4B,QAAQ,CAA+B,SAAS,CAAC,EAA5E,SAAS,QAAA,EAAE,YAAY,QAAqD,CAAC;IAEpF,0CAA0C;IAC1C,IAAM,cAAc,GAAG;QACnB,SAAS,CAAC,EAAE,CAAC,CAAC;QACd,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC,CAAA;IAED,wCAAwC;IACxC,IAAM,SAAS,GAAG;QACd,OAAO,CAAC,SAAS,CAAC,CAAC;QACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC,CAAA;IAED,kEAAkE;IAClE,IAAM,gBAAgB,GAAG,IAAI,CAAC;IAC9B,IAAI,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC;IAC3B,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC;IAC5B,IAAM,QAAQ,GAAG,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,IAAM,UAAU,GAAG,EAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAC,CAAA;IAE5D,8DAA8D;IAC9D,+BAA+B;IAC/B,SAAS,CAAC;QACN,4BAA4B;QAC5B,IAAM,SAAS,GAAG;;;;;;wBAEA,qBAAM,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAA;;wBAA/C,UAAQ,SAAuC;wBACrD,QAAQ,CAAC,OAAK,CAAC,CAAC;;;;wBAEhB,OAAO,CAAC,GAAG,CAAC,GAAC,CAAC,CAAC;;;;;aAElB,CAAC;QACF,SAAS,EAAE,CAAC;IAChB,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,+CAA+C;IAC/C,IAAM,OAAO,GAAG;;;;;;oBAER,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI;wBAAE,sBAAO;oBAIxC,KAAK,GAAG,SAAS,CAAC;wBACpB,MAAM,QAAA;wBACN,IAAI,MAAA;wBACJ,SAAS,WAAA;wBACT,UAAU,YAAA;qBACb,CAAC,CAAC;oBACH,IAAI,KAAK,KAAK,SAAS;wBAAE,sBAAO;oBAEhB,qBAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAA;;oBAAhC,OAAO,GAAG,SAAsB;oBAChC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7C,wEAAwE;oBACxE,wEAAwE;oBACxE,YAAY,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;;oBAE3E,OAAO,CAAC,GAAG,CAAC,GAAC,CAAC,CAAC;;;;;SAEtB,CAAC;IAEF,IAAM,gBAAgB,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEjD,4DAA4D;IAC5D,SAAS,CAAC;QACN,OAAO,EAAE,CAAC;IACd,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,SAAS,CAAC;QACN,gBAAgB,EAAE,CAAC;IACvB,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;IAEV,OAAO,CACH,6BAAK,SAAS,EAAC,WAAW;QACtB,oBAAC,GAAG,IAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE;YACpB,oBAAC,IAAI,IAAC,SAAS,QAAC,OAAO,EAAE,CAAC;gBACtB,oBAAC,IAAI,IAAC,IAAI,QAAC,EAAE,EAAE,CAAC;oBACZ,oBAAC,KAAK,IAAC,OAAO,EAAE,CAAC,EAAE,SAAS,EAAC,mBAAmB;wBAC5C,oBAAC,IAAI;4BACD,oBAAC,WAAW;gCACR,oBAAC,UAAU,IAAC,SAAS,EAAC,8BAA8B,EAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;oCAC9D,oBAAC,eAAe,OAAG;oDACV;gCACb,oBAAC,KAAK;oCACF,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAM,OAAA,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAA7B,CAA6B,EAAE,OAAO,EAAC,WAAW,aAAgB,CAChF;oCACb,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAC,WAAW,EAAC,KAAK,EAAC,OAAO,gBAAmB,CACvE,CACT,CACE,CACX;wBACP,oBAAC,IAAI;4BACD,oBAAC,WAAW;gCACR,oBAAC,UAAU,IAAC,SAAS,EAAC,8BAA8B,EAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;oCAC9D,oBAAC,SAAS,OAAG;8CACJ;gCACb,oBAAC,KAAK;oCACF,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAM,OAAA,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAhC,CAAgC,EAAE,OAAO,EAAC,WAAW,0BAA6B,CAChG;oCACb,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAM,OAAA,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAhC,CAAgC,EAAE,OAAO,EAAC,WAAW,0BAA6B,CAChG;oCACb,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAc,EAAE,OAAO,EAAC,WAAW,EAAC,KAAK,EAAC,OAAO,gBAAmB,CAC5E,CACT,CACE,CACX;wBACP,oBAAC,MAAM,IAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,cAAM,OAAA,eAAe,CAAC,SAAU,EAAE,KAAK,CAAC,EAAlC,CAAkC,EAAE,EAAE,EAAE,EAAC,EAAE,EAAE,CAAC,EAAC,EAAE,SAAS,EAAC,wBAAwB,EAAC,SAAS,EAAE,oBAAC,SAAS,OAAG,EAAE,IAAI,EAAC,OAAO,EAAC,KAAK,EAAC,SAAS,EAAC,OAAO,EAAC,WAAW,gBAAmB,CAC1N,CACL;gBACP,oBAAC,IAAI,IAAC,IAAI,QAAC,EAAE,EAAE,CAAC;oBACZ,oBAAC,IAAI,IACD,KAAK,EAAE,KAAK,EACZ,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,IAAI,EACV,kBAAkB,EAAE,UAAC,SAAkB,IAAK,OAAA,SAAS,CAAC,SAAS,CAAC,EAApB,CAAoB,EAChE,uBAAuB,EAAE,UAAC,OAAgC,IAAK,OAAA,OAAO,CAAC,OAAO,CAAC,EAAhB,CAAgB,GAC7E,CACH,CACJ,CACL,CACJ,CACT,CAAC;AACN,CAAC,CAAA;AAED,eAAe,eAAe,CAAC"} -------------------------------------------------------------------------------- /dist/Tool.d.ts: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { ToolProps } from "./helpers/Interfaces"; 3 | declare const Tool: ({ handleClicksChange, handleBoundingBoxChange, image, maskImage, bbox, clicks, mode }: ToolProps) => React.JSX.Element; 4 | export default Tool; 5 | //# sourceMappingURL=Tool.d.ts.map -------------------------------------------------------------------------------- /dist/Tool.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"Tool.d.ts","sourceRoot":"","sources":["../src/Tool.tsx"],"names":[],"mappings":"AAAA,OAAO,KAAsC,MAAM,OAAO,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAY,MAAM,sBAAsB,CAAC;AAM3D,QAAA,MAAM,IAAI,0FASP,SAAS,sBAmIX,CAAC;AAEF,eAAe,IAAI,CAAC"} -------------------------------------------------------------------------------- /dist/Tool.js: -------------------------------------------------------------------------------- 1 | var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { 2 | if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { 3 | if (ar || !(i in from)) { 4 | if (!ar) ar = Array.prototype.slice.call(from, 0, i); 5 | ar[i] = from[i]; 6 | } 7 | } 8 | return to.concat(ar || Array.prototype.slice.call(from)); 9 | }; 10 | import React, { useEffect, useState, useRef } from "react"; 11 | import { ToolMode } from "./helpers/Interfaces"; 12 | import { Stage, Layer, Circle, Rect, Image } from 'react-konva'; 13 | var Tool = function (_a) { 14 | var 15 | // handleMouseMove, 16 | handleClicksChange = _a.handleClicksChange, handleBoundingBoxChange = _a.handleBoundingBoxChange, image = _a.image, maskImage = _a.maskImage, bbox = _a.bbox, _b = _a.clicks, clicks = _b === void 0 ? [] : _b, _c = _a.mode, mode = _c === void 0 ? ToolMode.Default : _c; 17 | var containerRef = useRef(null); 18 | var _d = useState(null), bboxStart = _d[0], setBboxStart = _d[1]; 19 | var _e = useState(0), width = _e[0], setWidth = _e[1]; 20 | var _f = useState(0), height = _f[0], setHeight = _f[1]; 21 | var bodyEl = document.body; 22 | var fitToPage = function () { 23 | if (!image || !containerRef) 24 | return; 25 | // const imageAspectRatio = image.width / image.height; 26 | // const screenAspectRatio = window.innerWidth / window.innerHeight; 27 | var containerRatio = containerRef.current.offsetWidth / image.width; 28 | setWidth(containerRef.current.offsetWidth); 29 | setHeight(image.height * containerRatio); 30 | }; 31 | var resizeObserver = new ResizeObserver(function (entries) { 32 | for (var _i = 0, entries_1 = entries; _i < entries_1.length; _i++) { 33 | var entry = entries_1[_i]; 34 | if (entry.target === bodyEl) { 35 | fitToPage(); 36 | } 37 | } 38 | }); 39 | useEffect(function () { 40 | fitToPage(); 41 | resizeObserver.observe(bodyEl); 42 | return function () { 43 | resizeObserver.unobserve(bodyEl); 44 | }; 45 | }, [image]); 46 | var targetSelectEnabled = mode !== ToolMode.Default; 47 | var onMouseDown = function (e) { 48 | if (mode === ToolMode.Default) 49 | return; 50 | if (!containerRef) 51 | return; 52 | var pos = e.target.getStage().getPointerPosition(); 53 | var containerRatio = containerRef.current.offsetWidth / image.width; 54 | var x = pos.x / containerRatio; 55 | var y = pos.y / containerRatio; 56 | if (mode == ToolMode.PositivePoints) { 57 | if (!handleClicksChange) 58 | return; 59 | handleClicksChange(__spreadArray(__spreadArray([], clicks, true), [{ x: x, y: y, pointType: 1 }], false)); 60 | } 61 | if (mode == ToolMode.NegativePoints) { 62 | if (!handleClicksChange) 63 | return; 64 | handleClicksChange(__spreadArray(__spreadArray([], clicks, true), [{ x: x, y: y, pointType: 0 }], false)); 65 | } 66 | if (mode == ToolMode.BoundingBox) { 67 | setBboxStart({ x: x, y: y }); 68 | } 69 | }; 70 | var onMouseUp = function () { 71 | if (mode === ToolMode.Default) 72 | return; 73 | if (!containerRef) 74 | return; 75 | if (mode == ToolMode.BoundingBox) { 76 | if (!handleBoundingBoxChange) 77 | return; 78 | if (!bboxStart) 79 | return; 80 | // handleBoundingBoxChange({x, y}); 81 | setBboxStart(null); 82 | } 83 | }; 84 | var onMouseMove = function (e) { 85 | if (mode !== ToolMode.BoundingBox) 86 | return; 87 | if (!containerRef) 88 | return; 89 | if (!handleBoundingBoxChange) 90 | return; 91 | if (!bboxStart) 92 | return; 93 | var pos = e.target.getStage().getPointerPosition(); 94 | var containerRatio = containerRef.current.offsetWidth / image.width; 95 | var x = pos.x / containerRatio; 96 | var y = pos.y / containerRatio; 97 | var top = Math.min(bboxStart.y, y); 98 | var bottom = Math.max(bboxStart.y, y); 99 | var left = Math.min(bboxStart.x, x); 100 | var right = Math.max(bboxStart.x, x); 101 | handleBoundingBoxChange({ topLeft: { x: left, y: top }, bottomRight: { x: right, y: bottom } }); 102 | }; 103 | var containerRatio = containerRef && containerRef.current ? containerRef.current.offsetWidth / image.width : 1; 104 | // Render the image and the predicted mask image on top 105 | return (React.createElement("div", { className: "react-sam-tool", ref: containerRef }, 106 | React.createElement(Stage, { width: width, height: height, style: { cursor: targetSelectEnabled ? 'crossHair' : 'default' } }, 107 | React.createElement(Layer, { onMouseDown: function (e) { return onMouseDown(e); }, onTouchStart: function (e) { return onMouseDown(e); }, onMouseMove: function (e) { return onMouseMove(e); }, onMouseUp: function () { return onMouseUp(); }, onTouchEnd: function () { return onMouseUp(); } }, 108 | React.createElement(Image, { image: image, x: 0, y: 0, width: width, height: height }), 109 | maskImage ? React.createElement(Image, { image: maskImage, x: 0, y: 0, width: width, height: height, opacity: 0.5 }) : null, 110 | bbox ? (React.createElement(Rect, { x: bbox.topLeft.x * containerRatio, y: bbox.topLeft.y * containerRatio, width: (bbox.bottomRight.x - bbox.topLeft.x) * containerRatio, height: (bbox.bottomRight.y - bbox.topLeft.y) * containerRatio, stroke: "green" })) : null, 111 | clicks.map(function (click, index) { return (React.createElement(Circle, { key: index, radius: 5, fill: click.pointType == 1 ? 'green' : 'red', stroke: 'white', strokeWidth: 1, opacity: 0.75, x: click.x * containerRatio, y: click.y * containerRatio })); }))))); 112 | }; 113 | export default Tool; 114 | //# sourceMappingURL=Tool.js.map -------------------------------------------------------------------------------- /dist/Tool.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"Tool.js","sourceRoot":"","sources":["../src/Tool.tsx"],"names":[],"mappings":";;;;;;;;;AAAA,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,OAAO,CAAC;AAC3D,OAAO,EAAa,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEhE,IAAM,IAAI,GAAG,UAAC,EASF;;IARR,mBAAmB;IACnB,kBAAkB,wBAAA,EAClB,uBAAuB,6BAAA,EACvB,KAAK,WAAA,EACL,SAAS,eAAA,EACT,IAAI,UAAA,EACJ,cAAW,EAAX,MAAM,mBAAG,EAAE,KAAA,EACX,YAAuB,EAAvB,IAAI,mBAAG,QAAQ,CAAC,OAAO,KAAA;IAGzB,IAAM,YAAY,GAAG,MAAM,CAAiB,IAAI,CAAC,CAAC;IAC5C,IAAA,KAA4B,QAAQ,CAAgC,IAAI,CAAC,EAAxE,SAAS,QAAA,EAAE,YAAY,QAAiD,CAAC;IAC1E,IAAA,KAAoB,QAAQ,CAAC,CAAC,CAAC,EAA9B,KAAK,QAAA,EAAE,QAAQ,QAAe,CAAC;IAChC,IAAA,KAAsB,QAAQ,CAAC,CAAC,CAAC,EAAhC,MAAM,QAAA,EAAE,SAAS,QAAe,CAAC;IACxC,IAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC7B,IAAM,SAAS,GAAG;QAChB,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY;YAAE,OAAO;QACpC,uDAAuD;QACvD,oEAAoE;QACpE,IAAM,cAAc,GAAG,YAAa,CAAC,OAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;QACxE,QAAQ,CAAC,YAAa,CAAC,OAAQ,CAAC,WAAW,CAAC,CAAC;QAC7C,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,cAAc,CAAC,CAAC;IAC3C,CAAC,CAAC;IACF,IAAM,cAAc,GAAG,IAAI,cAAc,CAAC,UAAC,OAAO;QAChD,KAAoB,UAAO,EAAP,mBAAO,EAAP,qBAAO,EAAP,IAAO,EAAE;YAAxB,IAAM,KAAK,gBAAA;YACd,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE;gBAC3B,SAAS,EAAE,CAAC;aACb;SACF;IACH,CAAC,CAAC,CAAC;IACH,SAAS,CAAC;QACR,SAAS,EAAE,CAAC;QACZ,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO;YACL,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC,CAAC;IACJ,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IAEZ,IAAM,mBAAmB,GAAG,IAAI,KAAK,QAAQ,CAAC,OAAO,CAAC;IAEtD,IAAM,WAAW,GAAG,UAAC,CAAM;QACzB,IAAI,IAAI,KAAK,QAAQ,CAAC,OAAO;YAAE,OAAO;QACtC,IAAI,CAAC,YAAY;YAAE,OAAO;QAE1B,IAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,kBAAkB,EAAE,CAAC;QACrD,IAAM,cAAc,GAAG,YAAa,CAAC,OAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;QAExE,IAAM,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC;QACjC,IAAM,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC;QAEjC,IAAI,IAAI,IAAI,QAAQ,CAAC,cAAc,EAAE;YACnC,IAAI,CAAC,kBAAkB;gBAAE,OAAO;YAChC,kBAAkB,iCAAK,MAAM,UAAE,EAAC,CAAC,GAAA,EAAE,CAAC,GAAA,EAAE,SAAS,EAAE,CAAC,EAAC,UAAE,CAAC;SACvD;QAED,IAAI,IAAI,IAAI,QAAQ,CAAC,cAAc,EAAE;YACnC,IAAI,CAAC,kBAAkB;gBAAE,OAAO;YAChC,kBAAkB,iCAAK,MAAM,UAAE,EAAC,CAAC,GAAA,EAAE,CAAC,GAAA,EAAE,SAAS,EAAE,CAAC,EAAC,UAAE,CAAC;SACvD;QAED,IAAI,IAAI,IAAI,QAAQ,CAAC,WAAW,EAAE;YAChC,YAAY,CAAC,EAAC,CAAC,GAAA,EAAE,CAAC,GAAA,EAAC,CAAC,CAAC;SACtB;IACH,CAAC,CAAC;IAEF,IAAM,SAAS,GAAG;QAChB,IAAI,IAAI,KAAK,QAAQ,CAAC,OAAO;YAAE,OAAO;QACtC,IAAI,CAAC,YAAY;YAAE,OAAO;QAG1B,IAAI,IAAI,IAAI,QAAQ,CAAC,WAAW,EAAE;YAChC,IAAI,CAAC,uBAAuB;gBAAE,OAAO;YACrC,IAAI,CAAC,SAAS;gBAAE,OAAO;YACvB,mCAAmC;YACnC,YAAY,CAAC,IAAI,CAAC,CAAC;SACpB;IACH,CAAC,CAAA;IAED,IAAM,WAAW,GAAG,UAAC,CAAM;QACzB,IAAI,IAAI,KAAK,QAAQ,CAAC,WAAW;YAAE,OAAO;QAC1C,IAAI,CAAC,YAAY;YAAE,OAAO;QAC1B,IAAI,CAAC,uBAAuB;YAAE,OAAO;QACrC,IAAI,CAAC,SAAS;YAAE,OAAO;QAEvB,IAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,kBAAkB,EAAE,CAAC;QACrD,IAAM,cAAc,GAAG,YAAa,CAAC,OAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC;QAExE,IAAM,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC;QACjC,IAAM,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC;QAEjC,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACrC,IAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxC,IAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtC,IAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvC,uBAAuB,CAAC,EAAC,OAAO,EAAE,EAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAC,EAAE,WAAW,EAAE,EAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAC,EAAC,CAAC,CAAC;IAC5F,CAAC,CAAC;IAGF,IAAM,cAAc,GAAG,YAAY,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,YAAa,CAAC,OAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnH,uDAAuD;IACvD,OAAO,CACL,6BAAK,SAAS,EAAC,gBAAgB,EAAC,GAAG,EAAE,YAAY;QAC/C,oBAAC,KAAK,IAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,EAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,EAAC;YAC/F,oBAAC,KAAK,IACJ,WAAW,EAAE,UAAC,CAAC,IAAK,OAAA,WAAW,CAAC,CAAC,CAAC,EAAd,CAAc,EAClC,YAAY,EAAE,UAAC,CAAC,IAAK,OAAA,WAAW,CAAC,CAAC,CAAC,EAAd,CAAc,EACnC,WAAW,EAAE,UAAC,CAAC,IAAK,OAAA,WAAW,CAAC,CAAC,CAAC,EAAd,CAAc,EAClC,SAAS,EAAE,cAAM,OAAA,SAAS,EAAE,EAAX,CAAW,EAC5B,UAAU,EAAE,cAAM,OAAA,SAAS,EAAE,EAAX,CAAW;gBAE3B,oBAAC,KAAK,IAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,GAAI;gBAChE,SAAS,CAAC,CAAC,CAAC,oBAAC,KAAK,IAAC,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,GAAI,CAAC,CAAC,CAAC,IAAI;gBACtG,IAAI,CAAC,CAAC,CAAC,CACN,oBAAC,IAAI,IACH,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,cAAc,EAClC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,cAAc,EAClC,KAAK,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,cAAc,EAC7D,MAAM,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,cAAc,EAC9D,MAAM,EAAC,OAAO,GACd,CACH,CAAC,CAAC,CAAC,IAAI;gBACP,MAAM,CAAC,GAAG,CAAC,UAAC,KAAK,EAAE,KAAK,IAAK,OAAA,CAC5B,oBAAC,MAAM,IAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAC3B,IAAI,EAAE,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAC5C,MAAM,EAAC,OAAO,EACd,WAAW,EAAE,CAAC,EACd,OAAO,EAAE,IAAI,EACb,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,cAAc,EAC3B,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,cAAc,GAAI,CAClC,EAR6B,CAQ7B,CAAC,CACE,CACJ,CAIJ,CACP,CAAC;AACJ,CAAC,CAAC;AAEF,eAAe,IAAI,CAAC"} -------------------------------------------------------------------------------- /dist/helpers/Interfaces.d.ts: -------------------------------------------------------------------------------- 1 | import { Tensor } from "onnxruntime-web"; 2 | export interface ModelScale { 3 | samScale: number; 4 | height: number; 5 | width: number; 6 | } 7 | export interface Click { 8 | x: number; 9 | y: number; 10 | pointType: number; 11 | } 12 | export interface BoundingBox { 13 | topLeft: { 14 | x: number; 15 | y: number; 16 | }; 17 | bottomRight: { 18 | x: number; 19 | y: number; 20 | }; 21 | } 22 | export interface ModelData { 23 | clicks?: Array; 24 | bbox?: BoundingBox; 25 | embedding: Tensor; 26 | modelScale: ModelScale; 27 | } 28 | export interface SegmentAnythingProps { 29 | handleMaskSaved: (mask: HTMLImageElement, image: HTMLImageElement) => void; 30 | image: HTMLImageElement; 31 | embedding: Tensor; 32 | modelUrl: string; 33 | } 34 | export declare enum ToolMode { 35 | Default = "default", 36 | BoundingBox = "boundingBox", 37 | PositivePoints = "positivePoints", 38 | NegativePoints = "negativePoints" 39 | } 40 | export interface ToolProps { 41 | image: HTMLImageElement; 42 | maskImage?: HTMLImageElement; 43 | mode?: ToolMode; 44 | clicks?: Array; 45 | bbox?: BoundingBox; 46 | handleClicksChange?: (e: any) => void; 47 | handleBoundingBoxChange?: (e: any) => void; 48 | } 49 | //# sourceMappingURL=Interfaces.d.ts.map -------------------------------------------------------------------------------- /dist/helpers/Interfaces.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"Interfaces.d.ts","sourceRoot":"","sources":["../../src/helpers/Interfaces.tsx"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,KAAK;IACpB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAC,CAAC;IAChC,WAAW,EAAE;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAC,CAAC;CACrC;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACtB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,eAAe,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC3E,KAAK,EAAE,gBAAgB,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oBAAY,QAAQ;IAClB,OAAO,YAAY;IACnB,WAAW,gBAAgB;IAC3B,cAAc,mBAAmB;IACjC,cAAc,mBAAmB;CAClC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,gBAAgB,CAAC;IACxB,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,MAAM,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACtB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,kBAAkB,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,IAAI,CAAC;IACtC,uBAAuB,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,IAAI,CAAC;CAC5C"} -------------------------------------------------------------------------------- /dist/helpers/Interfaces.js: -------------------------------------------------------------------------------- 1 | export var ToolMode; 2 | (function (ToolMode) { 3 | ToolMode["Default"] = "default"; 4 | ToolMode["BoundingBox"] = "boundingBox"; 5 | ToolMode["PositivePoints"] = "positivePoints"; 6 | ToolMode["NegativePoints"] = "negativePoints"; 7 | })(ToolMode || (ToolMode = {})); 8 | //# sourceMappingURL=Interfaces.js.map -------------------------------------------------------------------------------- /dist/helpers/Interfaces.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"Interfaces.js","sourceRoot":"","sources":["../../src/helpers/Interfaces.tsx"],"names":[],"mappings":"AAiCA,MAAM,CAAN,IAAY,QAKX;AALD,WAAY,QAAQ;IAClB,+BAAmB,CAAA;IACnB,uCAA2B,CAAA;IAC3B,6CAAiC,CAAA;IACjC,6CAAiC,CAAA;AACnC,CAAC,EALW,QAAQ,KAAR,QAAQ,QAKnB"} -------------------------------------------------------------------------------- /dist/helpers/maskUtils.d.ts: -------------------------------------------------------------------------------- 1 | export declare function onnxMaskToImage(input: any, width: number, height: number): HTMLImageElement; 2 | //# sourceMappingURL=maskUtils.d.ts.map -------------------------------------------------------------------------------- /dist/helpers/maskUtils.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"maskUtils.d.ts","sourceRoot":"","sources":["../../src/helpers/maskUtils.tsx"],"names":[],"mappings":"AA4CA,wBAAgB,eAAe,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,oBAExE"} -------------------------------------------------------------------------------- /dist/helpers/maskUtils.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // This source code is licensed under the license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | // Convert the onnx model mask prediction to ImageData 6 | function arrayToImageData(input, width, height) { 7 | var _a = [0, 114, 189, 255], r = _a[0], g = _a[1], b = _a[2], a = _a[3]; // the masks's blue color 8 | var arr = new Uint8ClampedArray(4 * width * height).fill(0); 9 | for (var i = 0; i < input.length; i++) { 10 | // Threshold the onnx model mask prediction at 0.0 11 | // This is equivalent to thresholding the mask using predictor.model.mask_threshold 12 | // in python 13 | if (input[i] > 0.0) { 14 | arr[4 * i + 0] = r; 15 | arr[4 * i + 1] = g; 16 | arr[4 * i + 2] = b; 17 | arr[4 * i + 3] = a; 18 | } 19 | } 20 | return new ImageData(arr, height, width); 21 | } 22 | // Use a Canvas element to produce an image from ImageData 23 | function imageDataToImage(imageData) { 24 | var canvas = imageDataToCanvas(imageData); 25 | var image = new Image(); 26 | image.src = canvas.toDataURL(); 27 | return image; 28 | } 29 | // Canvas elements can be created from ImageData 30 | function imageDataToCanvas(imageData) { 31 | var canvas = document.createElement("canvas"); 32 | var ctx = canvas.getContext("2d"); 33 | canvas.width = imageData.width; 34 | canvas.height = imageData.height; 35 | ctx === null || ctx === void 0 ? void 0 : ctx.putImageData(imageData, 0, 0); 36 | return canvas; 37 | } 38 | // Convert the onnx model mask output to an HTMLImageElement 39 | export function onnxMaskToImage(input, width, height) { 40 | return imageDataToImage(arrayToImageData(input, width, height)); 41 | } 42 | //# sourceMappingURL=maskUtils.js.map -------------------------------------------------------------------------------- /dist/helpers/maskUtils.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"maskUtils.js","sourceRoot":"","sources":["../../src/helpers/maskUtils.tsx"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,uBAAuB;AAEvB,8DAA8D;AAC9D,0DAA0D;AAE1D,sDAAsD;AACtD,SAAS,gBAAgB,CAAC,KAAU,EAAE,KAAa,EAAE,MAAc;IAC3D,IAAA,KAAe,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAhC,CAAC,QAAA,EAAE,CAAC,QAAA,EAAE,CAAC,QAAA,EAAE,CAAC,QAAsB,CAAC,CAAC,yBAAyB;IAClE,IAAM,GAAG,GAAG,IAAI,iBAAiB,CAAC,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAErC,kDAAkD;QAClD,mFAAmF;QACnF,YAAY;QACZ,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE;YAClB,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;SACpB;KACF;IACD,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,0DAA0D;AAC1D,SAAS,gBAAgB,CAAC,SAAoB;IAC5C,IAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;IAC1B,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAC/B,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gDAAgD;AAChD,SAAS,iBAAiB,CAAC,SAAoB;IAC7C,IAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAC/B,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;IACjC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,YAAY,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,eAAe,CAAC,KAAU,EAAE,KAAa,EAAE,MAAc;IACvE,OAAO,gBAAgB,CAAC,gBAAgB,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC"} -------------------------------------------------------------------------------- /dist/helpers/onnxModelAPI.d.ts: -------------------------------------------------------------------------------- 1 | import { Tensor } from "onnxruntime-web"; 2 | import { ModelData } from "./Interfaces"; 3 | declare const modelData: ({ clicks, bbox, embedding, modelScale }: ModelData) => { 4 | image_embeddings: Tensor; 5 | point_coords: import("onnxruntime-web").TypedTensor<"float32">; 6 | point_labels: import("onnxruntime-web").TypedTensor<"float32">; 7 | orig_im_size: import("onnxruntime-web").TypedTensor<"float32">; 8 | mask_input: import("onnxruntime-web").TypedTensor<"float32">; 9 | has_mask_input: import("onnxruntime-web").TypedTensor<"float32">; 10 | } | undefined; 11 | export { modelData }; 12 | //# sourceMappingURL=onnxModelAPI.d.ts.map -------------------------------------------------------------------------------- /dist/helpers/onnxModelAPI.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"onnxModelAPI.d.ts","sourceRoot":"","sources":["../../src/helpers/onnxModelAPI.tsx"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,QAAA,MAAM,SAAS,4CAA6C,SAAS;;;;;;;aAmEpE,CAAC;AAEF,OAAO,EAAE,SAAS,EAAE,CAAC"} -------------------------------------------------------------------------------- /dist/helpers/onnxModelAPI.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // This source code is licensed under the license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | import { Tensor } from "onnxruntime-web"; 6 | var modelData = function (_a) { 7 | var clicks = _a.clicks, bbox = _a.bbox, embedding = _a.embedding, modelScale = _a.modelScale; 8 | var pointCoords; 9 | var pointLabels; 10 | var pointCoordsTensor; 11 | var pointLabelsTensor; 12 | // Check there are input click prompts 13 | if (clicks) { 14 | var n = clicks.length; 15 | var padding = bbox ? 2 : 1; 16 | pointCoords = new Float32Array(2 * (n + padding)); 17 | pointLabels = new Float32Array(n + padding); 18 | // Add clicks and scale to what SAM expects 19 | for (var i = 0; i < n; i++) { 20 | pointCoords[2 * i] = clicks[i].x * modelScale.samScale; 21 | pointCoords[2 * i + 1] = clicks[i].y * modelScale.samScale; 22 | pointLabels[i] = clicks[i].pointType; 23 | } 24 | if (bbox) { 25 | // Add in the extra point/label for the bounding box 26 | pointCoords[2 * n] = bbox.topLeft.x * modelScale.samScale; 27 | pointCoords[2 * n + 1] = bbox.topLeft.y * modelScale.samScale; 28 | pointLabels[n] = 2.0; 29 | pointCoords[2 * (n + 1)] = bbox.bottomRight.x * modelScale.samScale; 30 | pointCoords[2 * (n + 1) + 1] = bbox.bottomRight.y * modelScale.samScale; 31 | pointLabels[n + 1] = 3.0; 32 | } 33 | else { 34 | // Add in the extra point/label when only clicks and no box 35 | // The extra point is at (0, 0) with label -1 36 | pointCoords[2 * n] = 0.0; 37 | pointCoords[2 * n + 1] = 0.0; 38 | pointLabels[n] = -1.0; 39 | } 40 | // Create the tensor 41 | pointCoordsTensor = new Tensor("float32", pointCoords, [1, n + padding, 2]); 42 | pointLabelsTensor = new Tensor("float32", pointLabels, [1, n + padding]); 43 | } 44 | var imageSizeTensor = new Tensor("float32", [ 45 | modelScale.height, 46 | modelScale.width, 47 | ]); 48 | if (pointCoordsTensor === undefined || pointLabelsTensor === undefined) 49 | return; 50 | // There is no previous mask, so default to an empty tensor 51 | var maskInput = new Tensor("float32", new Float32Array(256 * 256), [1, 1, 256, 256]); 52 | // There is no previous mask, so default to 0 53 | var hasMaskInput = new Tensor("float32", [0]); 54 | return { 55 | image_embeddings: embedding, 56 | point_coords: pointCoordsTensor, 57 | point_labels: pointLabelsTensor, 58 | orig_im_size: imageSizeTensor, 59 | mask_input: maskInput, 60 | has_mask_input: hasMaskInput, 61 | }; 62 | }; 63 | export { modelData }; 64 | //# sourceMappingURL=onnxModelAPI.js.map -------------------------------------------------------------------------------- /dist/helpers/onnxModelAPI.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"onnxModelAPI.js","sourceRoot":"","sources":["../../src/helpers/onnxModelAPI.tsx"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,uBAAuB;AAEvB,8DAA8D;AAC9D,0DAA0D;AAE1D,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGzC,IAAM,SAAS,GAAG,UAAC,EAAkD;QAAhD,MAAM,YAAA,EAAE,IAAI,UAAA,EAAE,SAAS,eAAA,EAAE,UAAU,gBAAA;IACtD,IAAI,WAAW,CAAC;IAChB,IAAI,WAAW,CAAC;IAChB,IAAI,iBAAiB,CAAC;IACtB,IAAI,iBAAiB,CAAC;IAEtB,sCAAsC;IACtC,IAAI,MAAM,EAAE;QACV,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAEtB,IAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,WAAW,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QAClD,WAAW,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;QAE5C,2CAA2C;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAC1B,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC;YACvD,WAAW,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC;YAC3D,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;SACtC;QAED,IAAI,IAAI,EAAE;YACR,oDAAoD;YACpD,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC;YAC1D,WAAW,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC;YAC9D,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;YACrB,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC;YAClE,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC;YACtE,WAAW,CAAC,CAAC,GAAC,CAAC,CAAC,GAAG,GAAG,CAAC;SACxB;aAAM;YACL,2DAA2D;YAC3D,6CAA6C;YAC7C,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;YACzB,WAAW,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;YAC7B,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;SACvB;QAGD,oBAAoB;QACpB,iBAAiB,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,iBAAiB,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;KAC1E;IACD,IAAM,eAAe,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE;QAC5C,UAAU,CAAC,MAAM;QACjB,UAAU,CAAC,KAAK;KACjB,CAAC,CAAC;IAEH,IAAI,iBAAiB,KAAK,SAAS,IAAI,iBAAiB,KAAK,SAAS;QACpE,OAAO;IAET,2DAA2D;IAC3D,IAAM,SAAS,GAAG,IAAI,MAAM,CAC1B,SAAS,EACT,IAAI,YAAY,CAAC,GAAG,GAAG,GAAG,CAAC,EAC3B,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CACjB,CAAC;IACF,6CAA6C;IAC7C,IAAM,YAAY,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhD,OAAO;QACL,gBAAgB,EAAE,SAAS;QAC3B,YAAY,EAAE,iBAAiB;QAC/B,YAAY,EAAE,iBAAiB;QAC/B,YAAY,EAAE,eAAe;QAC7B,UAAU,EAAE,SAAS;QACrB,cAAc,EAAE,YAAY;KAC7B,CAAC;AACJ,CAAC,CAAC;AAEF,OAAO,EAAE,SAAS,EAAE,CAAC"} -------------------------------------------------------------------------------- /dist/helpers/scaleHelper.d.ts: -------------------------------------------------------------------------------- 1 | declare const handleImageScale: (image: HTMLImageElement) => { 2 | height: number; 3 | width: number; 4 | samScale: number; 5 | }; 6 | export { handleImageScale }; 7 | //# sourceMappingURL=scaleHelper.d.ts.map -------------------------------------------------------------------------------- /dist/helpers/scaleHelper.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"scaleHelper.d.ts","sourceRoot":"","sources":["../../src/helpers/scaleHelper.tsx"],"names":[],"mappings":"AAQA,QAAA,MAAM,gBAAgB,UAAW,gBAAgB;;;;CAOhD,CAAC;AAEF,OAAO,EAAE,gBAAgB,EAAE,CAAC"} -------------------------------------------------------------------------------- /dist/helpers/scaleHelper.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | // This source code is licensed under the license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | // Helper function for handling image scaling needed for SAM 6 | var handleImageScale = function (image) { 7 | // Input images to SAM must be resized so the longest side is 1024 8 | var LONG_SIDE_LENGTH = 1024; 9 | var w = image.naturalWidth; 10 | var h = image.naturalHeight; 11 | var samScale = LONG_SIDE_LENGTH / Math.max(h, w); 12 | return { height: h, width: w, samScale: samScale }; 13 | }; 14 | export { handleImageScale }; 15 | //# sourceMappingURL=scaleHelper.js.map -------------------------------------------------------------------------------- /dist/helpers/scaleHelper.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"scaleHelper.js","sourceRoot":"","sources":["../../src/helpers/scaleHelper.tsx"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,uBAAuB;AAEvB,8DAA8D;AAC9D,0DAA0D;AAG1D,4DAA4D;AAC5D,IAAM,gBAAgB,GAAG,UAAC,KAAuB;IAC/C,kEAAkE;IAClE,IAAM,gBAAgB,GAAG,IAAI,CAAC;IAC9B,IAAI,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC;IAC3B,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC;IAC5B,IAAM,QAAQ,GAAG,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,UAAA,EAAE,CAAC;AAC3C,CAAC,CAAC;AAEF,OAAO,EAAE,gBAAgB,EAAE,CAAC"} -------------------------------------------------------------------------------- /dist/index.d.ts: -------------------------------------------------------------------------------- 1 | import { SegmentAnythingProps } from './helpers/Interfaces'; 2 | import React from 'react'; 3 | export declare const SegmentAnything: ({ image, embedding, modelUrl }: SegmentAnythingProps) => React.JSX.Element; 4 | //# sourceMappingURL=index.d.ts.map -------------------------------------------------------------------------------- /dist/index.d.ts.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.tsx"],"names":[],"mappings":"AAuBA,OAAO,EAAE,oBAAoB,EAAgC,MAAM,sBAAsB,CAAC;AAG1F,OAAO,KAA8B,MAAM,OAAO,CAAC;AAKnD,eAAO,MAAM,eAAe,mCAAkC,oBAAoB,sBA6IjF,CAAA"} -------------------------------------------------------------------------------- /dist/index.js: -------------------------------------------------------------------------------- 1 | // import * as React from 'react' 2 | // import styles from './styles.module.css' 3 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 4 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 5 | return new (P || (P = Promise))(function (resolve, reject) { 6 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 7 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 8 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 9 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 10 | }); 11 | }; 12 | var __generator = (this && this.__generator) || function (thisArg, body) { 13 | var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; 14 | return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; 15 | function verb(n) { return function (v) { return step([n, v]); }; } 16 | function step(op) { 17 | if (f) throw new TypeError("Generator is already executing."); 18 | while (g && (g = 0, op[0] && (_ = 0)), _) try { 19 | if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; 20 | if (y = 0, t) op = [op[0] & 2, t.value]; 21 | switch (op[0]) { 22 | case 0: case 1: t = op; break; 23 | case 4: _.label++; return { value: op[1], done: false }; 24 | case 5: _.label++; y = op[1]; op = [0]; continue; 25 | case 7: op = _.ops.pop(); _.trys.pop(); continue; 26 | default: 27 | if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } 28 | if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } 29 | if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } 30 | if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } 31 | if (t[2]) _.ops.pop(); 32 | _.trys.pop(); continue; 33 | } 34 | op = body.call(thisArg, _); 35 | } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } 36 | if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; 37 | } 38 | }; 39 | // export * as SegmentAnything from './SegmentAnything'; 40 | // import React from 'react' 41 | import { Button, Stack, Card, CardContent, Box, Typography, Grid } from "@mui/material"; 42 | import BoundingBoxIcon from '@mui/icons-material/HighlightAlt'; 43 | import PointIcon from '@mui/icons-material/AdsClick'; 44 | import CheckIcon from '@mui/icons-material/Check'; 45 | import { modelData } from './helpers/onnxModelAPI'; 46 | import { onnxMaskToImage } from "./helpers/maskUtils"; 47 | import { ToolMode } from './helpers/Interfaces'; 48 | import Tool from './Tool'; 49 | import React, { useEffect } from 'react'; 50 | import { InferenceSession } from "onnxruntime-web"; 51 | /* @ts-ignore */ 52 | import * as _ from "underscore"; 53 | export var SegmentAnything = function (_a) { 54 | var image = _a.image, embedding = _a.embedding, modelUrl = _a.modelUrl; 55 | // ONNX model 56 | var _b = React.useState(null), model = _b[0], setModel = _b[1]; 57 | // Tool mode 58 | var _c = React.useState(ToolMode.Default), mode = _c[0], setMode = _c[1]; // Tool mode 59 | // Model parameters (clicks and bounding box) 60 | var _d = React.useState([]), clicks = _d[0], setClicks = _d[1]; 61 | var _e = React.useState(undefined), bbox = _e[0], setBbox = _e[1]; 62 | // Mask image 63 | var _f = React.useState(undefined), maskImage = _f[0], setMaskImage = _f[1]; 64 | // Handler for the clear all clicks button 65 | var clearAllClicks = function () { 66 | setClicks([]); 67 | setMode(ToolMode.Default); 68 | }; 69 | // Handler for the clear all bbox button 70 | var clearBBox = function () { 71 | setBbox(undefined); 72 | setMode(ToolMode.Default); 73 | }; 74 | // Input images to SAM must be resized so the longest side is 1024 75 | var LONG_SIDE_LENGTH = 1024; 76 | var w = image.naturalWidth; 77 | var h = image.naturalHeight; 78 | var samScale = LONG_SIDE_LENGTH / Math.max(h, w); 79 | var modelScale = { width: w, height: h, samScale: samScale }; 80 | // Initialize the ONNX model. load the image, and load the SAM 81 | // pre-computed image embedding 82 | useEffect(function () { 83 | // Initialize the ONNX model 84 | var initModel = function () { return __awaiter(void 0, void 0, void 0, function () { 85 | var model_1, e_1; 86 | return __generator(this, function (_a) { 87 | switch (_a.label) { 88 | case 0: 89 | _a.trys.push([0, 2, , 3]); 90 | return [4 /*yield*/, InferenceSession.create(modelUrl)]; 91 | case 1: 92 | model_1 = _a.sent(); 93 | setModel(model_1); 94 | return [3 /*break*/, 3]; 95 | case 2: 96 | e_1 = _a.sent(); 97 | console.log(e_1); 98 | return [3 /*break*/, 3]; 99 | case 3: return [2 /*return*/]; 100 | } 101 | }); 102 | }); }; 103 | initModel(); 104 | }, []); 105 | // Run the ONNX model and generate a mask image 106 | var runONNX = function () { return __awaiter(void 0, void 0, void 0, function () { 107 | var feeds, results, output, e_2; 108 | return __generator(this, function (_a) { 109 | switch (_a.label) { 110 | case 0: 111 | _a.trys.push([0, 2, , 3]); 112 | if (model === null || clicks === null) 113 | return [2 /*return*/]; 114 | feeds = modelData({ 115 | clicks: clicks, 116 | bbox: bbox, 117 | embedding: embedding, 118 | modelScale: modelScale, 119 | }); 120 | if (feeds === undefined) 121 | return [2 /*return*/]; 122 | return [4 /*yield*/, model.run(feeds)]; 123 | case 1: 124 | results = _a.sent(); 125 | output = results[model.outputNames[0]]; 126 | // The predicted mask returned from the ONNX model is an array which is 127 | // rendered as an HTML image using onnxMaskToImage() from maskUtils.tsx. 128 | setMaskImage(onnxMaskToImage(output.data, output.dims[2], output.dims[3])); 129 | return [3 /*break*/, 3]; 130 | case 2: 131 | e_2 = _a.sent(); 132 | console.log(e_2); 133 | return [3 /*break*/, 3]; 134 | case 3: return [2 /*return*/]; 135 | } 136 | }); 137 | }); }; 138 | var throttledRunONNX = _.throttle(runONNX, 15); 139 | // Run the ONNX model every time clicks or bbox have changed 140 | useEffect(function () { 141 | runONNX(); 142 | }, [clicks]); 143 | useEffect(function () { 144 | throttledRunONNX(); 145 | }, [bbox]); 146 | return (React.createElement("div", { className: "react-sam" }, 147 | React.createElement(Box, { sx: { flexGrow: 1 } }, 148 | React.createElement(Grid, { container: true, spacing: 2 }, 149 | React.createElement(Grid, { item: true, xs: 3 }, 150 | React.createElement(Stack, { spacing: 2, className: "react-sam-toolbar" }, 151 | React.createElement(Card, null, 152 | React.createElement(CardContent, null, 153 | React.createElement(Typography, { className: "react-sam-toolbar-item-title", sx: { mb: 2 } }, 154 | React.createElement(BoundingBoxIcon, null), 155 | " Bounding Box"), 156 | React.createElement(Stack, null, 157 | React.createElement(Typography, { sx: { mb: 1 } }, 158 | React.createElement(Button, { onClick: function () { return setMode(ToolMode.BoundingBox); }, variant: "contained" }, "Select")), 159 | React.createElement(Typography, { sx: { mb: 1 } }, 160 | React.createElement(Button, { onClick: clearBBox, variant: "contained", color: "error" }, "Clear All"))))), 161 | React.createElement(Card, null, 162 | React.createElement(CardContent, null, 163 | React.createElement(Typography, { className: "react-sam-toolbar-item-title", sx: { mb: 2 } }, 164 | React.createElement(PointIcon, null), 165 | " Points"), 166 | React.createElement(Stack, null, 167 | React.createElement(Typography, { sx: { mb: 1 } }, 168 | React.createElement(Button, { onClick: function () { return setMode(ToolMode.PositivePoints); }, variant: "contained" }, "Add Positive Points")), 169 | React.createElement(Typography, { sx: { mb: 1 } }, 170 | React.createElement(Button, { onClick: function () { return setMode(ToolMode.NegativePoints); }, variant: "contained" }, "Add Negative Points")), 171 | React.createElement(Typography, { sx: { mb: 1 } }, 172 | React.createElement(Button, { onClick: clearAllClicks, variant: "contained", color: "error" }, "Clear All"))))), 173 | React.createElement(Button, { sx: { mt: 2 }, className: 'react-sam-toolbar-save', startIcon: React.createElement(CheckIcon, null), size: "large", color: "success", variant: "contained" }, "Save Mask"))), 174 | React.createElement(Grid, { item: true, xs: 9 }, 175 | React.createElement(Tool, { image: image, maskImage: maskImage, mode: mode, clicks: clicks, bbox: bbox, handleClicksChange: function (newClicks) { return setClicks(newClicks); }, handleBoundingBoxChange: function (newBbox) { return setBbox(newBbox); } })))))); 176 | }; 177 | // export const SegmentAnything; 178 | //# sourceMappingURL=index.js.map -------------------------------------------------------------------------------- /dist/index.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.tsx"],"names":[],"mappings":"AAAA,iCAAiC;AACjC,2CAA2C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE3C,wDAAwD;AAExD,4BAA4B;AAE5B,OAAO,EACH,MAAM,EACN,KAAK,EACL,IAAI,EACJ,WAAW,EACX,GAAG,EACH,UAAU,EACV,IAAI,EACP,MAAM,eAAe,CAAC;AAEvB,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAC/D,OAAO,SAAS,MAAM,8BAA8B,CAAC;AACrD,OAAO,SAAS,MAAM,2BAA2B,CAAC;AAElD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAwB,QAAQ,EAAsB,MAAM,sBAAsB,CAAC;AAC1F,OAAO,IAAI,MAAM,QAAQ,CAAC;AAE1B,OAAO,KAAK,EAAE,EAAY,SAAS,EAAE,MAAM,OAAO,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,gBAAgB;AAChB,OAAO,KAAK,CAAC,MAAM,YAAY,CAAC;AAEhC,MAAM,CAAC,IAAM,eAAe,GAAG,UAAC,EAAkD;QAAjD,KAAK,WAAA,EAAE,SAAS,eAAA,EAAE,QAAQ,cAAA;IACvD,aAAa;IACP,IAAA,KAAoB,KAAK,CAAC,QAAQ,CAA0B,IAAI,CAAC,EAAhE,KAAK,QAAA,EAAE,QAAQ,QAAiD,CAAC;IAExE,YAAY;IACN,IAAA,KAAkB,KAAK,CAAC,QAAQ,CAAW,QAAQ,CAAC,OAAO,CAAC,EAA3D,IAAI,QAAA,EAAE,OAAO,QAA8C,CAAC,CAAC,YAAY;IAEhF,6CAA6C;IACvC,IAAA,KAAsB,KAAK,CAAC,QAAQ,CAAU,EAAE,CAAC,EAAhD,MAAM,QAAA,EAAE,SAAS,QAA+B,CAAC;IAClD,IAAA,KAAkB,KAAK,CAAC,QAAQ,CAA0B,SAAS,CAAC,EAAnE,IAAI,QAAA,EAAE,OAAO,QAAsD,CAAC;IAE3E,aAAa;IACP,IAAA,KAA4B,KAAK,CAAC,QAAQ,CAA+B,SAAS,CAAC,EAAlF,SAAS,QAAA,EAAE,YAAY,QAA2D,CAAC;IAE1F,0CAA0C;IAC1C,IAAM,cAAc,GAAG;QACnB,SAAS,CAAC,EAAE,CAAC,CAAC;QACd,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC,CAAA;IAED,wCAAwC;IACxC,IAAM,SAAS,GAAG;QACd,OAAO,CAAC,SAAS,CAAC,CAAC;QACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC,CAAA;IAED,kEAAkE;IAClE,IAAM,gBAAgB,GAAG,IAAI,CAAC;IAC9B,IAAI,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC;IAC3B,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC;IAC5B,IAAM,QAAQ,GAAG,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,IAAM,UAAU,GAAG,EAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAC,CAAA;IAE5D,8DAA8D;IAC9D,+BAA+B;IAC/B,SAAS,CAAC;QACN,4BAA4B;QAC5B,IAAM,SAAS,GAAG;;;;;;wBAEA,qBAAM,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAA;;wBAA/C,UAAQ,SAAuC;wBACrD,QAAQ,CAAC,OAAK,CAAC,CAAC;;;;wBAEhB,OAAO,CAAC,GAAG,CAAC,GAAC,CAAC,CAAC;;;;;aAElB,CAAC;QACF,SAAS,EAAE,CAAC;IAChB,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,+CAA+C;IAC/C,IAAM,OAAO,GAAG;;;;;;oBAER,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI;wBAAE,sBAAO;oBAIxC,KAAK,GAAG,SAAS,CAAC;wBACpB,MAAM,QAAA;wBACN,IAAI,MAAA;wBACJ,SAAS,WAAA;wBACT,UAAU,YAAA;qBACb,CAAC,CAAC;oBACH,IAAI,KAAK,KAAK,SAAS;wBAAE,sBAAO;oBAEhB,qBAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAA;;oBAAhC,OAAO,GAAG,SAAsB;oBAChC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7C,wEAAwE;oBACxE,wEAAwE;oBACxE,YAAY,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;;;;oBAE3E,OAAO,CAAC,GAAG,CAAC,GAAC,CAAC,CAAC;;;;;SAEtB,CAAC;IAEF,IAAM,gBAAgB,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAEjD,4DAA4D;IAC5D,SAAS,CAAC;QACN,OAAO,EAAE,CAAC;IACd,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;IAEb,SAAS,CAAC;QACN,gBAAgB,EAAE,CAAC;IACvB,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;IAEV,OAAO,CACH,6BAAK,SAAS,EAAC,WAAW;QACtB,oBAAC,GAAG,IAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE;YACpB,oBAAC,IAAI,IAAC,SAAS,QAAC,OAAO,EAAE,CAAC;gBACtB,oBAAC,IAAI,IAAC,IAAI,QAAC,EAAE,EAAE,CAAC;oBACZ,oBAAC,KAAK,IAAC,OAAO,EAAE,CAAC,EAAE,SAAS,EAAC,mBAAmB;wBAC5C,oBAAC,IAAI;4BACD,oBAAC,WAAW;gCACR,oBAAC,UAAU,IAAC,SAAS,EAAC,8BAA8B,EAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;oCAC9D,oBAAC,eAAe,OAAG;oDACV;gCACb,oBAAC,KAAK;oCACF,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAM,OAAA,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAA7B,CAA6B,EAAE,OAAO,EAAC,WAAW,aAAgB,CAChF;oCACb,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAC,WAAW,EAAC,KAAK,EAAC,OAAO,gBAAmB,CACvE,CACT,CACE,CACX;wBACP,oBAAC,IAAI;4BACD,oBAAC,WAAW;gCACR,oBAAC,UAAU,IAAC,SAAS,EAAC,8BAA8B,EAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;oCAC9D,oBAAC,SAAS,OAAG;8CACJ;gCACb,oBAAC,KAAK;oCACF,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAM,OAAA,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAhC,CAAgC,EAAE,OAAO,EAAC,WAAW,0BAA6B,CAChG;oCACb,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAM,OAAA,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAhC,CAAgC,EAAE,OAAO,EAAC,WAAW,0BAA6B,CAChG;oCACb,oBAAC,UAAU,IAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;wCACrB,oBAAC,MAAM,IAAC,OAAO,EAAE,cAAc,EAAE,OAAO,EAAC,WAAW,EAAC,KAAK,EAAC,OAAO,gBAAmB,CAC5E,CACT,CACE,CACX;wBACP,oBAAC,MAAM,IAAC,EAAE,EAAE,EAAC,EAAE,EAAE,CAAC,EAAC,EAAE,SAAS,EAAC,wBAAwB,EAAC,SAAS,EAAE,oBAAC,SAAS,OAAG,EAAE,IAAI,EAAC,OAAO,EAAC,KAAK,EAAC,SAAS,EAAC,OAAO,EAAC,WAAW,gBAAmB,CACjJ,CACL;gBACP,oBAAC,IAAI,IAAC,IAAI,QAAC,EAAE,EAAE,CAAC;oBACZ,oBAAC,IAAI,IACD,KAAK,EAAE,KAAK,EACZ,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,IAAI,EACV,kBAAkB,EAAE,UAAC,SAAkB,IAAK,OAAA,SAAS,CAAC,SAAS,CAAC,EAApB,CAAoB,EAChE,uBAAuB,EAAE,UAAC,OAAgC,IAAK,OAAA,OAAO,CAAC,OAAO,CAAC,EAAhB,CAAgB,GAC7E,CACH,CACJ,CACL,CACJ,CACT,CAAC;AACN,CAAC,CAAA;AAED,gCAAgC"} -------------------------------------------------------------------------------- /dist/types.d.ts: -------------------------------------------------------------------------------- 1 | import { Tensor } from 'onnxruntime-web'; 2 | import React from 'react'; 3 | 4 | interface SegmentAnythingProps { 5 | handleMaskSaved: (mask: HTMLImageElement, image: HTMLImageElement) => void; 6 | image: HTMLImageElement; 7 | embedding: Tensor; 8 | modelUrl: string; 9 | } 10 | 11 | declare const SegmentAnything: ({ image, embedding, modelUrl }: SegmentAnythingProps) => React.JSX.Element; 12 | 13 | export { SegmentAnything }; 14 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | This example was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 2 | 3 | It is linked to the react-segment-anything package in the parent directory for development purposes. 4 | 5 | You can run `npm install` and then `npm start` to test your package. 6 | -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "react-segment-anything-example", 3 | "homepage": ".", 4 | "version": "0.0.0", 5 | "private": true, 6 | "scripts": { 7 | "start": "node node_modules/react-scripts/bin/react-scripts.js start", 8 | "build": "node node_modules/react-scripts/bin/react-scripts.js build", 9 | "test": "node node_modules/react-scripts/bin/react-scripts.js test", 10 | "eject": "node node_modules/react-scripts/bin/react-scripts.js eject" 11 | }, 12 | "dependencies": { 13 | "@emotion/react": "^11.10.6", 14 | "@emotion/styled": "^11.10.6", 15 | "@fontsource/roboto": "^4.5.8", 16 | "@mui/icons-material": "file:../node_modules/@mui/icons-material", 17 | "@mui/material": "file:../node_modules/@mui/material", 18 | "@testing-library/jest-dom": "^5.16.5", 19 | "@testing-library/react": "^13.4.0", 20 | "@testing-library/user-event": "^13.5.0", 21 | "@types/jest": "^27.5.2", 22 | "@types/node": "file:../node_modules/@types/node", 23 | "@types/react": "file:../node_modules/@types/react", 24 | "@types/react-dom": "file:../node_modules/@types/react-dom", 25 | "@types/resize-observer-browser": "^0.1.7", 26 | "@typescript-eslint/eslint-plugin": "^2.26.0", 27 | "@typescript-eslint/parser": "^2.26.0", 28 | "array-move": "^4.0.0", 29 | "konva": "file:../node_modules/konva", 30 | "npyjs": "^0.4.0", 31 | "onnxruntime-web": "1.14.0", 32 | "postcss-preset-env": "^8.4.2", 33 | "react": "file:../node_modules/react", 34 | "react-dom": "file:../node_modules/react-dom", 35 | "react-konva": "file:../node_modules/react-konva", 36 | "react-scripts": "5.0.1", 37 | "react-segment-anything": "file:..", 38 | "typescript": "^5.1.3", 39 | "underscore": "^1.13.6", 40 | "use-image": "^1.1.0" 41 | }, 42 | "devDependencies": { 43 | "@babel/plugin-syntax-object-rest-spread": "^7.8.3" 44 | }, 45 | "browserslist": { 46 | "production": [ 47 | ">0.2%", 48 | "not dead", 49 | "not op_mini all" 50 | ], 51 | "development": [ 52 | "last 1 chrome version", 53 | "last 1 firefox version", 54 | "last 1 safari version" 55 | ] 56 | }, 57 | "browser": { 58 | "fs": false, 59 | "path": false, 60 | "os": false 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /example/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/favicon.ico -------------------------------------------------------------------------------- /example/public/groceries.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/groceries.jpg -------------------------------------------------------------------------------- /example/public/groceries_embedding.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/groceries_embedding.npy -------------------------------------------------------------------------------- /example/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 10 | 11 | 12 | 16 | 17 | 18 | 27 | 28 | 64 | react-segment-anything 65 | 66 | 67 | 68 | 71 | 72 |
73 | 74 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /example/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/logo192.png -------------------------------------------------------------------------------- /example/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "react-segment-anything", 3 | "name": "react-segment-anything", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /example/public/sam_onnx_quantized_example.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/sam_onnx_quantized_example.onnx -------------------------------------------------------------------------------- /example/public/static/js/ort-wasm-simd-threaded.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/static/js/ort-wasm-simd-threaded.wasm -------------------------------------------------------------------------------- /example/public/static/js/ort-wasm-simd.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/static/js/ort-wasm-simd.wasm -------------------------------------------------------------------------------- /example/public/static/js/ort-wasm-threaded.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/static/js/ort-wasm-threaded.wasm -------------------------------------------------------------------------------- /example/public/static/js/ort-wasm.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mmurray/react-segment-anything/dd23472715fc68242892577b35cfe2eb79143e2a/example/public/static/js/ort-wasm.wasm -------------------------------------------------------------------------------- /example/src/DemoApp.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { Tensor } from "onnxruntime-web"; 3 | 4 | /* @ts-ignore */ 5 | import npyjs from "npyjs"; 6 | 7 | import Container from '@mui/material/Container'; 8 | import { SegmentAnything } from 'react-segment-anything'; 9 | 10 | const ort = require("onnxruntime-web"); 11 | 12 | const IMAGE_EMBEDDING = "/groceries_embedding.npy"; 13 | const IMAGE_PATH = "/groceries.jpg"; 14 | 15 | const DemoApp = () => { 16 | 17 | const [image, setImage] = useState(undefined); 18 | const loadImage = async (imageFile: string) => { 19 | const img = new Image(); 20 | img.src = imageFile; 21 | img.onload = () => setImage(img); 22 | }; 23 | 24 | const [tensor, setTensor] = useState(null); 25 | const loadNpyTensor = async (tensorFile: string, dType: string) => { 26 | let npLoader = new npyjs(); 27 | const npArray = await npLoader.load(tensorFile); 28 | const tensor = new ort.Tensor(dType, npArray.data, npArray.shape); 29 | return tensor; 30 | }; 31 | 32 | useEffect(() => { 33 | Promise.resolve(loadNpyTensor(IMAGE_EMBEDDING, "float32")).then( 34 | (embedding) => setTensor(embedding) 35 | ); 36 | Promise.resolve(loadImage(IMAGE_PATH)); 37 | }, []); 38 | 39 | if (!image || !tensor) return
Loading...
; 40 | 41 | return ( 42 | 43 | {}} 45 | image={image} 46 | embedding={tensor} 47 | modelUrl="/sam_onnx_quantized_example.onnx" 48 | /> 49 | 50 | ); 51 | } 52 | 53 | export default DemoApp; -------------------------------------------------------------------------------- /example/src/helpers/Interfaces.tsx: -------------------------------------------------------------------------------- 1 | import { Tensor } from "onnxruntime-web"; 2 | 3 | export interface ModelScale { 4 | samScale: number; 5 | height: number; 6 | width: number; 7 | } 8 | 9 | export interface Click { 10 | x: number; 11 | y: number; 12 | pointType: number; 13 | } 14 | 15 | export interface BoundingBox { 16 | topLeft: {x: number, y: number}; 17 | bottomRight: {x: number, y: number}; 18 | } 19 | 20 | export interface ModelData { 21 | clicks?: Array; 22 | bbox?: BoundingBox; 23 | embedding: Tensor; 24 | modelScale: ModelScale; 25 | } 26 | 27 | export interface SegmentAnythingProps { 28 | handleMaskSaved: (e: any) => void; 29 | image: HTMLImageElement; 30 | embedding: Tensor; 31 | modelUrl: string; 32 | } 33 | 34 | export enum ToolMode { 35 | Default = "default", 36 | BoundingBox = "boundingBox", 37 | PositivePoints = "positivePoints", 38 | NegativePoints = "negativePoints", 39 | } 40 | 41 | export interface ToolProps { 42 | image: HTMLImageElement; 43 | maskImage?: HTMLImageElement; 44 | mode?: ToolMode; 45 | clicks?: Array; 46 | bbox?: BoundingBox; 47 | handleClicksChange?: (e: any) => void; 48 | handleBoundingBoxChange?: (e: any) => void; 49 | } 50 | -------------------------------------------------------------------------------- /example/src/helpers/maskUtils.tsx: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | 4 | // This source code is licensed under the license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | // Convert the onnx model mask prediction to ImageData 8 | function arrayToImageData(input: any, width: number, height: number) { 9 | const [r, g, b, a] = [0, 114, 189, 255]; // the masks's blue color 10 | const arr = new Uint8ClampedArray(4 * width * height).fill(0); 11 | for (let i = 0; i < input.length; i++) { 12 | 13 | // Threshold the onnx model mask prediction at 0.0 14 | // This is equivalent to thresholding the mask using predictor.model.mask_threshold 15 | // in python 16 | if (input[i] > 0.0) { 17 | arr[4 * i + 0] = r; 18 | arr[4 * i + 1] = g; 19 | arr[4 * i + 2] = b; 20 | arr[4 * i + 3] = a; 21 | } 22 | } 23 | return new ImageData(arr, height, width); 24 | } 25 | 26 | // Use a Canvas element to produce an image from ImageData 27 | function imageDataToImage(imageData: ImageData) { 28 | const canvas = imageDataToCanvas(imageData); 29 | const image = new Image(); 30 | image.src = canvas.toDataURL(); 31 | return image; 32 | } 33 | 34 | // Canvas elements can be created from ImageData 35 | function imageDataToCanvas(imageData: ImageData) { 36 | const canvas = document.createElement("canvas"); 37 | const ctx = canvas.getContext("2d"); 38 | canvas.width = imageData.width; 39 | canvas.height = imageData.height; 40 | ctx?.putImageData(imageData, 0, 0); 41 | return canvas; 42 | } 43 | 44 | // Convert the onnx model mask output to an HTMLImageElement 45 | export function onnxMaskToImage(input: any, width: number, height: number) { 46 | return imageDataToImage(arrayToImageData(input, width, height)); 47 | } 48 | -------------------------------------------------------------------------------- /example/src/helpers/onnxModelAPI.tsx: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | 4 | // This source code is licensed under the license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | import { Tensor } from "onnxruntime-web"; 8 | import { ModelData } from "./Interfaces"; 9 | 10 | const modelData = ({ clicks, bbox, embedding, modelScale }: ModelData) => { 11 | let pointCoords; 12 | let pointLabels; 13 | let pointCoordsTensor; 14 | let pointLabelsTensor; 15 | 16 | // Check there are input click prompts 17 | if (clicks) { 18 | let n = clicks.length; 19 | 20 | const padding = bbox ? 2 : 1; 21 | pointCoords = new Float32Array(2 * (n + padding)); 22 | pointLabels = new Float32Array(n + padding); 23 | 24 | // Add clicks and scale to what SAM expects 25 | for (let i = 0; i < n; i++) { 26 | pointCoords[2 * i] = clicks[i].x * modelScale.samScale; 27 | pointCoords[2 * i + 1] = clicks[i].y * modelScale.samScale; 28 | pointLabels[i] = clicks[i].pointType; 29 | } 30 | 31 | if (bbox) { 32 | // Add in the extra point/label for the bounding box 33 | pointCoords[2 * n] = bbox.topLeft.x * modelScale.samScale; 34 | pointCoords[2 * n + 1] = bbox.topLeft.y * modelScale.samScale; 35 | pointLabels[n] = 2.0; 36 | pointCoords[2 * (n+1)] = bbox.bottomRight.x * modelScale.samScale; 37 | pointCoords[2 * (n+1) + 1] = bbox.bottomRight.y * modelScale.samScale; 38 | pointLabels[n+1] = 3.0; 39 | } else { 40 | // Add in the extra point/label when only clicks and no box 41 | // The extra point is at (0, 0) with label -1 42 | pointCoords[2 * n] = 0.0; 43 | pointCoords[2 * n + 1] = 0.0; 44 | pointLabels[n] = -1.0; 45 | } 46 | 47 | 48 | // Create the tensor 49 | pointCoordsTensor = new Tensor("float32", pointCoords, [1, n + padding, 2]); 50 | pointLabelsTensor = new Tensor("float32", pointLabels, [1, n + padding]); 51 | } 52 | const imageSizeTensor = new Tensor("float32", [ 53 | modelScale.height, 54 | modelScale.width, 55 | ]); 56 | 57 | if (pointCoordsTensor === undefined || pointLabelsTensor === undefined) 58 | return; 59 | 60 | // There is no previous mask, so default to an empty tensor 61 | const maskInput = new Tensor( 62 | "float32", 63 | new Float32Array(256 * 256), 64 | [1, 1, 256, 256] 65 | ); 66 | // There is no previous mask, so default to 0 67 | const hasMaskInput = new Tensor("float32", [0]); 68 | 69 | return { 70 | image_embeddings: embedding, 71 | point_coords: pointCoordsTensor, 72 | point_labels: pointLabelsTensor, 73 | orig_im_size: imageSizeTensor, 74 | mask_input: maskInput, 75 | has_mask_input: hasMaskInput, 76 | }; 77 | }; 78 | 79 | export { modelData }; 80 | -------------------------------------------------------------------------------- /example/src/helpers/scaleHelper.tsx: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | 4 | // This source code is licensed under the license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | 8 | // Helper function for handling image scaling needed for SAM 9 | const handleImageScale = (image: HTMLImageElement) => { 10 | // Input images to SAM must be resized so the longest side is 1024 11 | const LONG_SIDE_LENGTH = 1024; 12 | let w = image.naturalWidth; 13 | let h = image.naturalHeight; 14 | const samScale = LONG_SIDE_LENGTH / Math.max(h, w); 15 | return { height: h, width: w, samScale }; 16 | }; 17 | 18 | export { handleImageScale }; 19 | -------------------------------------------------------------------------------- /example/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import DemoApp from './DemoApp'; 4 | 5 | // Add some helpers to the window for easily retrieving params from the URL 6 | declare global { 7 | interface Window { 8 | params: (key: string, value: string) => string; 9 | exp: (key: string) => boolean; 10 | } 11 | } 12 | 13 | const searchParams = (new URL(document.location.toString())).searchParams; 14 | function params(key: string, defaultValue: string = "") { 15 | const val = searchParams.get(key); 16 | if (val == null) { 17 | return defaultValue; 18 | } 19 | return val; 20 | } 21 | 22 | const exps = params("exp", "").split(","); 23 | function exp(key: string): boolean { 24 | return exps.includes(key); 25 | } 26 | 27 | window.params = params 28 | window.exp = exp 29 | 30 | // Start the react app 31 | const root = ReactDOM.createRoot( 32 | document.getElementById('root') as HTMLElement 33 | ); 34 | root.render( 35 | 36 | ); 37 | -------------------------------------------------------------------------------- /example/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "dist", 4 | "module": "esnext", 5 | "lib": ["dom", "esnext"], 6 | "moduleResolution": "node", 7 | "jsx": "react", 8 | "sourceMap": true, 9 | "declaration": true, 10 | "esModuleInterop": true, 11 | "noImplicitReturns": true, 12 | "noImplicitThis": true, 13 | "noImplicitAny": true, 14 | "strictNullChecks": true, 15 | "suppressImplicitAnyIndexErrors": true, 16 | "noUnusedLocals": true, 17 | "noUnusedParameters": true, 18 | "allowSyntheticDefaultImports": true 19 | }, 20 | "include": ["src"], 21 | "exclude": ["node_modules", "build"] 22 | } 23 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "react-segment-anything", 3 | "version": "1.0.2", 4 | "description": "React based web interface for the Segment Anything Model (SAM)", 5 | "main": "dist/index.min.js", 6 | "module": "dist/index.es.js", 7 | "types": "dist/index.d.ts", 8 | "scripts": { 9 | "test": "echo \"Error: no test specified\" && exit 1", 10 | "build": "tsc && rollup -c rollup.config.js --bundleConfigAsCjs" 11 | }, 12 | "files": [ 13 | "dist" 14 | ], 15 | "repository": "https://github.com/mmurray/react-segment-anything", 16 | "author": "Michael Murray", 17 | "license": "MIT", 18 | "devDependencies": { 19 | "@mui/icons-material": "^5.11.16", 20 | "@mui/material": "^5.13.4", 21 | "@rollup/plugin-commonjs": "^25.0.0", 22 | "@rollup/plugin-json": "^6.0.0", 23 | "@rollup/plugin-node-resolve": "^15.1.0", 24 | "@rollup/plugin-typescript": "^11.1.1", 25 | "@types/node": "^20.2.5", 26 | "@types/react": "^18.2.8", 27 | "@types/react-dom": "^18.2.0", 28 | "konva": "^9.2.0", 29 | "onnxruntime-web": "1.14.0", 30 | "postcss": "^8.4.24", 31 | "react": "^18.2.0", 32 | "react-dom": "^18.2.0", 33 | "react-konva": "^18.2.9", 34 | "rollup": "3.21.0", 35 | "rollup-plugin-dts": "^5.3.0", 36 | "rollup-plugin-peer-deps-external": "^2.2.4", 37 | "rollup-plugin-terser": "^7.0.2", 38 | "rollup-plugin-postcss": "^4.0.2", 39 | "rollup-plugin-sourcemaps": "^0.6.3", 40 | "rollup-plugin-typescript2": "^0.34.1", 41 | "underscore": "^1.13.6", 42 | "tslib": "^2.5.3", 43 | "typescript": "^5.1.3" 44 | }, 45 | "peerDependencies": { 46 | "react": ">=18", 47 | "react-dom": ">=18", 48 | "@mui/icons-material": ">=5", 49 | "@mui/material": ">=5" 50 | }, 51 | "browser": { 52 | "fs": false, 53 | "path": false, 54 | "os": false, 55 | "crypto": false, 56 | "worker_threads": false, 57 | "util": false, 58 | "perf_hooks": false, 59 | "canvas": false 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import resolve from "@rollup/plugin-node-resolve"; 2 | import commonjs from "@rollup/plugin-commonjs"; 3 | import typescript from "@rollup/plugin-typescript"; 4 | import dts from "rollup-plugin-dts"; 5 | import { terser } from "rollup-plugin-terser"; 6 | import peerDepsExternal from "rollup-plugin-peer-deps-external"; 7 | 8 | const packageJson = require("./package.json"); 9 | 10 | export default [ 11 | { 12 | input: "src/index.tsx", 13 | output: [ 14 | { 15 | file: packageJson.main, 16 | format: "cjs", 17 | sourcemap: true, 18 | }, 19 | { 20 | file: packageJson.module, 21 | format: "esm", 22 | sourcemap: true, 23 | }, 24 | ], 25 | plugins: [ 26 | peerDepsExternal(), 27 | resolve(), 28 | commonjs(), 29 | typescript({ tsconfig: "./tsconfig.json" }), 30 | terser(), 31 | ] 32 | }, 33 | { 34 | input: "src/index.tsx", 35 | output: [{ file: "dist/types.d.ts", format: "es" }], 36 | plugins: [dts.default()], 37 | external: ["react", "react-dom"] 38 | }, 39 | ]; -------------------------------------------------------------------------------- /src/Tool.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState, useRef } from "react"; 2 | import { ToolProps, ToolMode } from "./helpers/Interfaces"; 3 | /* @ts-ignore */ 4 | import * as _ from "underscore"; 5 | 6 | import { Stage, Layer, Circle, Rect, Image } from 'react-konva'; 7 | 8 | const Tool = ({ 9 | // handleMouseMove, 10 | handleClicksChange, 11 | handleBoundingBoxChange, 12 | image, 13 | maskImage, 14 | bbox, 15 | clicks = [], 16 | mode = ToolMode.Default 17 | }: ToolProps) => { 18 | 19 | const containerRef = useRef(null); 20 | const [bboxStart, setBboxStart] = useState<{x: number, y: number} | null>(null); 21 | const [width, setWidth] = useState(0); 22 | const [height, setHeight] = useState(0); 23 | const bodyEl = document.body; 24 | const fitToPage = () => { 25 | if (!image || !containerRef) return; 26 | // const imageAspectRatio = image.width / image.height; 27 | // const screenAspectRatio = window.innerWidth / window.innerHeight; 28 | const containerRatio = containerRef!.current!.offsetWidth / image.width; 29 | setWidth(containerRef!.current!.offsetWidth); 30 | setHeight(image.height * containerRatio); 31 | }; 32 | const resizeObserver = new ResizeObserver((entries) => { 33 | for (const entry of entries) { 34 | if (entry.target === bodyEl) { 35 | fitToPage(); 36 | } 37 | } 38 | }); 39 | useEffect(() => { 40 | fitToPage(); 41 | resizeObserver.observe(bodyEl); 42 | return () => { 43 | resizeObserver.unobserve(bodyEl); 44 | }; 45 | }, [image]); 46 | 47 | const targetSelectEnabled = mode !== ToolMode.Default; 48 | 49 | const onMouseDown = (e: any) => { 50 | if (mode === ToolMode.Default) return; 51 | if (!containerRef) return; 52 | 53 | const pos = e.target.getStage().getPointerPosition(); 54 | const containerRatio = containerRef!.current!.offsetWidth / image.width; 55 | 56 | const x = pos.x / containerRatio; 57 | const y = pos.y / containerRatio; 58 | 59 | if (mode == ToolMode.PositivePoints) { 60 | if (!handleClicksChange) return; 61 | handleClicksChange([...clicks, {x, y, pointType: 1}]); 62 | } 63 | 64 | if (mode == ToolMode.NegativePoints) { 65 | if (!handleClicksChange) return; 66 | handleClicksChange([...clicks, {x, y, pointType: 0}]); 67 | } 68 | 69 | if (mode == ToolMode.BoundingBox) { 70 | setBboxStart({x, y}); 71 | } 72 | }; 73 | 74 | const onMouseUp = () => { 75 | if (mode === ToolMode.Default) return; 76 | if (!containerRef) return; 77 | 78 | 79 | if (mode == ToolMode.BoundingBox) { 80 | if (!handleBoundingBoxChange) return; 81 | if (!bboxStart) return; 82 | // handleBoundingBoxChange({x, y}); 83 | setBboxStart(null); 84 | } 85 | } 86 | 87 | const onMouseMove = (e: any) => { 88 | if (mode !== ToolMode.BoundingBox) return; 89 | if (!containerRef) return; 90 | if (!handleBoundingBoxChange) return; 91 | if (!bboxStart) return; 92 | 93 | const pos = e.target.getStage().getPointerPosition(); 94 | const containerRatio = containerRef!.current!.offsetWidth / image.width; 95 | 96 | const x = pos.x / containerRatio; 97 | const y = pos.y / containerRatio; 98 | 99 | const top = Math.min(bboxStart.y, y); 100 | const bottom = Math.max(bboxStart.y, y); 101 | const left = Math.min(bboxStart.x, x); 102 | const right = Math.max(bboxStart.x, x); 103 | 104 | handleBoundingBoxChange({topLeft: {x: left, y: top}, bottomRight: {x: right, y: bottom}}); 105 | }; 106 | 107 | 108 | const containerRatio = containerRef && containerRef.current ? containerRef!.current!.offsetWidth / image.width : 1; 109 | 110 | // Render the image and the predicted mask image on top 111 | return ( 112 |
113 | 114 | {width > 0 && height > 0 ? ( 115 | onMouseDown(e)} 117 | onTouchStart={(e) => onMouseDown(e)} 118 | onMouseMove={(e) => onMouseMove(e)} 119 | onMouseUp={() => onMouseUp()} 120 | onTouchEnd={() => onMouseUp()} 121 | > 122 | 123 | {maskImage ? : null} 124 | {bbox ? ( 125 | 132 | ) : null} 133 | {clicks.map((click, index) => ( 134 | 141 | ))} 142 | 143 | ) : null} 144 | 145 | 146 | {/* {clicks.map((click, index) =>
  • {click.x},{click.y},{click.pointType}
    • )} */} 147 | 148 |
    149 | ); 150 | }; 151 | 152 | export default Tool; -------------------------------------------------------------------------------- /src/helpers/Interfaces.tsx: -------------------------------------------------------------------------------- 1 | import { Tensor } from "onnxruntime-web"; 2 | 3 | export interface ModelScale { 4 | samScale: number; 5 | height: number; 6 | width: number; 7 | } 8 | 9 | export interface Click { 10 | x: number; 11 | y: number; 12 | pointType: number; 13 | } 14 | 15 | export interface BoundingBox { 16 | topLeft: {x: number, y: number}; 17 | bottomRight: {x: number, y: number}; 18 | } 19 | 20 | export interface ModelData { 21 | clicks?: Array; 22 | bbox?: BoundingBox; 23 | embedding: Tensor; 24 | modelScale: ModelScale; 25 | } 26 | 27 | export interface SegmentAnythingProps { 28 | handleMaskSaved: (mask: HTMLImageElement, image: HTMLImageElement) => void; 29 | image: HTMLImageElement; 30 | embedding: Tensor; 31 | modelUrl: string; 32 | initialClicks?: Array; 33 | initialBBox?: BoundingBox; 34 | } 35 | 36 | export enum ToolMode { 37 | Default = "default", 38 | BoundingBox = "boundingBox", 39 | PositivePoints = "positivePoints", 40 | NegativePoints = "negativePoints", 41 | } 42 | 43 | export interface ToolProps { 44 | image: HTMLImageElement; 45 | maskImage?: HTMLImageElement; 46 | mode?: ToolMode; 47 | clicks?: Array; 48 | bbox?: BoundingBox; 49 | handleClicksChange?: (e: any) => void; 50 | handleBoundingBoxChange?: (e: any) => void; 51 | } -------------------------------------------------------------------------------- /src/helpers/maskUtils.tsx: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | 4 | // This source code is licensed under the license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | // Convert the onnx model mask prediction to ImageData 8 | function arrayToImageData(input: any, width: number, height: number) { 9 | const [r, g, b, a] = [0, 114, 189, 255]; // the masks's blue color 10 | const arr = new Uint8ClampedArray(4 * width * height).fill(0); 11 | for (let i = 0; i < input.length; i++) { 12 | 13 | // Threshold the onnx model mask prediction at 0.0 14 | // This is equivalent to thresholding the mask using predictor.model.mask_threshold 15 | // in python 16 | if (input[i] > 0.0) { 17 | arr[4 * i + 0] = r; 18 | arr[4 * i + 1] = g; 19 | arr[4 * i + 2] = b; 20 | arr[4 * i + 3] = a; 21 | } 22 | } 23 | return new ImageData(arr, height, width); 24 | } 25 | 26 | // Use a Canvas element to produce an image from ImageData 27 | function imageDataToImage(imageData: ImageData) { 28 | const canvas = imageDataToCanvas(imageData); 29 | const image = new Image(); 30 | image.src = canvas.toDataURL(); 31 | return image; 32 | } 33 | 34 | // Canvas elements can be created from ImageData 35 | function imageDataToCanvas(imageData: ImageData) { 36 | const canvas = document.createElement("canvas"); 37 | const ctx = canvas.getContext("2d"); 38 | canvas.width = imageData.width; 39 | canvas.height = imageData.height; 40 | ctx?.putImageData(imageData, 0, 0); 41 | return canvas; 42 | } 43 | 44 | // Convert the onnx model mask output to an HTMLImageElement 45 | export function onnxMaskToImage(input: any, width: number, height: number) { 46 | return imageDataToImage(arrayToImageData(input, width, height)); 47 | } 48 | -------------------------------------------------------------------------------- /src/helpers/onnxModelAPI.tsx: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | 4 | // This source code is licensed under the license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | import { Tensor } from "onnxruntime-web"; 8 | import { ModelData } from "./Interfaces"; 9 | 10 | const modelData = ({ clicks, bbox, embedding, modelScale }: ModelData) => { 11 | let pointCoords; 12 | let pointLabels; 13 | let pointCoordsTensor; 14 | let pointLabelsTensor; 15 | 16 | // Check there are input click prompts 17 | if (clicks) { 18 | let n = clicks.length; 19 | 20 | const padding = bbox ? 2 : 1; 21 | pointCoords = new Float32Array(2 * (n + padding)); 22 | pointLabels = new Float32Array(n + padding); 23 | 24 | // Add clicks and scale to what SAM expects 25 | for (let i = 0; i < n; i++) { 26 | pointCoords[2 * i] = clicks[i].x * modelScale.samScale; 27 | pointCoords[2 * i + 1] = clicks[i].y * modelScale.samScale; 28 | pointLabels[i] = clicks[i].pointType; 29 | } 30 | 31 | if (bbox) { 32 | // Add in the extra point/label for the bounding box 33 | pointCoords[2 * n] = bbox.topLeft.x * modelScale.samScale; 34 | pointCoords[2 * n + 1] = bbox.topLeft.y * modelScale.samScale; 35 | pointLabels[n] = 2.0; 36 | pointCoords[2 * (n+1)] = bbox.bottomRight.x * modelScale.samScale; 37 | pointCoords[2 * (n+1) + 1] = bbox.bottomRight.y * modelScale.samScale; 38 | pointLabels[n+1] = 3.0; 39 | } else { 40 | // Add in the extra point/label when only clicks and no box 41 | // The extra point is at (0, 0) with label -1 42 | pointCoords[2 * n] = 0.0; 43 | pointCoords[2 * n + 1] = 0.0; 44 | pointLabels[n] = -1.0; 45 | } 46 | 47 | 48 | // Create the tensor 49 | pointCoordsTensor = new Tensor("float32", pointCoords, [1, n + padding, 2]); 50 | pointLabelsTensor = new Tensor("float32", pointLabels, [1, n + padding]); 51 | } 52 | const imageSizeTensor = new Tensor("float32", [ 53 | modelScale.height, 54 | modelScale.width, 55 | ]); 56 | 57 | if (pointCoordsTensor === undefined || pointLabelsTensor === undefined) 58 | return; 59 | 60 | // There is no previous mask, so default to an empty tensor 61 | const maskInput = new Tensor( 62 | "float32", 63 | new Float32Array(256 * 256), 64 | [1, 1, 256, 256] 65 | ); 66 | // There is no previous mask, so default to 0 67 | const hasMaskInput = new Tensor("float32", [0]); 68 | 69 | return { 70 | image_embeddings: embedding, 71 | point_coords: pointCoordsTensor, 72 | point_labels: pointLabelsTensor, 73 | orig_im_size: imageSizeTensor, 74 | mask_input: maskInput, 75 | has_mask_input: hasMaskInput, 76 | }; 77 | }; 78 | 79 | export { modelData }; 80 | -------------------------------------------------------------------------------- /src/helpers/scaleHelper.tsx: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // All rights reserved. 3 | 4 | // This source code is licensed under the license found in the 5 | // LICENSE file in the root directory of this source tree. 6 | 7 | 8 | // Helper function for handling image scaling needed for SAM 9 | const handleImageScale = (image: HTMLImageElement) => { 10 | // Input images to SAM must be resized so the longest side is 1024 11 | const LONG_SIDE_LENGTH = 1024; 12 | let w = image.naturalWidth; 13 | let h = image.naturalHeight; 14 | const samScale = LONG_SIDE_LENGTH / Math.max(h, w); 15 | return { height: h, width: w, samScale }; 16 | }; 17 | 18 | export { handleImageScale }; 19 | -------------------------------------------------------------------------------- /src/index.tsx: -------------------------------------------------------------------------------- 1 | // import * as React from 'react' 2 | // import styles from './styles.module.css' 3 | 4 | // export * as SegmentAnything from './SegmentAnything'; 5 | 6 | // import React from 'react' 7 | 8 | import { 9 | Button, 10 | Stack, 11 | Card, 12 | CardContent, 13 | Box, 14 | Typography, 15 | Grid 16 | } from "@mui/material"; 17 | 18 | import BoundingBoxIcon from '@mui/icons-material/HighlightAlt'; 19 | import PointIcon from '@mui/icons-material/AdsClick'; 20 | import CheckIcon from '@mui/icons-material/Check'; 21 | 22 | import { modelData } from './helpers/onnxModelAPI'; 23 | import { onnxMaskToImage } from "./helpers/maskUtils"; 24 | import { SegmentAnythingProps, ToolMode, Click, BoundingBox } from './helpers/Interfaces'; 25 | import Tool from './Tool'; 26 | 27 | import React, { useState, useEffect } from 'react'; 28 | import { InferenceSession } from "onnxruntime-web"; 29 | /* @ts-ignore */ 30 | import * as _ from "underscore"; 31 | 32 | export const SegmentAnything = ({image, embedding, modelUrl, handleMaskSaved, initialClicks, initialBBox}: SegmentAnythingProps) => { 33 | // ONNX model 34 | const [model, setModel] = React.useState(null); 35 | 36 | // Tool mode 37 | const [mode, setMode] = React.useState(ToolMode.Default); // Tool mode 38 | 39 | // Model parameters (clicks and bounding box) 40 | const [clicks, setClicks] = React.useState(initialClicks ? initialClicks : []); 41 | const [bbox, setBbox] = React.useState(initialBBox); 42 | 43 | // Mask image 44 | const [maskImage, setMaskImage] = React.useState(undefined); 45 | 46 | // Handler for the clear all clicks button 47 | const clearAllClicks = () => { 48 | setClicks([]); 49 | setMode(ToolMode.Default); 50 | } 51 | 52 | // Handler for the clear all bbox button 53 | const clearBBox = () => { 54 | setBbox(undefined); 55 | setMode(ToolMode.Default); 56 | } 57 | 58 | // Input images to SAM must be resized so the longest side is 1024 59 | const LONG_SIDE_LENGTH = 1024; 60 | let w = image.naturalWidth; 61 | let h = image.naturalHeight; 62 | const samScale = LONG_SIDE_LENGTH / Math.max(h, w); 63 | const modelScale = {width: w, height: h, samScale: samScale} 64 | 65 | // Initialize the ONNX model. load the image, and load the SAM 66 | // pre-computed image embedding 67 | useEffect(() => { 68 | // Initialize the ONNX model 69 | const initModel = async () => { 70 | try { 71 | const model = await InferenceSession.create(modelUrl); 72 | setModel(model); 73 | } catch (e) { 74 | console.log(e); 75 | } 76 | }; 77 | initModel(); 78 | }, []); 79 | 80 | // Run the ONNX model and generate a mask image 81 | const runONNX = async () => { 82 | try { 83 | if (model === null || clicks === null) return; 84 | 85 | // Preapre the model input in the correct format for SAM. 86 | // The modelData function is from onnxModelAPI.tsx. 87 | const feeds = modelData({ 88 | clicks, 89 | bbox, 90 | embedding, 91 | modelScale, 92 | }); 93 | if (feeds === undefined) return; 94 | // Run the SAM ONNX model with the feeds returned from modelData() 95 | const results = await model.run(feeds); 96 | const output = results[model.outputNames[0]]; 97 | // The predicted mask returned from the ONNX model is an array which is 98 | // rendered as an HTML image using onnxMaskToImage() from maskUtils.tsx. 99 | setMaskImage(onnxMaskToImage(output.data, output.dims[2], output.dims[3])); 100 | } catch (e) { 101 | console.log(e); 102 | } 103 | }; 104 | 105 | const throttledRunONNX = _.throttle(runONNX, 15); 106 | 107 | // Run the ONNX model every time clicks or bbox have changed 108 | useEffect(() => { 109 | runONNX(); 110 | }, [clicks, model]); 111 | 112 | useEffect(() => { 113 | throttledRunONNX(); 114 | }, [bbox]) 115 | 116 | return ( 117 |
    118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | Bounding Box 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | Points 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | setClicks(newClicks)} 166 | handleBoundingBoxChange={(newBbox: BoundingBox | undefined) => setBbox(newBbox)} 167 | /> 168 | 169 | 170 | 171 |
    172 | ); 173 | } 174 | 175 | // export const SegmentAnything; 176 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "ESNext", 5 | "lib": [ 6 | "ESNext", 7 | "DOM" 8 | ], 9 | "declaration": true, 10 | "declarationDir": "dist", 11 | "declarationMap": true, 12 | "downlevelIteration": false, 13 | "outDir": "dist", 14 | "sourceMap": true, 15 | "strict": true, 16 | "esModuleInterop": true, 17 | "skipLibCheck": true, 18 | "forceConsistentCasingInFileNames": true, 19 | "noFallthroughCasesInSwitch": true, 20 | "allowJs": false, 21 | "moduleResolution": "node", 22 | "resolveJsonModule": true, 23 | "jsx": "react", 24 | "baseUrl": "./src", 25 | "paths": { 26 | "tslib": [ 27 | "node_modules/tslib/tslib.d.ts" 28 | ] 29 | }, 30 | "plugins": [ 31 | { 32 | "name": "typescript-plugin-css-modules" 33 | } 34 | ], 35 | }, 36 | "include": [ 37 | "src/**/*" 38 | ], 39 | "exclude": [ 40 | "node_modules", 41 | "dist", 42 | "src/__tests__", 43 | "**/*.test.tsx", 44 | "**/*.spec.tsx", 45 | ] 46 | } -------------------------------------------------------------------------------- /types/SegmentAnything.d.ts: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { SegmentAnythingProps } from './helpers/Interfaces'; 3 | declare const SegmentAnything: ({ image, embedding, modelUrl, handleMaskSaved }: SegmentAnythingProps) => React.JSX.Element; 4 | export default SegmentAnything; 5 | -------------------------------------------------------------------------------- /types/Tool.d.ts: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { ToolProps } from "./helpers/Interfaces"; 3 | declare const Tool: ({ handleClicksChange, handleBoundingBoxChange, image, maskImage, bbox, clicks, mode }: ToolProps) => React.JSX.Element; 4 | export default Tool; 5 | -------------------------------------------------------------------------------- /types/helpers/Interfaces.d.ts: -------------------------------------------------------------------------------- 1 | import { Tensor } from "onnxruntime-web"; 2 | export interface ModelScale { 3 | samScale: number; 4 | height: number; 5 | width: number; 6 | } 7 | export interface Click { 8 | x: number; 9 | y: number; 10 | pointType: number; 11 | } 12 | export interface BoundingBox { 13 | topLeft: { 14 | x: number; 15 | y: number; 16 | }; 17 | bottomRight: { 18 | x: number; 19 | y: number; 20 | }; 21 | } 22 | export interface ModelData { 23 | clicks?: Array; 24 | bbox?: BoundingBox; 25 | embedding: Tensor; 26 | modelScale: ModelScale; 27 | } 28 | export interface SegmentAnythingProps { 29 | handleMaskSaved: (mask: HTMLImageElement, image: HTMLImageElement) => void; 30 | image: HTMLImageElement; 31 | embedding: Tensor; 32 | modelUrl: string; 33 | } 34 | export declare enum ToolMode { 35 | Default = "default", 36 | BoundingBox = "boundingBox", 37 | PositivePoints = "positivePoints", 38 | NegativePoints = "negativePoints" 39 | } 40 | export interface ToolProps { 41 | image: HTMLImageElement; 42 | maskImage?: HTMLImageElement; 43 | mode?: ToolMode; 44 | clicks?: Array; 45 | bbox?: BoundingBox; 46 | handleClicksChange?: (e: any) => void; 47 | handleBoundingBoxChange?: (e: any) => void; 48 | } 49 | -------------------------------------------------------------------------------- /types/helpers/maskUtils.d.ts: -------------------------------------------------------------------------------- 1 | export declare function onnxMaskToImage(input: any, width: number, height: number): HTMLImageElement; 2 | -------------------------------------------------------------------------------- /types/helpers/onnxModelAPI.d.ts: -------------------------------------------------------------------------------- 1 | import { Tensor } from "onnxruntime-web"; 2 | import { ModelData } from "./Interfaces"; 3 | declare const modelData: ({ clicks, bbox, embedding, modelScale }: ModelData) => { 4 | image_embeddings: Tensor; 5 | point_coords: import("onnxruntime-web").TypedTensor<"float32">; 6 | point_labels: import("onnxruntime-web").TypedTensor<"float32">; 7 | orig_im_size: import("onnxruntime-web").TypedTensor<"float32">; 8 | mask_input: import("onnxruntime-web").TypedTensor<"float32">; 9 | has_mask_input: import("onnxruntime-web").TypedTensor<"float32">; 10 | } | undefined; 11 | export { modelData }; 12 | -------------------------------------------------------------------------------- /types/helpers/scaleHelper.d.ts: -------------------------------------------------------------------------------- 1 | declare const handleImageScale: (image: HTMLImageElement) => { 2 | height: number; 3 | width: number; 4 | samScale: number; 5 | }; 6 | export { handleImageScale }; 7 | -------------------------------------------------------------------------------- /types/index.d.ts: -------------------------------------------------------------------------------- 1 | import { SegmentAnythingProps } from './helpers/Interfaces'; 2 | import React from 'react'; 3 | export declare const SegmentAnything: ({ image, embedding, modelUrl }: SegmentAnythingProps) => React.JSX.Element; 4 | --------------------------------------------------------------------------------