Vite + React + TS

├── frontend ├── src │ ├── vite-env.d.ts │ ├── holo │ │ ├── objects │ │ │ ├── socketstatus.ts │ │ │ ├── coords.ts │ │ │ ├── hand.ts │ │ │ └── InteractionState.ts │ │ ├── types │ │ │ └── streamstatus.ts │ │ ├── provider │ │ │ ├── WebSocketContext.tsx │ │ │ └── VideoStreamContext.tsx │ │ ├── hooks │ │ │ └── useSkeleton.ts │ │ └── components │ │ │ └── HolohandsOverlay.tsx │ ├── App.tsx │ ├── main.tsx │ ├── App.css │ ├── utils │ │ ├── local.ts │ │ ├── geometry.ts │ │ ├── scene.ts │ │ └── io.ts │ ├── store │ │ ├── agentTimeline.ts │ │ └── editor.ts │ ├── index.css │ ├── hooks │ │ └── useShortcuts.ts │ ├── agent │ │ ├── types.ts │ │ ├── agent.ts │ │ └── tools.ts │ ├── components │ │ ├── Layout.tsx │ │ ├── SnapPanel.tsx │ │ ├── Toolbar.tsx │ │ ├── BooleanPanel.tsx │ │ ├── VideoStream.tsx │ │ ├── AgentTimeline.tsx │ │ ├── Topbar.tsx │ │ └── ShapeIcons.tsx │ ├── provider │ │ └── ViewportContext.tsx │ ├── types.ts │ └── assets │ │ └── react.svg ├── tsconfig.json ├── vite.config.ts ├── index.html ├── eslint.config.js ├── tsconfig.node.json ├── tsconfig.app.json ├── package.json ├── public │ └── vite.svg ├── sample-server-response.json └── server │ └── index.mjs ├── .gitignore ├── package.json ├── requirements.txt ├── backend ├── processor.py ├── detector.py ├── hand_processor.py └── webserver.py └── README.md /frontend/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .env 3 | .venv 4 | .DS_Store 5 | old/ 6 | .env.local -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@fal-ai/client": "^1.6.2" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /frontend/src/holo/objects/socketstatus.ts: -------------------------------------------------------------------------------- 1 | export type SocketStatus = 2 | | "Connecting..." 3 | | "Connected" 4 | | "Disconnected" 5 | | "Error"; 6 | -------------------------------------------------------------------------------- /frontend/src/holo/types/streamstatus.ts: -------------------------------------------------------------------------------- 1 | export type StreamStatus = 2 | | "idle" 3 | | "loading" 4 | | "error" 5 | | "streaming" 6 | | "stopped"; 7 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { "path": "./tsconfig.app.json" }, 5 | { "path": "./tsconfig.node.json" } 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /frontend/src/App.tsx: -------------------------------------------------------------------------------- 1 | import Layout from './components/Layout' 2 | 3 | function App() { 4 | return ( 5 | 6 | ) 7 | } 8 | 9 | export default App 10 | -------------------------------------------------------------------------------- /frontend/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import react from '@vitejs/plugin-react' 3 | 4 | // https://vite.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | }) 8 | -------------------------------------------------------------------------------- /frontend/src/holo/objects/coords.ts: -------------------------------------------------------------------------------- 1 | interface Coords { 2 | x: number; 3 | y: number; 4 | } 5 | 6 | const DEFAULT_COORDS: Coords = { x: 0, y: 0 }; 7 | 8 | export type { Coords }; 9 | export { DEFAULT_COORDS }; 10 | -------------------------------------------------------------------------------- /frontend/src/holo/objects/hand.ts: -------------------------------------------------------------------------------- 1 | export interface Hand { 2 | handedness: "Left" | "Right"; 3 | landmarks: number[][]; 4 | connections: number[][]; 5 | detected_symbols?: [string, number][]; 6 | } 7 | 8 | export const HAND_COLORS: Record<"Left" | "Right", string> = { 9 | Left: "#FF0000", 10 | Right: "#00FF00", 11 | }; 12 | -------------------------------------------------------------------------------- /frontend/src/main.tsx: -------------------------------------------------------------------------------- 1 | import { StrictMode } from "react"; 2 | import { createRoot } from "react-dom/client"; 3 | import App from "./App.tsx"; 4 | import { ViewportProvider } from "./provider/ViewportContext.tsx"; 5 | 6 | createRoot(document.getElementById("root")!).render( 7 | 8 | 9 | 10 | 11 | 12 | ); 13 | -------------------------------------------------------------------------------- /frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Vite + React + TS 8 | 9 | 10 |

11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /frontend/eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js' 2 | import globals from 'globals' 3 | import reactHooks from 'eslint-plugin-react-hooks' 4 | import reactRefresh from 'eslint-plugin-react-refresh' 5 | import tseslint from 'typescript-eslint' 6 | import { globalIgnores } from 'eslint/config' 7 | 8 | export default tseslint.config([ 9 | globalIgnores(['dist']), 10 | { 11 | files: ['**/*.{ts,tsx}'], 12 | extends: [ 13 | js.configs.recommended, 14 | tseslint.configs.recommended, 15 | reactHooks.configs['recommended-latest'], 16 | reactRefresh.configs.vite, 17 | ], 18 | languageOptions: { 19 | ecmaVersion: 2020, 20 | globals: globals.browser, 21 | }, 22 | }, 23 | ]) 24 | -------------------------------------------------------------------------------- /frontend/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", 4 | "target": "ES2023", 5 | "lib": ["ES2023"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "verbatimModuleSyntax": true, 13 | "moduleDetection": "force", 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "erasableSyntaxOnly": true, 21 | "noFallthroughCasesInSwitch": true, 22 | "noUncheckedSideEffectImports": true 23 | }, 24 | "include": ["vite.config.ts"] 25 | } 26 | -------------------------------------------------------------------------------- /frontend/src/App.css: -------------------------------------------------------------------------------- 1 | #root { 2 | max-width: 1280px; 3 | margin: 0 auto; 4 | padding: 2rem; 5 | text-align: center; 6 | } 7 | 8 | .logo { 9 | height: 6em; 10 | padding: 1.5em; 11 | will-change: filter; 12 | transition: filter 300ms; 13 | } 14 | .logo:hover { 15 | filter: drop-shadow(0 0 2em #646cffaa); 16 | } 17 | .logo.react:hover { 18 | filter: drop-shadow(0 0 2em #61dafbaa); 19 | } 20 | 21 | @keyframes logo-spin { 22 | from { 23 | transform: rotate(0deg); 24 | } 25 | to { 26 | transform: rotate(360deg); 27 | } 28 | } 29 | 30 | @media (prefers-reduced-motion: no-preference) { 31 | a:nth-of-type(2) .logo { 32 | animation: logo-spin infinite 20s linear; 33 | } 34 | } 35 | 36 | .card { 37 | padding: 2em; 38 | } 39 | 40 | .read-the-docs { 41 | color: #888; 42 | } 43 | -------------------------------------------------------------------------------- /frontend/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", 4 | "target": "ES2022", 5 | "useDefineForClassFields": true, 6 | "lib": ["ES2022", "DOM", "DOM.Iterable"], 7 | "module": "ESNext", 8 | "skipLibCheck": true, 9 | 10 | /* Bundler mode */ 11 | "moduleResolution": "bundler", 12 | "allowImportingTsExtensions": true, 13 | "verbatimModuleSyntax": true, 14 | "moduleDetection": "force", 15 | "noEmit": true, 16 | "jsx": "react-jsx", 17 | 18 | /* Linting */ 19 | "strict": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "erasableSyntaxOnly": true, 23 | "noFallthroughCasesInSwitch": true, 24 | "noUncheckedSideEffectImports": true 25 | }, 26 | "include": ["src"] 27 | } 28 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==2.3.1 2 | attrs==25.3.0 3 | blinker==1.9.0 4 | cffi==2.0.0 5 | click==8.2.1 6 | colorama==0.4.6 7 | contourpy==1.3.3 8 | cycler==0.12.1 9 | dnspython==2.8.0 10 | eventlet==0.40.3 11 | Flask==3.1.2 12 | flask-cors==6.0.1 13 | flatbuffers==25.2.10 14 | fonttools==4.59.2 15 | greenlet==3.2.4 16 | itsdangerous==2.2.0 17 | jax==0.7.1 18 | jaxlib==0.7.1 19 | Jinja2==3.1.6 20 | kiwisolver==1.4.9 21 | MarkupSafe==3.0.2 22 | matplotlib==3.10.6 23 | mediapipe==0.10.21 24 | ml_dtypes==0.5.3 25 | numpy==2.3.3 26 | opencv-contrib-python==4.11.0.86 27 | opencv-python==4.12.0.88 28 | opt_einsum==3.4.0 29 | orjson==3.11.3 30 | packaging==25.0 31 | pillow==11.3.0 32 | protobuf==4.25.8 33 | pycparser==2.23 34 | pyparsing==3.2.4 35 | python-dateutil==2.9.0.post0 36 | scipy==1.16.2 37 | sentencepiece==0.2.1 38 | six==1.17.0 39 | sounddevice==0.5.2 40 | Werkzeug==3.1.3 41 | -------------------------------------------------------------------------------- /frontend/src/holo/objects/InteractionState.ts: -------------------------------------------------------------------------------- 1 | import { Coords } from "./coords"; 2 | 3 | interface InteractionState { 4 | Left: InteractionStateHand | null; 5 | Right: InteractionStateHand | null; 6 | angleBetween: number; 7 | } 8 | 9 | interface InteractionStateHand { 10 | isHolding: boolean; 11 | isPinching: boolean; 12 | cursor: { 13 | coords: Coords; 14 | angle: number; 15 | } | null; 16 | depth: number; 17 | } 18 | 19 | const DEFAULT_INTERACTION_STATE_HAND: InteractionStateHand = { 20 | isHolding: false, 21 | isPinching: false, 22 | cursor: null, 23 | depth: 0, 24 | }; 25 | 26 | const DEFAULT_INTERACTION_STATE: InteractionState = { 27 | Left: null, 28 | Right: null, 29 | angleBetween: 0, 30 | }; 31 | 32 | export type { InteractionState, InteractionStateHand }; 33 | export { DEFAULT_INTERACTION_STATE, DEFAULT_INTERACTION_STATE_HAND }; 34 | -------------------------------------------------------------------------------- /frontend/src/utils/local.ts: -------------------------------------------------------------------------------- 1 | import type { EditorState, SceneObject } from '../types' 2 | import { jsonToScene, sceneToJSON, type SavedScene } from './scene' 3 | 4 | const KEY = 'ai-blender-scene' 5 | 6 | export function saveScene(state: EditorState) { 7 | const json = sceneToJSON(state) 8 | localStorage.setItem(KEY, json) 9 | } 10 | 11 | export function loadScene(): SavedScene | null { 12 | const json = localStorage.getItem(KEY) 13 | if (!json) return null 14 | // First try the new format 15 | const decoded = jsonToScene(json) 16 | if (decoded) return decoded 17 | // Backward compatibility: legacy saves were an array of SceneObject 18 | try { 19 | const legacy = JSON.parse(json) as SceneObject[] 20 | if (Array.isArray(legacy)) { 21 | return { 22 | version: 1, 23 | editor: { 24 | objects: legacy, 25 | selectedId: null, 26 | mode: 'translate', 27 | editorMode: 'object', 28 | snap: { enableSnapping: false, translateSnap: 0.5, rotateSnap: Math.PI / 12, scaleSnap: 0.1 }, 29 | }, 30 | } 31 | } 32 | } catch {} 33 | return null 34 | } 35 | -------------------------------------------------------------------------------- /backend/processor.py: -------------------------------------------------------------------------------- 1 | import eventlet 2 | eventlet.monkey_patch() 3 | 4 | import cv2 5 | from webserver import socketio, app 6 | from detector import HandDetector 7 | 8 | class VisionProcessor: 9 | def __init__(self): 10 | self.cap = cv2.VideoCapture(0) 11 | self.detector = HandDetector() 12 | self.last_hands = [] 13 | 14 | def process_video(self): 15 | with app.app_context(): 16 | while True: 17 | success, frame = self.cap.read() 18 | if not success: 19 | continue 20 | 21 | frame = cv2.flip(frame, 1) 22 | hands_data = self.detector.detect(frame) 23 | 24 | if hands_data != self.last_hands: 25 | try: 26 | socketio.emit('hand_data', hands_data) 27 | self.last_hands = hands_data 28 | except Exception as e: 29 | print("Emit error:", str(e)) 30 | 31 | eventlet.sleep(0.001) 32 | 33 | if __name__ == '__main__': 34 | processor = VisionProcessor() 35 | processor.process_video() -------------------------------------------------------------------------------- /frontend/src/store/agentTimeline.ts: -------------------------------------------------------------------------------- 1 | import { create } from "zustand"; 2 | import type { StepResult, SceneSnapshot } from "../agent/types"; 3 | 4 | interface AgentUiState { 5 | dryRun: boolean; 6 | setDryRun: (v: boolean) => void; 7 | steps: StepResult[]; 8 | setSteps: (s: StepResult[]) => void; 9 | snapshot: SceneSnapshot | null; 10 | setSnapshot: (s: SceneSnapshot | null) => void; 11 | lastExportName: string | null; 12 | setLastExportName: (n: string | null) => void; 13 | showTimeline: boolean; 14 | setShowTimeline: (v: boolean) => void; 15 | toggleTimeline: () => void; 16 | } 17 | 18 | export const useAgentTimeline = create()((set) => ({ 19 | dryRun: false, 20 | setDryRun: (v) => set((s) => ({ ...s, dryRun: v })), 21 | steps: [], 22 | setSteps: (steps) => set((s) => ({ ...s, steps })), 23 | snapshot: null, 24 | setSnapshot: (snapshot) => set((s) => ({ ...s, snapshot })), 25 | lastExportName: null, 26 | setLastExportName: (n) => set((s) => ({ ...s, lastExportName: n })), 27 | showTimeline: true, 28 | setShowTimeline: (v) => set((s) => ({ ...s, showTimeline: v })), 29 | toggleTimeline: () => set((s) => ({ ...s, showTimeline: !s.showTimeline })), 30 | })); -------------------------------------------------------------------------------- /backend/detector.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | 4 | class HandDetector: 5 | def __init__(self): 6 | self.mp_hands = mp.solutions.hands 7 | self.hands = self.mp_hands.Hands( 8 | static_image_mode=False, 9 | max_num_hands=2, 10 | min_detection_confidence=0.75, 11 | min_tracking_confidence=0.75) 12 | 13 | def detect(self, frame): 14 | rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 15 | results = self.hands.process(rgb) 16 | return self._serialize_results(results) 17 | 18 | def _serialize_results(self, results): 19 | hands = [] 20 | if results.multi_hand_landmarks: 21 | for idx, landmarks in enumerate(results.multi_hand_landmarks): 22 | try: 23 | handedness = results.multi_handedness[idx].classification[0].label 24 | hand = { 25 | "handedness": handedness, 26 | "landmarks": [(float(lm.x), float(lm.y), float(lm.z)) for lm in landmarks.landmark], 27 | "connections": [[int(conn[0]), int(conn[1])] for conn in self.mp_hands.HAND_CONNECTIONS] 28 | } 29 | hands.append(hand) 30 | except (IndexError, AttributeError): 31 | continue 32 | return hands -------------------------------------------------------------------------------- /frontend/src/index.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: system-ui, Avenir, Helvetica, Arial, sans-serif; 3 | line-height: 1.5; 4 | font-weight: 400; 5 | 6 | color-scheme: light dark; 7 | color: rgba(255, 255, 255, 0.87); 8 | background-color: #242424; 9 | 10 | font-synthesis: none; 11 | text-rendering: optimizeLegibility; 12 | -webkit-font-smoothing: antialiased; 13 | -moz-osx-font-smoothing: grayscale; 14 | } 15 | 16 | a { 17 | font-weight: 500; 18 | color: #646cff; 19 | text-decoration: inherit; 20 | } 21 | a:hover { 22 | color: #535bf2; 23 | } 24 | 25 | body { 26 | margin: 0; 27 | display: flex; 28 | place-items: center; 29 | min-width: 320px; 30 | min-height: 100vh; 31 | } 32 | 33 | h1 { 34 | font-size: 3.2em; 35 | line-height: 1.1; 36 | } 37 | 38 | button { 39 | border-radius: 8px; 40 | border: 1px solid transparent; 41 | padding: 0.6em 1.2em; 42 | font-size: 1em; 43 | font-weight: 500; 44 | font-family: inherit; 45 | background-color: #1a1a1a; 46 | cursor: pointer; 47 | transition: border-color 0.25s; 48 | } 49 | button:hover { 50 | border-color: #646cff; 51 | } 52 | button:focus, 53 | button:focus-visible { 54 | outline: 4px auto -webkit-focus-ring-color; 55 | } 56 | 57 | @media (prefers-color-scheme: light) { 58 | :root { 59 | color: #213547; 60 | background-color: #ffffff; 61 | } 62 | a:hover { 63 | color: #747bff; 64 | } 65 | button { 66 | background-color: #f9f9f9; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ai-blender", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "engines": { 7 | "node": ">=18" 8 | }, 9 | "scripts": { 10 | "dev": "vite", 11 | "build": "tsc -b && vite build", 12 | "lint": "eslint .", 13 | "preview": "vite preview", 14 | "server": "node server/index.mjs" 15 | }, 16 | "dependencies": { 17 | "@react-three/drei": "^10.7.5", 18 | "@react-three/fiber": "^9.3.0", 19 | "@types/file-saver": "^2.0.7", 20 | "@types/styled-components": "^5.1.34", 21 | "dotenv": "^16.4.5", 22 | "express": "^4.19.2", 23 | "file-saver": "^2.0.5", 24 | "immer": "^10.1.3", 25 | "react": "^19.1.1", 26 | "react-dom": "^19.1.1", 27 | "styled-components": "^6.1.19", 28 | "three": "^0.180.0", 29 | "three-bvh-csg": "^0.0.17", 30 | "three-csg-ts": "^3.2.0", 31 | "three-mesh-bvh": "^0.9.1", 32 | "three-stdlib": "^2.36.0", 33 | "zustand": "^5.0.8", 34 | "@fal-ai/client": "^1.0.0" 35 | }, 36 | "devDependencies": { 37 | "@eslint/js": "^9.33.0", 38 | "@types/react": "^19.1.10", 39 | "@types/react-dom": "^19.1.7", 40 | "@vitejs/plugin-react": "^5.0.0", 41 | "eslint": "^9.33.0", 42 | "eslint-plugin-react-hooks": "^5.2.0", 43 | "eslint-plugin-react-refresh": "^0.4.20", 44 | "globals": "^16.3.0", 45 | "typescript": "~5.8.3", 46 | "typescript-eslint": "^8.39.1", 47 | "vite": "^7.1.2" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /frontend/public/vite.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/utils/geometry.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three' 2 | import type { SerializableGeometry } from '../types' 3 | 4 | export function serializeGeometry(geometry: THREE.BufferGeometry): SerializableGeometry { 5 | const pos = geometry.getAttribute('position') as THREE.BufferAttribute | null 6 | const norm = geometry.getAttribute('normal') as THREE.BufferAttribute | null 7 | const uv = geometry.getAttribute('uv') as THREE.BufferAttribute | null 8 | const index = geometry.getIndex() 9 | return { 10 | positions: pos ? Array.from(pos.array as Float32Array) : [], 11 | normals: norm ? Array.from(norm.array as Float32Array) : undefined, 12 | uvs: uv ? Array.from(uv.array as Float32Array) : undefined, 13 | indices: index ? Array.from(index.array as Uint16Array | Uint32Array) : undefined, 14 | } 15 | } 16 | 17 | export function deserializeGeometry(data: SerializableGeometry): THREE.BufferGeometry { 18 | const geometry = new THREE.BufferGeometry() 19 | if (data.positions && data.positions.length > 0) { 20 | geometry.setAttribute('position', new THREE.BufferAttribute(new Float32Array(data.positions), 3)) 21 | } 22 | if (data.normals && data.normals.length > 0) { 23 | geometry.setAttribute('normal', new THREE.BufferAttribute(new Float32Array(data.normals), 3)) 24 | } 25 | if (data.uvs && data.uvs.length > 0) { 26 | geometry.setAttribute('uv', new THREE.BufferAttribute(new Float32Array(data.uvs), 2)) 27 | } 28 | if (data.indices && data.indices.length > 0) { 29 | const needsUint32 = Math.max(...data.indices) > 65535 30 | geometry.setIndex(new THREE.BufferAttribute(needsUint32 ? new Uint32Array(data.indices) : new Uint16Array(data.indices), 1)) 31 | } 32 | geometry.computeBoundingBox() 33 | geometry.computeBoundingSphere() 34 | if (!geometry.getAttribute('normal')) geometry.computeVertexNormals() 35 | return geometry 36 | } 37 | -------------------------------------------------------------------------------- /frontend/src/hooks/useShortcuts.ts: -------------------------------------------------------------------------------- 1 | import { useEffect } from 'react' 2 | import { useEditor } from '../store/editor' 3 | 4 | export function useShortcuts() { 5 | const add = useEditor(s => s.addObject) 6 | const del = useEditor(s => s.deleteSelected) 7 | const dup = useEditor(s => s.duplicateSelected) 8 | const undo = useEditor(s => s.undo) 9 | const redo = useEditor(s => s.redo) 10 | const setMode = useEditor(s => s.setMode) 11 | 12 | useEffect(() => { 13 | function onKeyDown(e: KeyboardEvent) { 14 | if (e.target && (e.target as HTMLElement).tagName === 'INPUT') return 15 | 16 | // Command/Ctrl + L to show chat panel 17 | if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === 'l') { 18 | e.preventDefault() 19 | try { (useEditor as any).setState((s: any) => ({ ...s, showChatPanel: true })); } catch {} 20 | return 21 | } 22 | 23 | if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === 'z') { 24 | e.preventDefault() 25 | if (e.shiftKey) redo(); else undo(); 26 | return 27 | } 28 | if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === 'd') { 29 | e.preventDefault(); dup(); return 30 | } 31 | if (e.key === 'Delete' || e.key === 'Backspace') { del(); return } 32 | if (e.key.toLowerCase() === 'g') { setMode('translate'); return } 33 | if (e.key.toLowerCase() === 'r') { setMode('rotate'); return } 34 | if (e.key.toLowerCase() === 's') { setMode('scale'); return } 35 | if (e.key.toLowerCase() === '1') { add('box'); return } 36 | if (e.key.toLowerCase() === '2') { add('sphere'); return } 37 | if (e.key.toLowerCase() === '3') { add('cylinder'); return } 38 | } 39 | window.addEventListener('keydown', onKeyDown) 40 | return () => window.removeEventListener('keydown', onKeyDown) 41 | }, [add, del, dup, undo, redo, setMode]) 42 | } 43 | -------------------------------------------------------------------------------- /frontend/src/agent/types.ts: -------------------------------------------------------------------------------- 1 | export type SceneEntityType = "mesh" | "light" | "camera" | "empty"; 2 | 3 | export type SceneSnapshot = { 4 | units: "meters"; 5 | upAxis: "Y"; 6 | counts: { meshes: number; lights: number; cameras: number }; 7 | selection: string[]; 8 | entities: Array<{ 9 | id: string; 10 | name: string; 11 | type: SceneEntityType; 12 | parentId?: string; 13 | childrenIds?: string[]; 14 | transform: { 15 | position: [number, number, number]; 16 | rotation: [number, number, number]; 17 | scale: [number, number, number]; 18 | }; 19 | geom?: { 20 | kind?: "cube" | "sphere" | "plane" | "cylinder"; 21 | stats?: { vertices: number; triangles: number }; 22 | bounds?: { min: [number, number, number]; max: [number, number, number] }; 23 | }; 24 | material?: { 25 | id: string; 26 | name: string; 27 | baseColorHex?: string; 28 | metalness?: number; 29 | roughness?: number; 30 | }; 31 | }>; 32 | bounds?: { min: [number, number, number]; max: [number, number, number] }; 33 | capabilities: string[]; 34 | }; 35 | 36 | export type Diff = { 37 | addedIds: string[]; 38 | removedIds: string[]; 39 | updatedIds: string[]; 40 | }; 41 | 42 | export type ToolResult = { 43 | success: boolean; 44 | diagnostics?: string[]; 45 | snapshot?: SceneSnapshot; 46 | changes?: Diff; 47 | payload?: any; 48 | }; 49 | 50 | export type ToolInvocation = { 51 | name: 52 | | "get_scene_summary" 53 | | "find" 54 | | "get_selection" 55 | | "create_primitive" 56 | | "select" 57 | | "transform" 58 | | "duplicate" 59 | | "delete" 60 | | "create_material" 61 | | "assign_material" 62 | | "export_glb" 63 | | "image_to_3d" 64 | | "update_name"; // small extension for rename 65 | args: any; 66 | }; 67 | 68 | export type StepResult = { 69 | tool: ToolInvocation; 70 | ok: boolean; 71 | diagnostics?: string[]; 72 | diff?: Diff; 73 | }; -------------------------------------------------------------------------------- /frontend/src/components/Layout.tsx: -------------------------------------------------------------------------------- 1 | import styled, { createGlobalStyle } from "styled-components"; 2 | import Topbar from "./Topbar"; 3 | import Toolbar from "./Toolbar"; 4 | import Inspector from "./Inspector"; 5 | // Viewport removed in favor of legacy Three.js renderer wired via HolohandsOverlay 6 | import { useShortcuts } from "../hooks/useShortcuts"; 7 | import BooleanPanel from "./BooleanPanel"; 8 | import HolohandsOverlay from "../holo/components/HolohandsOverlay"; 9 | import Viewport from "./Viewport"; 10 | import ChatPanel from "./ChatPanel"; 11 | import { useEditor } from "../store/editor"; 12 | import VideoStream from "./VideoStream"; 13 | import { VideoStreamProvider } from "../holo/provider/VideoStreamContext"; 14 | import AgentTimeline from "./AgentTimeline"; 15 | import { useAgentTimeline } from "../store/agentTimeline"; 16 | 17 | const Global = createGlobalStyle` 18 | html, body, #root { 19 | height: 100%; 20 | } 21 | body { 22 | margin: 0; 23 | background: #0b0e14; 24 | color: #e6e9ef; 25 | overflow: hidden; 26 | font-family: Inter, ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, "Apple Color Emoji", "Segoe UI Emoji"; 27 | } 28 | * { box-sizing: border-box; } 29 | button { cursor: pointer; } 30 | `; 31 | 32 | const Root = styled.div` 33 | position: relative; 34 | width: 100vw; 35 | height: 100vh; 36 | `; 37 | 38 | // const ViewportWrap = styled.div` 39 | // position: absolute; 40 | // inset: 56px 0 0 0; 41 | // `; 42 | 43 | export function Layout() { 44 | useShortcuts(); 45 | const showChat = useEditor((s) => s.showChatPanel); 46 | const editorMode = useEditor((s) => s.editorMode); 47 | const steps = useAgentTimeline((s) => s.steps); 48 | const showTimeline = useAgentTimeline((s) => s.showTimeline); 49 | return ( 50 | 51 | 52 | 53 | 54 | {/* Render different viewport based on editor mode */} 55 | {editorMode === "render" ? : } 56 | 57 | 58 | 59 | {showTimeline ? : null} 60 | {showChat ? : null} 61 | 62 | 63 | 64 | ); 65 | } 66 | 67 | export default Layout; 68 | -------------------------------------------------------------------------------- /frontend/src/components/SnapPanel.tsx: -------------------------------------------------------------------------------- 1 | import styled from "styled-components"; 2 | import { useEditor } from "../store/editor"; 3 | 4 | const Panel = styled.div` 5 | position: absolute; 6 | bottom: 12px; 7 | left: 228px; 8 | display: flex; 9 | height: 50px; 10 | gap: 8px; 11 | align-items: center; 12 | background: rgba(18, 20, 26, 0.85); 13 | border: 1px solid rgba(255, 255, 255, 0.08); 14 | padding: 8px 10px; 15 | border-radius: 10px; 16 | `; 17 | 18 | const Label = styled.label` 19 | opacity: 0.8; 20 | font-size: 12px; 21 | `; 22 | 23 | const Input = styled.input` 24 | width: 64px; 25 | background: #0f1116; 26 | border: 1px solid rgba(255, 255, 255, 0.08); 27 | color: #e6e9ef; 28 | border-radius: 6px; 29 | padding: 4px 6px; 30 | `; 31 | 32 | export function SnapPanel() { 33 | const snap = useEditor((s) => s.snap); 34 | const toggle = useEditor((s) => s.toggleSnap); 35 | const setSnap = useEditor((s) => s.setSnap); 36 | return ( 37 | 38 | 39 | toggle(e.target.checked)} 43 | />{" "} 44 | Snap 45 | 46 | 47 | Move 48 | 53 | setSnap({ translateSnap: parseFloat(e.target.value) }) 54 | } 55 | /> 56 | 57 | 58 | Rotate 59 | 64 | setSnap({ rotateSnap: parseFloat(e.target.value) }) 65 | } 66 | /> 67 | 68 | 69 | Scale 70 | 75 | setSnap({ scaleSnap: parseFloat(e.target.value) }) 76 | } 77 | /> 78 | 79 | 80 | ); 81 | } 82 | 83 | export default SnapPanel; 84 | -------------------------------------------------------------------------------- /frontend/src/components/Toolbar.tsx: -------------------------------------------------------------------------------- 1 | import styled from "styled-components"; 2 | import { useEditor } from "../store/editor"; 3 | import { 4 | BoxIcon, 5 | SphereIcon, 6 | CylinderIcon, 7 | ConeIcon, 8 | TorusIcon, 9 | PlaneIcon, 10 | } from "./ShapeIcons"; 11 | 12 | const Rail = styled.div` 13 | position: absolute; 14 | top: 56px; 15 | bottom: 12px; 16 | left: 12px; 17 | width: 52px; 18 | display: flex; 19 | flex-direction: column; 20 | gap: 8px; 21 | `; 22 | 23 | const Btn = styled.button` 24 | width: 52px; 25 | height: 40px; 26 | background: rgba(18, 20, 26, 0.9); 27 | color: #e6e9ef; 28 | border: 1px solid rgba(255, 255, 255, 0.08); 29 | border-radius: 10px; 30 | display: flex; 31 | align-items: center; 32 | justify-content: center; 33 | transition: all 0.2s ease; 34 | 35 | &:hover { 36 | background: rgba(18, 20, 26, 1); 37 | border-color: rgba(255, 255, 255, 0.15); 38 | transform: translateY(-1px); 39 | } 40 | 41 | &:active { 42 | transform: translateY(0); 43 | } 44 | `; 45 | 46 | export function Toolbar() { 47 | const add = useEditor((s) => s.addObject); 48 | const editorMode = useEditor((s) => s.editorMode); 49 | 50 | return ( 51 | 52 | {/* Shape buttons - only show in object and edit modes */} 53 | {editorMode !== "render" && ( 54 | <> 55 | add("box")} title="Add Box"> 56 | 57 | 58 | add("sphere")} title="Add Sphere"> 59 | 60 | 61 | add("cylinder")} title="Add Cylinder"> 62 | 63 | 64 | add("cone")} title="Add Cone"> 65 | 66 | 67 | add("torus")} title="Add Torus"> 68 | 69 | 70 | add("plane")} title="Add Plane"> 71 | 72 | 73 | 74 | )} 75 | 76 | {/* Lighting buttons removed for render mode */} 77 | 78 | ); 79 | } 80 | 81 | export default Toolbar; 82 | -------------------------------------------------------------------------------- /frontend/src/components/BooleanPanel.tsx: -------------------------------------------------------------------------------- 1 | import { useMemo, useState } from "react"; 2 | import styled from "styled-components"; 3 | import { useEditor } from "../store/editor"; 4 | 5 | const Panel = styled.div` 6 | position: absolute; 7 | bottom: 12px; 8 | right: 308px; 9 | display: flex; 10 | gap: 8px; 11 | align-items: center; 12 | height: 50px; 13 | background: rgba(18, 20, 26, 0.85); 14 | border: 1px solid rgba(255, 255, 255, 0.08); 15 | padding: 8px 10px; 16 | border-radius: 10px; 17 | `; 18 | 19 | const Select = styled.select` 20 | background: #0f1116; 21 | border: 1px solid rgba(255, 255, 255, 0.08); 22 | color: #e6e9ef; 23 | border-radius: 6px; 24 | padding: 6px 8px; 25 | `; 26 | 27 | const Btn = styled.button` 28 | background: #12141a; 29 | color: #e6e9ef; 30 | border: 1px solid rgba(255, 255, 255, 0.08); 31 | padding: 6px 10px; 32 | border-radius: 8px; 33 | `; 34 | 35 | export function BooleanPanel() { 36 | const objects = useEditor((s) => s.objects); 37 | const booleanOp = useEditor((s) => s.booleanOp); 38 | const [aId, setA] = useState(""); 39 | const [bId, setB] = useState(""); 40 | 41 | const options = useMemo( 42 | () => objects.map((o) => ({ id: o.id, name: o.name })), 43 | [objects] 44 | ); 45 | 46 | return ( 47 | 48 | 56 | 64 | booleanOp("union", aId, bId)} 67 | > 68 | Union 69 | 70 | booleanOp("subtract", aId, bId)} 73 | > 74 | Subtract 75 | 76 | booleanOp("intersect", aId, bId)} 79 | > 80 | Intersect 81 | 82 | 83 | ); 84 | } 85 | 86 | export default BooleanPanel; 87 | -------------------------------------------------------------------------------- /frontend/src/utils/scene.ts: -------------------------------------------------------------------------------- 1 | import type { EditorState, SceneObject, SnapSettings } from '../types' 2 | 3 | interface SavedEditorState { 4 | objects: SceneObject[] 5 | selectedId: string | null 6 | mode: EditorState['mode'] 7 | editorMode: EditorState['editorMode'] 8 | snap: SnapSettings 9 | } 10 | 11 | export interface SavedScene { 12 | version: 1 13 | editor: SavedEditorState 14 | } 15 | 16 | function isObject(value: unknown): value is Record { 17 | return typeof value === 'object' && value !== null 18 | } 19 | 20 | function coerceSceneObjectArray(value: unknown): SceneObject[] { 21 | if (!Array.isArray(value)) return [] 22 | return value.filter((o) => isObject(o)) as unknown as SceneObject[] 23 | } 24 | 25 | function coerceSnapSettings(value: unknown): SnapSettings { 26 | if (!isObject(value)) { 27 | return { enableSnapping: false, translateSnap: 0.5, rotateSnap: Math.PI / 12, scaleSnap: 0.1 } 28 | } 29 | const v = value as Partial 30 | return { 31 | enableSnapping: Boolean(v.enableSnapping), 32 | translateSnap: typeof v.translateSnap === 'number' ? v.translateSnap : 0.5, 33 | rotateSnap: typeof v.rotateSnap === 'number' ? v.rotateSnap : Math.PI / 12, 34 | scaleSnap: typeof v.scaleSnap === 'number' ? v.scaleSnap : 0.1, 35 | } 36 | } 37 | 38 | export function serializeScene(state: EditorState): SavedScene { 39 | const editor: SavedEditorState = { 40 | objects: JSON.parse(JSON.stringify(state.objects)), 41 | selectedId: state.selectedId, 42 | mode: state.mode, 43 | editorMode: state.editorMode, 44 | snap: { ...state.snap }, 45 | } 46 | return { version: 1, editor } 47 | } 48 | 49 | export function deserializeScene(input: unknown): SavedScene | null { 50 | if (!isObject(input)) return null 51 | const version = (input as any).version 52 | const editor = (input as any).editor 53 | if (version !== 1 || !isObject(editor)) return null 54 | const ed = editor as Record 55 | const objects = coerceSceneObjectArray(ed.objects) 56 | const selectedId = typeof ed.selectedId === 'string' || ed.selectedId === null ? (ed.selectedId as string | null) : null 57 | const mode = (ed.mode === 'translate' || ed.mode === 'rotate' || ed.mode === 'scale') ? (ed.mode as EditorState['mode']) : 'translate' 58 | const editorMode = (ed.editorMode === 'object' || ed.editorMode === 'edit') ? (ed.editorMode as EditorState['editorMode']) : 'object' 59 | const snap = coerceSnapSettings(ed.snap) 60 | return { version: 1, editor: { objects, selectedId, mode, editorMode, snap } } 61 | } 62 | 63 | export function sceneToJSON(state: EditorState): string { 64 | return JSON.stringify(serializeScene(state)) 65 | } 66 | 67 | export function jsonToScene(json: string): SavedScene | null { 68 | try { 69 | const data = JSON.parse(json) 70 | return deserializeScene(data) 71 | } catch { 72 | return null 73 | } 74 | } 75 | 76 | 77 | -------------------------------------------------------------------------------- /frontend/src/provider/ViewportContext.tsx: -------------------------------------------------------------------------------- 1 | import React, { createContext, useContext, useMemo, useRef } from "react"; 2 | 3 | type ViewportActions = { 4 | resetCamera: () => void; 5 | createCube: () => void; 6 | createSphere: () => void; 7 | startHandDrag: () => void; 8 | updateHandDragNormalized: (u: number, v: number) => void; 9 | endHandDrag: () => void; 10 | orbitRotate: (dxN: number, dyN: number) => void; 11 | orbitPan: (dxN: number, dyN: number) => void; 12 | orbitDolly: (delta: number) => void; 13 | }; 14 | 15 | type RegisteredActions = Partial; 16 | 17 | const noop = () => {}; 18 | 19 | const ViewportContext = createContext(null); 20 | 21 | export function ViewportProvider({ children }: { children: React.ReactNode }) { 22 | const actionsRef = useRef({}); 23 | 24 | const value = useMemo(() => { 25 | return { 26 | resetCamera: () => (actionsRef.current.resetCamera || noop)(), 27 | createCube: () => (actionsRef.current.createCube || noop)(), 28 | createSphere: () => (actionsRef.current.createSphere || noop)(), 29 | startHandDrag: () => (actionsRef.current.startHandDrag || noop)(), 30 | updateHandDragNormalized: (u: number, v: number) => 31 | (actionsRef.current.updateHandDragNormalized || noop)(u, v), 32 | endHandDrag: () => (actionsRef.current.endHandDrag || noop)(), 33 | orbitRotate: (dxN: number, dyN: number) => 34 | (actionsRef.current.orbitRotate || noop)(dxN, dyN), 35 | orbitPan: (dxN: number, dyN: number) => 36 | (actionsRef.current.orbitPan || noop)(dxN, dyN), 37 | orbitDolly: (delta: number) => 38 | (actionsRef.current.orbitDolly || noop)(delta), 39 | }; 40 | }, []); 41 | 42 | // Hidden registrar: expose a way for the viewport to register real actions 43 | // eslint-disable-next-line react/display-name 44 | (value as any).__registerViewportActions = (impl: RegisteredActions) => { 45 | actionsRef.current = { ...actionsRef.current, ...impl }; 46 | }; 47 | 48 | return ( 49 | 50 | {children} 51 | 52 | ); 53 | } 54 | 55 | export function useViewportActions() { 56 | const ctx = useContext(ViewportContext); 57 | if (!ctx) 58 | throw new Error( 59 | "useViewportActions must be used within ViewportProvider" 60 | ); 61 | return ctx; 62 | } 63 | 64 | export function useRegisterViewportActions() { 65 | const ctx = useContext(ViewportContext) as any; 66 | if (!ctx) 67 | throw new Error( 68 | "useRegisterViewportActions must be used within ViewportProvider" 69 | ); 70 | const register = (impl: RegisteredActions) => 71 | ctx.__registerViewportActions?.(impl); 72 | return register as (impl: RegisteredActions) => void; 73 | } 74 | -------------------------------------------------------------------------------- /backend/hand_processor.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import mediapipe as mp 4 | 5 | class HandProcessor: 6 | def __init__(self): 7 | self.mp_drawing = mp.solutions.drawing_utils 8 | self.mp_hands = mp.solutions.hands 9 | self.grid_size = 400 10 | self.margin = 0.3 11 | 12 | def process_hand(self, frame, landmarks, handedness): 13 | # Draw on main frame 14 | annotated_frame = frame.copy() 15 | self.mp_drawing.draw_landmarks( 16 | annotated_frame, 17 | landmarks, 18 | self.mp_hands.HAND_CONNECTIONS, 19 | self.mp_drawing.DrawingSpec(color=(0,255,0), thickness=2), 20 | self.mp_drawing.DrawingSpec(color=(0,0,255), thickness=2)) 21 | 22 | # Create grid visualization 23 | grid_canvas = np.zeros((self.grid_size, self.grid_size, 3), dtype=np.uint8) 24 | self._draw_grid(grid_canvas) 25 | 26 | try: 27 | # Safety checks for landmarks 28 | xs = [lm.x for lm in landmarks.landmark] 29 | ys = [lm.y for lm in landmarks.landmark] 30 | if not xs or not ys: 31 | return annotated_frame, grid_canvas 32 | 33 | min_x, max_x = min(xs), max(xs) 34 | min_y, max_y = min(ys), max(ys) 35 | 36 | # Handle edge cases where hand is off-screen 37 | if max_x - min_x < 0.01 or max_y - min_y < 0.01: 38 | return annotated_frame, grid_canvas 39 | 40 | self._draw_upright_connections(grid_canvas, landmarks, min_x, max_x, min_y, max_y) 41 | 42 | hand_type = "Right" if "Left" in handedness.classification[0].label else "Left" 43 | cv2.putText(grid_canvas, hand_type, (10, 30), 44 | cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2) 45 | 46 | except Exception as e: 47 | print(f"Hand processing error: {str(e)}") 48 | 49 | return annotated_frame, grid_canvas 50 | 51 | def _draw_upright_connections(self, canvas, landmarks, min_x, max_x, min_y, max_y): 52 | connections = self.mp_hands.HAND_CONNECTIONS 53 | for connection in connections: 54 | start_idx, end_idx = connection 55 | start_lm = landmarks.landmark[start_idx] 56 | end_lm = landmarks.landmark[end_idx] 57 | 58 | # Convert to grid coordinates with safety checks 59 | try: 60 | x1 = int((start_lm.x - min_x) / (max_x - min_x) * self.grid_size) 61 | y1 = self.grid_size - int((start_lm.y - min_y) / (max_y - min_y) * self.grid_size) 62 | x2 = int((end_lm.x - min_x) / (max_x - min_x) * self.grid_size) 63 | y2 = self.grid_size - int((end_lm.y - min_y) / (max_y - min_y) * self.grid_size) 64 | 65 | cv2.line(canvas, (x1, y1), (x2, y2), (255, 255, 255), 2) 66 | cv2.circle(canvas, (x1, y1), 3, (0, 255, 0), -1) 67 | cv2.circle(canvas, (x2, y2), 3, (0, 255, 0), -1) 68 | except: 69 | continue 70 | 71 | def _draw_grid(self, canvas, spacing=50): 72 | color = (60, 60, 60) 73 | for x in range(0, self.grid_size, spacing): 74 | cv2.line(canvas, (x, 0), (x, self.grid_size), color, 1) 75 | for y in range(0, self.grid_size, spacing): 76 | cv2.line(canvas, (0, y), (self.grid_size, y), color, 1) -------------------------------------------------------------------------------- /frontend/sample-server-response.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "success", 3 | "hands": [ 4 | { 5 | "handedness": "Right", 6 | "landmarks": [ 7 | [556.812, 288.275, 0], 8 | [509.324, 290.542, -0.028], 9 | [462.918, 269.514, -0.04], 10 | [431.867, 244.691, -0.049], 11 | [403.483, 225.03, -0.058], 12 | [459.716, 203.35, -0.008], 13 | [433.972, 167.59, -0.027], 14 | [420.523, 145.474, -0.046], 15 | [409.252, 124.545, -0.061], 16 | [483.825, 186.076, -0.011], 17 | [459.016, 145.968, -0.026], 18 | [443.771, 120.206, -0.043], 19 | [429.827, 97.841, -0.058], 20 | [510.547, 179.039, -0.02], 21 | [490.237, 141.539, -0.037], 22 | [476.988, 119.693, -0.049], 23 | [462.791, 100.651, -0.057], 24 | [540.184, 178.932, -0.032], 25 | [525.056, 147.603, -0.049], 26 | [515.683, 130.163, -0.054], 27 | [505.583, 114.745, -0.058] 28 | ], 29 | "connections": [ 30 | [3, 4], 31 | [0, 5], 32 | [17, 18], 33 | [0, 17], 34 | [13, 14], 35 | [13, 17], 36 | [18, 19], 37 | [5, 6], 38 | [5, 9], 39 | [14, 15], 40 | [0, 1], 41 | [9, 10], 42 | [1, 2], 43 | [9, 13], 44 | [10, 11], 45 | [19, 20], 46 | [6, 7], 47 | [15, 16], 48 | [2, 3], 49 | [11, 12], 50 | [7, 8] 51 | ], 52 | "detected_symbols": [] 53 | }, 54 | { 55 | "handedness": "Left", 56 | "landmarks": [ 57 | [131.743, 302.232, 0], 58 | [178.808, 293.657, -0.022], 59 | [220.768, 271.52, -0.031], 60 | [248.233, 250.009, -0.041], 61 | [269.883, 231.663, -0.051], 62 | [210.031, 206.221, -0.01], 63 | [233.451, 171.018, -0.029], 64 | [247.214, 149.509, -0.048], 65 | [258.922, 130.357, -0.062], 66 | [186.256, 192.054, -0.015], 67 | [206.491, 147.871, -0.03], 68 | [220.382, 121.189, -0.047], 69 | [232.372, 98.714, -0.06], 70 | [160.224, 188.502, -0.024], 71 | [175.314, 146.907, -0.04], 72 | [187.153, 123.523, -0.051], 73 | [198.883, 103.113, -0.059], 74 | [131.438, 194.408, -0.035], 75 | [139.224, 162.013, -0.05], 76 | [147.09, 142.235, -0.054], 77 | [156.221, 124.277, -0.055] 78 | ], 79 | "connections": [ 80 | [3, 4], 81 | [0, 5], 82 | [17, 18], 83 | [0, 17], 84 | [13, 14], 85 | [13, 17], 86 | [18, 19], 87 | [5, 6], 88 | [5, 9], 89 | [14, 15], 90 | [0, 1], 91 | [9, 10], 92 | [1, 2], 93 | [9, 13], 94 | [10, 11], 95 | [19, 20], 96 | [6, 7], 97 | [15, 16], 98 | [2, 3], 99 | [11, 12], 100 | [7, 8] 101 | ], 102 | "detected_symbols": [] 103 | } 104 | ], 105 | "image_size": { 106 | "width": 640, 107 | "height": 360 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /frontend/src/types.ts: -------------------------------------------------------------------------------- 1 | export interface Vector3 { 2 | x: number; 3 | y: number; 4 | z: number; 5 | } 6 | 7 | export interface Euler { 8 | x: number; 9 | y: number; 10 | z: number; 11 | } 12 | 13 | export type GeometryKind = 14 | | "box" 15 | | "sphere" 16 | | "cylinder" 17 | | "cone" 18 | | "torus" 19 | | "plane" 20 | | "custom"; 21 | 22 | export interface SerializableGeometry { 23 | positions: number[]; 24 | indices?: number[]; 25 | normals?: number[]; 26 | uvs?: number[]; 27 | } 28 | 29 | export interface GeometryParamsMap { 30 | box?: { 31 | width: number; 32 | height: number; 33 | depth: number; 34 | widthSegments?: number; 35 | heightSegments?: number; 36 | depthSegments?: number; 37 | }; 38 | sphere?: { 39 | radius: number; 40 | widthSegments?: number; 41 | heightSegments?: number; 42 | }; 43 | cylinder?: { 44 | radiusTop: number; 45 | radiusBottom: number; 46 | height: number; 47 | radialSegments?: number; 48 | }; 49 | cone?: { radius: number; height: number; radialSegments?: number }; 50 | torus?: { 51 | radius: number; 52 | tube: number; 53 | radialSegments?: number; 54 | tubularSegments?: number; 55 | }; 56 | plane?: { 57 | width: number; 58 | height: number; 59 | widthSegments?: number; 60 | heightSegments?: number; 61 | }; 62 | custom?: SerializableGeometry; 63 | } 64 | 65 | export interface MaterialProps { 66 | color: string; 67 | metalness: number; 68 | roughness: number; 69 | opacity: number; 70 | transparent: boolean; 71 | } 72 | 73 | export type LightType = "directional" | "point" | "spot" | "ambient"; 74 | 75 | export interface LightProps { 76 | color: string; 77 | intensity: number; 78 | distance?: number; // for point and spot lights 79 | angle?: number; // for spot lights 80 | penumbra?: number; // for spot lights 81 | decay?: number; // for point and spot lights 82 | } 83 | 84 | export interface SceneLight { 85 | id: string; 86 | name: string; 87 | type: LightType; 88 | position: Vector3; 89 | rotation: Euler; // for directional and spot lights 90 | props: LightProps; 91 | visible: boolean; 92 | castShadow: boolean; 93 | } 94 | 95 | export interface SceneObject { 96 | id: string; 97 | name: string; 98 | geometry: GeometryKind; 99 | geometryParams: GeometryParamsMap[keyof GeometryParamsMap] | undefined; 100 | position: Vector3; 101 | rotation: Euler; // radians 102 | scale: Vector3; 103 | material: MaterialProps; 104 | visible: boolean; 105 | locked: boolean; 106 | } 107 | 108 | export interface HistoryState { 109 | objects: SceneObject[]; 110 | selectedId: string | null; 111 | } 112 | 113 | export type TransformMode = "translate" | "rotate" | "scale"; 114 | 115 | export type EditorMode = "object" | "edit" | "render"; 116 | 117 | export interface SnapSettings { 118 | enableSnapping: boolean; 119 | translateSnap: number; // units 120 | rotateSnap: number; // radians 121 | scaleSnap: number; // unit step 122 | } 123 | 124 | export interface Checkpoint { 125 | id: string; 126 | label: string; 127 | timestamp: number; 128 | prompt?: string; 129 | response?: string; 130 | state: HistoryState; 131 | } 132 | 133 | export interface EditorState { 134 | objects: SceneObject[]; 135 | lights: SceneLight[]; 136 | selectedId: string | null; 137 | mode: TransformMode; 138 | editorMode?: EditorMode; 139 | snap: SnapSettings; 140 | past: HistoryState[]; 141 | future: HistoryState[]; 142 | isTransforming?: boolean; 143 | checkpoints: Checkpoint[]; 144 | } 145 | -------------------------------------------------------------------------------- /frontend/src/assets/react.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /frontend/src/components/VideoStream.tsx: -------------------------------------------------------------------------------- 1 | import styled from "styled-components"; 2 | import { useState, useEffect } from "react"; 3 | import { useVideoStream } from "../holo/provider/VideoStreamContext"; 4 | 5 | const VideoContainer = styled.div` 6 | position: absolute; 7 | bottom: 12px; 8 | left: 12px; 9 | width: 200px; 10 | background: rgba(18, 20, 26, 0.85); 11 | border: 1px solid rgba(255, 255, 255, 0.08); 12 | border-radius: 8px; 13 | padding: 6px; 14 | display: flex; 15 | flex-direction: column; 16 | gap: 4px; 17 | z-index: 10; 18 | `; 19 | 20 | const Video = styled.video` 21 | width: 100%; 22 | height: 100px; 23 | background: #0f1116; 24 | border-radius: 6px; 25 | transform: scaleX(-1); 26 | object-fit: cover; 27 | border: 1px solid rgba(255, 255, 255, 0.06); 28 | `; 29 | 30 | const Status = styled.div` 31 | font-size: 10px; 32 | color: #8c8c8c; 33 | background: rgba(0, 0, 0, 0.3); 34 | padding: 3px 6px; 35 | border-radius: 4px; 36 | text-align: center; 37 | border: 1px solid rgba(255, 255, 255, 0.04); 38 | `; 39 | 40 | const CameraSelector = styled.select` 41 | background: rgba(0, 0, 0, 0.4); 42 | color: #e6e9ef; 43 | border: 1px solid rgba(255, 255, 255, 0.08); 44 | border-radius: 4px; 45 | padding: 2px 4px; 46 | font-size: 9px; 47 | width: 100%; 48 | margin-top: 2px; 49 | 50 | option { 51 | background: #1e222c; 52 | color: #e6e9ef; 53 | } 54 | `; 55 | 56 | const Controls = styled.div` 57 | display: flex; 58 | flex-direction: column; 59 | gap: 2px; 60 | `; 61 | 62 | export function VideoStream() { 63 | const { videoRef, status, getAvailableCameras, setActiveCamera } = 64 | useVideoStream(); 65 | const [cameras, setCameras] = useState([]); 66 | const [selectedCamera, setSelectedCamera] = useState(""); 67 | 68 | useEffect(() => { 69 | const loadCameras = async () => { 70 | const availableCameras = await getAvailableCameras(); 71 | setCameras(availableCameras); 72 | if (availableCameras.length > 0 && !selectedCamera) { 73 | setSelectedCamera(availableCameras[0].deviceId); 74 | } 75 | }; 76 | loadCameras(); 77 | }, [getAvailableCameras]); 78 | 79 | // Listen for device changes 80 | useEffect(() => { 81 | const handleDeviceChange = () => { 82 | const loadCameras = async () => { 83 | const availableCameras = await getAvailableCameras(); 84 | setCameras(availableCameras); 85 | }; 86 | loadCameras(); 87 | }; 88 | 89 | navigator.mediaDevices.addEventListener( 90 | "devicechange", 91 | handleDeviceChange 92 | ); 93 | return () => { 94 | navigator.mediaDevices.removeEventListener( 95 | "devicechange", 96 | handleDeviceChange 97 | ); 98 | }; 99 | }, [getAvailableCameras]); 100 | 101 | const handleCameraChange = ( 102 | event: React.ChangeEvent 103 | ) => { 104 | const cameraId = event.target.value; 105 | console.log("Switching to camera:", cameraId); 106 | setSelectedCamera(cameraId); 107 | setActiveCamera(cameraId); 108 | }; 109 | 110 | return ( 111 | 112 |

133 | ); 134 | } 135 | 136 | export default VideoStream; 137 | -------------------------------------------------------------------------------- /frontend/src/components/AgentTimeline.tsx: -------------------------------------------------------------------------------- 1 | import styled from "styled-components"; 2 | import type { StepResult } from "../agent/types"; 3 | import { useAgentTimeline } from "../store/agentTimeline"; 4 | 5 | const Rail = styled.div` 6 | position: absolute; 7 | top: 56px; 8 | bottom: 12px; 9 | right: 300px; /* to the left of chat panel */ 10 | width: 260px; 11 | background: rgba(18, 20, 26, 0.9); 12 | color: #e6e9ef; 13 | border: 1px solid rgba(255, 255, 255, 0.08); 14 | border-radius: 12px; 15 | display: flex; 16 | flex-direction: column; 17 | overflow: hidden; 18 | z-index: 40; 19 | `; 20 | 21 | const Header = styled.div` 22 | padding: 8px 12px; 23 | font-weight: 600; 24 | border-bottom: 1px solid rgba(255, 255, 255, 0.06); 25 | display: flex; 26 | align-items: center; 27 | justify-content: space-between; 28 | `; 29 | 30 | const CloseBtn = styled.button` 31 | background: #12141a; 32 | color: #e6e9ef; 33 | border: 1px solid rgba(255,255,255,0.1); 34 | border-radius: 8px; 35 | padding: 4px 8px; 36 | font-size: 12px; 37 | `; 38 | 39 | const Steps = styled.div` 40 | flex: 1; 41 | overflow: auto; 42 | padding: 8px 10px; 43 | display: flex; 44 | flex-direction: column; 45 | gap: 6px; 46 | `; 47 | 48 | const Row = styled.div<{ ok: boolean }>` 49 | border: 1px solid rgba(255, 255, 255, 0.08); 50 | background: ${(p) => (p.ok ? "rgba(30, 34, 44, 0.7)" : "rgba(60, 20, 20, 0.5)")}; 51 | border-radius: 10px; 52 | padding: 8px; 53 | font-size: 12px; 54 | `; 55 | 56 | const Badge = styled.span` 57 | display: inline-block; 58 | padding: 2px 6px; 59 | margin-right: 6px; 60 | border: 1px solid rgba(255,255,255,0.12); 61 | border-radius: 999px; 62 | background: rgba(255,255,255,0.06); 63 | font-size: 11px; 64 | `; 65 | 66 | export function AgentTimeline({ steps }: { steps: StepResult[] }) { 67 | const setShow = useAgentTimeline((s) => s.setShowTimeline); 68 | return ( 69 | 70 |

71 |

Agent Timeline

72 | setShow(false)}>✕ 73 |

74 | 75 | {steps.length === 0 ? ( 76 |

No steps yet

77 | ) : ( 78 | steps.map((s, i) => { 79 | const tool = s.tool?.name; 80 | const args = s.tool?.args || {}; 81 | const diff = s.diff || { addedIds: [], removedIds: [], updatedIds: [] }; 82 | const diffs: string[] = []; 83 | if (diff.addedIds.length) diffs.push(`+${diff.addedIds.length} added`); 84 | if (diff.removedIds.length) diffs.push(`-${diff.removedIds.length} removed`); 85 | if (diff.updatedIds.length) diffs.push(`~${diff.updatedIds.length} updated`); 86 | return ( 87 | 88 |

89 |

{tool}

90 |

{s.ok ? "✅" : "❌"}

91 |

92 |

{shortArgs(args)}

93 |

94 | {diffs.map((d, j) => ( 95 | {d} 96 | ))} 97 |

98 | 99 | ); 100 | }) 101 | )} 102 | 103 | 104 | ); 105 | } 106 | 107 | function shortArgs(args: any): string { 108 | try { 109 | const pruned: any = {}; 110 | for (const k of Object.keys(args || {})) { 111 | const v = (args as any)[k]; 112 | if (v == null) continue; 113 | if (Array.isArray(v)) pruned[k] = v.map((x) => (typeof x === "number" ? Number(x.toFixed?.(3) ?? x) : x)); 114 | else if (typeof v === "object") pruned[k] = "{...}"; 115 | else pruned[k] = v; 116 | } 117 | return JSON.stringify(pruned); 118 | } catch { 119 | return ""; 120 | } 121 | } 122 | 123 | export default AgentTimeline; -------------------------------------------------------------------------------- /frontend/src/holo/provider/WebSocketContext.tsx: -------------------------------------------------------------------------------- 1 | import { createContext, useContext, useRef, useEffect, useMemo } from "react"; 2 | import type { ReactNode } from "react"; 3 | import type { SocketStatus } from "../objects/socketstatus"; 4 | 5 | interface WebSocketProps { 6 | url: string; 7 | children: ReactNode; 8 | } 9 | 10 | interface WebSocketContextType { 11 | sendFrame: (frame: ArrayBuffer) => boolean; 12 | getWebSocket: () => WebSocket | null; 13 | getConnectionStatus: () => SocketStatus; 14 | getAcknowledged: () => boolean; 15 | getData: () => object | null; 16 | getDataVersion: () => number; 17 | } 18 | 19 | const WebSocketContext = createContext(null); 20 | 21 | export const WebSocketProvider = ({ url, children }: WebSocketProps) => { 22 | const wsRef = useRef(null); 23 | const retryTimeoutRef = useRef(null); 24 | const acknowledgedRef = useRef(true); 25 | const dataRef = useRef