├── img
    ├── placeholder.txt
    ├── img_v1.png
    └── img_debug.png
├── requirements.txt
├── utils
    ├── index.ts
    ├── event.ts
    └── sub-renders.ts
├── LICENSE
├── config.py
├── stop_api.py
├── join_api.py
├── test_speaker.py
├── rag_server.py
├── emotion_led_service.py
├── my_city_info.txt
├── README.md
├── type.ts
├── index.ts
└── index.html


/img/placeholder.txt:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/img/img_v1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Seeed-Projects/Agora_Convo_AI_reSpeaker/main/img/img_v1.png


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Seeed-Projects/Agora_Convo_AI_reSpeaker/main/requirements.txt


--------------------------------------------------------------------------------
/img/img_debug.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Seeed-Projects/Agora_Convo_AI_reSpeaker/main/img/img_debug.png


--------------------------------------------------------------------------------
/utils/index.ts:
--------------------------------------------------------------------------------
1 | // export const factoryFormatLog =
2 | //   (options: { tag: string }) => (message: unknown) => {
3 | //     return `[${options.tag}] ${JSON.stringify(message)}`
4 | //   }
5 | export const factoryFormatLog =
6 |   (options: { tag: string }) =>
7 |   (...args: unknown[]) => {
8 |     return `[${options.tag}] ${args.map((arg) => JSON.stringify(arg)).join(' ')}`
9 |   }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2025 Seeed-Projects
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/config.py:
--------------------------------------------------------------------------------
 1 | # config.py
 2 | # Central configuration file for Agora AI Voice Chat
 3 | 
 4 | # ==========================
 5 | # AGORA CREDENTIALS
 6 | # ==========================
 7 | CUSTOMER_KEY = "xxxxxxxxxxxx"
 8 | CUSTOMER_SECRET = "xxxxxxxxxx"
 9 | APP_ID = "xxxxxxxxxxxx"
10 | 
11 | # ==========================
12 | # CHANNEL SETTINGS
13 | # ==========================
14 | CHANNEL_NAME = "test"
15 | AGORA_TEMP_TOKEN = "xxxxxxxxxxxxxxxxxxxxxxxxxxx"
16 | 
17 | # Agent and User UIDs
18 | AGENT_RTC_UID = "1001"
19 | USER_RTC_UID = "1002"
20 | 
21 | # ==========================
22 | # 3RD PARTY SERVICES
23 | # ==========================
24 | ASSEMBLY_AI_KEY = "xxxxxxxxxxxxx"
25 | GROQ_KEY = "xxxxxxxxxxxxxxxx"
26 | TTS_GROQ_KEY = "xxxxxxxxxxxxxxxxxx"
27 | 
28 | # ==========================
29 | # AGENT SETTINGS
30 | # ==========================
31 | IDLE_TIMEOUT = 120  # seconds
32 | MAX_HISTORY = 32
33 | 
34 | # System prompt for the AI
35 | SYSTEM_PROMPT = "You are a helpful chatbot."
36 | GREETING_MESSAGE = "Hello, how can I assist you?"
37 | FAILURE_MESSAGE = "Please hold on a second."
38 | 
39 | # LLM Model
40 | LLM_MODEL = "llama-3.3-70b-versatile"
41 | 
42 | # TTS Voice
43 | TTS_MODEL = "playai-tts"
44 | TTS_VOICE = "Arista-PlayAI"
45 | 
46 | # ASR Language
47 | ASR_LANGUAGE = "en-US"
48 | 


--------------------------------------------------------------------------------
/utils/event.ts:
--------------------------------------------------------------------------------
 1 | /* eslint-disable @typescript-eslint/no-explicit-any */
 2 | 
 3 | type EventHandler<T extends any[]> = (...data: T) => void
 4 | 
 5 | export class EventHelper<T> {
 6 |   private _eventMap: Map<keyof T, EventHandler<any[]>[]> = new Map()
 7 | 
 8 |   once<Key extends keyof T>(evt: Key, cb: T[Key]) {
 9 |     const wrapper = (...args: any[]) => {
10 |       this.off(evt, wrapper as any)
11 |       ;(cb as any)(...args)
12 |     }
13 |     this.on(evt, wrapper as any)
14 |     return this
15 |   }
16 | 
17 |   on<Key extends keyof T>(evt: Key, cb: T[Key]) {
18 |     const cbs = this._eventMap.get(evt) ?? []
19 |     cbs.push(cb as any)
20 |     this._eventMap.set(evt, cbs)
21 |     console.debug(`Subscribed to event: ${String(evt)}`)
22 |     return this
23 |   }
24 | 
25 |   off<Key extends keyof T>(evt: Key, cb: T[Key]) {
26 |     const cbs = this._eventMap.get(evt)
27 |     if (cbs) {
28 |       this._eventMap.set(
29 |         evt,
30 |         cbs.filter((it) => it !== cb)
31 |       )
32 |       console.debug(`Unsubscribed from event: ${String(evt)}`)
33 |     }
34 |     return this
35 |   }
36 | 
37 |   removeAllEventListeners(): void {
38 |     this._eventMap.clear()
39 |     console.debug('Removed all event listeners')
40 |   }
41 | 
42 |   emit<Key extends keyof T>(evt: Key, ...args: any[]) {
43 |     const cbs = this._eventMap.get(evt) ?? []
44 |     for (const cb of cbs) {
45 |       try {
46 |         // eslint-disable-next-line @typescript-eslint/no-unused-expressions
47 |         cb && cb(...args)
48 |       } catch (e) {
49 |         // cb exception should not affect other callbacks
50 |         const error = e as Error
51 |         const details = error.stack || error.message
52 |         console.error(`Error handling event ${String(evt)}: ${details}`)
53 |       }
54 |     }
55 |     console.debug({ args }, `Emitted event: ${String(evt)}`)
56 |     return this
57 |   }
58 | }


--------------------------------------------------------------------------------
/stop_api.py:
--------------------------------------------------------------------------------
 1 | # stop_api.py
 2 | # Stop the conversational AI agent
 3 | 
 4 | import base64
 5 | import requests
 6 | import sys
 7 | from config import CUSTOMER_KEY, CUSTOMER_SECRET, APP_ID
 8 | 
 9 | # ==========================
10 | # AGENT ID (Update this!)
11 | # ==========================
12 | # Copy the agent_id from join_api.py output
13 | AGENT_ID = "A42AC47HM54EV67MM93MC94VN86NR36T"
14 | 
15 | # ==========================
16 | # Base64 encode credentials
17 | # ==========================
18 | raw = f"{CUSTOMER_KEY}:{CUSTOMER_SECRET}"
19 | BASIC = base64.b64encode(raw.encode()).decode()
20 | 
21 | # ==========================
22 | # Stop Agent URL
23 | # ==========================
24 | url = f"https://api.agora.io/api/conversational-ai-agent/v2/projects/{APP_ID}/agents/{AGENT_ID}/leave"
25 | 
26 | headers = {
27 |     "Authorization": f"Basic {BASIC}",
28 |     "Content-Type": "application/json"
29 | }
30 | 
31 | # ==========================
32 | # Validation
33 | # ==========================
34 | if AGENT_ID == "YOUR_AGENT_ID_HERE":
35 |     print("=" * 50)
36 |     print("❌ ERROR: Please update AGENT_ID first!")
37 |     print("=" * 50)
38 |     print("\n📝 Steps:")
39 |     print("1. Run join_api.py")
40 |     print("2. Copy the agent_id from the output")
41 |     print("3. Paste it in stop_api.py (line 11)")
42 |     print("4. Run stop_api.py again")
43 |     print("=" * 50)
44 |     sys.exit(1)
45 | 
46 | # ==========================
47 | # SEND REQUEST
48 | # ==========================
49 | print("=" * 50)
50 | print("🛑 Stopping AI Agent...")
51 | print("=" * 50)
52 | print(f"Agent ID: {AGENT_ID}")
53 | print("=" * 50)
54 | 
55 | response = requests.post(url, headers=headers)
56 | 
57 | print("\n📊 Response:")
58 | print("-" * 50)
59 | print("Status Code:", response.status_code)
60 | 
61 | if response.status_code == 200:
62 |     print("\n✅ SUCCESS!")
63 |     print("Agent stopped and left the channel")
64 | else:
65 |     print("\n❌ FAILED!")
66 |     try:
67 |         print("Error:", response.json())
68 |     except:
69 |         print("Error:", response.text)
70 | 
71 | print("\n" + "=" * 50)


--------------------------------------------------------------------------------
/join_api.py:
--------------------------------------------------------------------------------
  1 | # join_api.py
  2 | # Start the conversational AI agent with RAG server and emotion support
  3 | 
  4 | import base64
  5 | import requests
  6 | import json
  7 | from config import *
  8 | 
  9 | # ==========================
 10 | # Encode customer_key:customer_secret → Base64
 11 | # ==========================
 12 | raw_cred = f"{CUSTOMER_KEY}:{CUSTOMER_SECRET}"
 13 | BASIC_AUTH = base64.b64encode(raw_cred.encode()).decode()
 14 | 
 15 | # ==========================
 16 | # Agora Join URL
 17 | # ==========================
 18 | url = f"https://api.agora.io/api/conversational-ai-agent/v2/projects/{APP_ID}/join"
 19 | 
 20 | # ==========================
 21 | # Headers
 22 | # ==========================
 23 | headers = {
 24 |     "Authorization": f"Basic {BASIC_AUTH}",
 25 |     "Content-Type": "application/json"
 26 | }
 27 | 
 28 | # ==========================
 29 | # RAG SERVER CONFIGURATION
 30 | # ==========================
 31 | RAG_SERVER_URL = "https://abcnoncontingently-stotious.ngrok-free.dev/rag/chat/completions"
 32 | USE_RAG = True
 33 | 
 34 | # ==========================
 35 | # ENHANCED SYSTEM PROMPT WITH EMOTIONS
 36 | # ==========================
 37 | ENHANCED_SYSTEM_PROMPT = """You are an enthusiastic and friendly tour guide for Central City Mall.
 38 | 
 39 | CRITICAL: Start EVERY response with EXACTLY ONE emotion label in square brackets.
 40 | 
 41 | Available emotions and when to use them:
 42 | - [excited] - Amazing sales (40% OFF!), incredible deals, special promotions
 43 | - [happy] - Good news, available services, pleasant information
 44 | - [surprised] - Hidden features (secret rooftop garden!), unexpected facts
 45 | - [sad] - Temporary closures, apologizing, unavailable services
 46 | - [helpful] - Giving directions, guiding visitors
 47 | - [thinking] - Searching for information, checking details
 48 | - [neutral] - Basic information, hours, standard locations
 49 | - [welcoming] - Greetings, hello messages
 50 | 
 51 | EMOTION MATCHING EXAMPLES:
 52 | - Sales/Discounts → [excited] "WOW! 40% OFF on mobile phones!"
 53 | - Hidden features → [surprised] "You won't believe this - there's a SECRET rooftop garden!"
 54 | - Closures → [sad] "Unfortunately, Indian Spice Junction is temporarily closed."
 55 | - Directions → [helpful] "The washroom is on the second floor near the escalators."
 56 | - Greetings → [welcoming] "Hello! Welcome to Central City Mall!"
 57 | 
 58 | Rules:
 59 | 1. ALWAYS start with [emotion]
 60 | 2. Use emphatic language matching the emotion
 61 | 3. Keep responses under 3 sentences
 62 | 4. Be specific with floor numbers
 63 | 5. Match emotion intensity to content"""
 64 | 
 65 | # ==========================
 66 | # Request Body
 67 | # ==========================
 68 | payload = {
 69 |     "name": "rag_agent_with_emotions",
 70 |     "properties": {
 71 |         "channel": CHANNEL_NAME,
 72 |         "token": AGORA_TEMP_TOKEN,
 73 |         "agent_rtc_uid": AGENT_RTC_UID,
 74 |         "remote_rtc_uids": [USER_RTC_UID],
 75 |         "idle_timeout": IDLE_TIMEOUT,
 76 | 
 77 |         "advanced_features": {
 78 |             "enable_aivad": True,
 79 |             "enable_rtm": True
 80 |         },
 81 | 
 82 |         "parameters": {
 83 |             "data_channel": "rtm"
 84 |         },
 85 | 
 86 |         # ========= LLM Configuration ==========
 87 |         "llm": {
 88 |             "url": RAG_SERVER_URL if USE_RAG else "https://api.groq.com/openai/v1/chat/completions",
 89 |             "api_key": "" if USE_RAG else GROQ_KEY,
 90 |             "system_messages": [
 91 |                 {
 92 |                     "role": "system",
 93 |                     "content": ENHANCED_SYSTEM_PROMPT
 94 |                 }
 95 |             ],
 96 |             "max_history": MAX_HISTORY,
 97 |             "greeting_message": "[welcoming] Hello! Welcome to Central City Mall. How can I assist you today?",
 98 |             "failure_message": "[thinking] Let me check that information for you. One moment please.",
 99 |             "params": {
100 |                 "model": LLM_MODEL
101 |             }
102 |         },
103 | 
104 |         # ========= TTS - Skip Emotion Labels ==========
105 |         "tts": {
106 |             "vendor": "groq",
107 |             "params": {
108 |                 "api_key": TTS_GROQ_KEY,
109 |                 "model": TTS_MODEL,
110 |                 "voice": TTS_VOICE
111 |             },
112 |             "skip_patterns": [4]  # Skip content in square brackets []
113 |         },
114 | 
115 |         # ========= ASR ==========
116 |         "asr": {
117 |             "vendor": "assemblyai",
118 |             "params": {
119 |                 "api_key": ASSEMBLY_AI_KEY,
120 |                 "language": ASR_LANGUAGE
121 |             }
122 |         }
123 |     }
124 | }
125 | 
126 | # ==========================
127 | # SEND REQUEST
128 | # ==========================
129 | print("=" * 60)
130 | print("🚀 Starting AI Agent with Enhanced Emotions & RAG")
131 | print("=" * 60)
132 | print(f"Channel: {CHANNEL_NAME}")
133 | print(f"Agent UID: {AGENT_RTC_UID}")
134 | print(f"User UID: {USER_RTC_UID}")
135 | print(f"RAG Mode: {'ENABLED ✅' if USE_RAG else 'DISABLED ❌'}")
136 | if USE_RAG:
137 |     print(f"RAG Server: {RAG_SERVER_URL}")
138 | print("=" * 60)
139 | 
140 | response = requests.post(url, headers=headers, data=json.dumps(payload))
141 | 
142 | print("\n📊 Response:")
143 | print("-" * 60)
144 | print("Status Code:", response.status_code)
145 | 
146 | if response.status_code == 200:
147 |     result = response.json()
148 |     print("\n✅ SUCCESS!")
149 |     print(f"Agent ID: {result['agent_id']}")
150 |     print(f"Status: {result['status']}")
151 |     print(f"Created: {result['create_ts']}")
152 |     print("\n" + "=" * 60)
153 |     print("⚠️  SAVE THIS AGENT ID FOR STOPPING:")
154 |     print(f"   {result['agent_id']}")
155 |     print("=" * 60)
156 | 
157 |     if USE_RAG:
158 |         print("\n💡 RAG with Emotions is ACTIVE!")
159 |         print("   The AI will answer based on enhanced my_city_info.txt")
160 |         print("\n🎭 Try these emotional queries:")
161 |         print("   - 'Are there any special offers?' → [excited]")
162 |         print("   - 'What are some hidden features?' → [surprised]")
163 |         print("   - 'Is there Indian food?' → [sad]")
164 |         print("   - 'Where is the washroom?' → [helpful]")
165 |         print("   - 'Hello!' → [welcoming]")
166 | else:
167 |     print("\n❌ FAILED!")
168 |     try:
169 |         print("Error:", response.json())
170 |     except:
171 |         print("Error:", response.text)
172 | 
173 | 
174 | print("\n✨ You can now open index.html and click 'Start Conversation'")
175 | 
176 | 


--------------------------------------------------------------------------------
/test_speaker.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | """
  3 | Test script for ReSpeaker LED Emotion Control
  4 | Tests all emotions with their corresponding colors
  5 | """
  6 | 
  7 | import requests
  8 | import time
  9 | import sys
 10 | 
 11 | LED_SERVICE_URL = "http://localhost:5000"
 12 | 
 13 | # Emotion test scenarios
 14 | TEST_SCENARIOS = [
 15 |     {
 16 |         "emotion": "welcoming",
 17 |         "description": "Pink breathing - Warm welcome",
 18 |         "example": "Hello! Welcome to Central City Mall!"
 19 |     },
 20 |     {
 21 |         "emotion": "excited",
 22 |         "description": "Magenta breathing - Vibrant energy",
 23 |         "example": "WOW! 40% OFF on mobile phones!"
 24 |     },
 25 |     {
 26 |         "emotion": "surprised",
 27 |         "description": "Orange breathing - Bright surprise",
 28 |         "example": "You won't believe this - there's a SECRET rooftop garden!"
 29 |     },
 30 |     {
 31 |         "emotion": "helpful",
 32 |         "description": "Green breathing - Friendly assistance",
 33 |         "example": "The washroom is on the second floor near the escalators."
 34 |     },
 35 |     {
 36 |         "emotion": "thinking",
 37 |         "description": "Cyan breathing - Cool contemplation",
 38 |         "example": "Let me check that information for you..."
 39 |     },
 40 |     {
 41 |         "emotion": "happy",
 42 |         "description": "Yellow breathing - Bright and cheerful",
 43 |         "example": "Coffee Breeze Café is on the ground floor!"
 44 |     },
 45 |     {
 46 |         "emotion": "sad",
 47 |         "description": "Blue breathing - Somber",
 48 |         "example": "Unfortunately, Indian Spice Junction is temporarily closed."
 49 |     },
 50 |     {
 51 |         "emotion": "neutral",
 52 |         "description": "Light blue breathing - Calm neutral",
 53 |         "example": "The mall is open from 9:00 AM to 9:00 PM."
 54 |     }
 55 | ]
 56 | 
 57 | 
 58 | def check_service():
 59 |     """Check if LED service is running"""
 60 |     try:
 61 |         response = requests.get(f"{LED_SERVICE_URL}/status", timeout=2)
 62 |         if response.status_code == 200:
 63 |             data = response.json()
 64 |             if data.get("device_found"):
 65 |                 print("✅ LED service is running")
 66 |                 print(f"✅ ReSpeaker device found")
 67 |                 return True
 68 |             else:
 69 |                 print("❌ LED service running but device not found")
 70 |                 return False
 71 |         else:
 72 |             print("❌ LED service returned error")
 73 |             return False
 74 |     except requests.exceptions.ConnectionError:
 75 |         print("❌ Cannot connect to LED service")
 76 |         print(f"   Make sure emotion_led_service.py is running on port 5000")
 77 |         return False
 78 |     except Exception as e:
 79 |         print(f"❌ Error: {e}")
 80 |         return False
 81 | 
 82 | 
 83 | def test_emotion(emotion, description, example):
 84 |     """Test a specific emotion"""
 85 |     print(f"\n{'='*70}")
 86 |     print(f"🎭 Testing: {emotion.upper()}")
 87 |     print(f"📝 Description: {description}")
 88 |     print(f"💬 Example: \"{example}\"")
 89 |     print(f"{'='*70}")
 90 |     
 91 |     try:
 92 |         response = requests.post(
 93 |             f"{LED_SERVICE_URL}/emotion",
 94 |             json={
 95 |                 "emotion": emotion,
 96 |                 "duration": 1.5,  # Slightly longer for demo
 97 |                 "text": example
 98 |             },
 99 |             timeout=5
100 |         )
101 |         
102 |         if response.status_code == 200:
103 |             data = response.json()
104 |             print(f"✅ {data['message']}")
105 |             print(f"💡 Watch the LED breathe in {emotion} color...")
106 |             time.sleep(2.0)  # Wait for animation + return to DoA
107 |             return True
108 |         else:
109 |             print(f"❌ Failed: {response.status_code}")
110 |             return False
111 |             
112 |     except Exception as e:
113 |         print(f"❌ Error: {e}")
114 |         return False
115 | 
116 | 
117 | def test_return_to_doa():
118 |     """Test returning to DoA mode"""
119 |     print(f"\n{'='*70}")
120 |     print("🔄 Testing: Return to DoA Mode")
121 |     print(f"{'='*70}")
122 |     
123 |     try:
124 |         response = requests.post(f"{LED_SERVICE_URL}/doa", timeout=2)
125 |         if response.status_code == 200:
126 |             print("✅ Successfully returned to DoA mode")
127 |             return True
128 |         else:
129 |             print("❌ Failed to return to DoA mode")
130 |             return False
131 |     except Exception as e:
132 |         print(f"❌ Error: {e}")
133 |         return False
134 | 
135 | 
136 | def main():
137 |     """Main test function"""
138 |     print("\n" + "="*70)
139 |     print("🎭 ReSpeaker LED Emotion Control Test")
140 |     print("="*70)
141 |     
142 |     # Check service
143 |     print("\n📡 Checking LED service...")
144 |     if not check_service():
145 |         print("\n❌ LED service check failed!")
146 |         print("\n📋 To start the LED service:")
147 |         print("   python emotion_led_service.py")
148 |         sys.exit(1)
149 |     
150 |     # Wait a moment
151 |     time.sleep(1)
152 |     
153 |     # Test each emotion
154 |     print("\n🎨 Testing all emotions...")
155 |     print("💡 Watch the ReSpeaker LED ring!\n")
156 |     
157 |     success_count = 0
158 |     for scenario in TEST_SCENARIOS:
159 |         if test_emotion(
160 |             scenario["emotion"],
161 |             scenario["description"],
162 |             scenario["example"]
163 |         ):
164 |             success_count += 1
165 |         time.sleep(0.5)  # Brief pause between tests
166 |     
167 |     # Return to DoA mode
168 |     test_return_to_doa()
169 |     
170 |     # Summary
171 |     print("\n" + "="*70)
172 |     print("📊 Test Summary")
173 |     print("="*70)
174 |     print(f"✅ Successful: {success_count}/{len(TEST_SCENARIOS)}")
175 |     print(f"❌ Failed: {len(TEST_SCENARIOS) - success_count}/{len(TEST_SCENARIOS)}")
176 |     
177 |     if success_count == len(TEST_SCENARIOS):
178 |         print("\n🎉 All tests passed! LED emotion control is working perfectly!")
179 |     else:
180 |         print("\n⚠️  Some tests failed. Check the LED service logs.")
181 |     
182 |     print("\n💡 The LED should now be in DoA mode (rotating directional indicator)")
183 |     print("="*70 + "\n")
184 | 
185 | 
186 | if __name__ == "__main__":
187 |     try:
188 |         main()
189 |     except KeyboardInterrupt:
190 |         print("\n\n⚠️  Test interrupted by user")
191 |         print("🔄 Returning to DoA mode...")
192 |         try:
193 |             requests.post(f"{LED_SERVICE_URL}/doa", timeout=2)
194 |         except:
195 |             pass
196 |         sys.exit(0)
197 | 


--------------------------------------------------------------------------------
/rag_server.py:
--------------------------------------------------------------------------------
  1 | """
  2 | RAG Server for Agora AI Voice Chat with Emotion Support
  3 | This server provides a custom LLM endpoint with RAG capabilities and emotion labeling
  4 | """
  5 | 
  6 | from fastapi import FastAPI, HTTPException
  7 | from fastapi.responses import StreamingResponse
  8 | from pydantic import BaseModel
  9 | from typing import List, Dict, Optional
 10 | import os
 11 | import json
 12 | import random
 13 | from openai import AsyncOpenAI
 14 | import logging
 15 | 
 16 | # Setup logging
 17 | logging.basicConfig(
 18 |     level=logging.DEBUG,
 19 |     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
 20 | )
 21 | logger = logging.getLogger(__name__)
 22 | 
 23 | app = FastAPI()
 24 | 
 25 | # ==========================
 26 | # CONFIGURATION
 27 | # ==========================
 28 | GROQ_API_KEY = "xxxxxxxxxxxxxxxxxxxxxx"
 29 | KNOWLEDGE_BASE_PATH = "./my_city_info.txt"
 30 | 
 31 | # ==========================
 32 | # EMOTION SYSTEM PROMPT
 33 | # ==========================
 34 | EMOTION_SYSTEM_PROMPT = """You are an enthusiastic and friendly tour guide for Central City Mall.
 35 | 
 36 | CRITICAL: Start EVERY response with EXACTLY ONE emotion label in square brackets. Match the emotion to the content!
 37 | 
 38 | Available emotions and when to use them:
 39 | - [excited] - Amazing sales (40% OFF phones!), incredible deals, special promotions, unbelievable prices
 40 | - [happy] - Good news, available services, welcoming visitors, pleasant information
 41 | - [surprised] - Hidden features (secret rooftop garden!), unexpected facts, unique amenities most people don't know
 42 | - [sad] - Temporary closures (Indian restaurant closed), apologizing, unavailable services
 43 | - [helpful] - Giving directions, guiding visitors, showing locations
 44 | - [thinking] - Searching for information, checking details
 45 | - [neutral] - Basic information, hours, standard locations
 46 | - [welcoming] - Greetings, hello messages, welcoming guests
 47 | 
 48 | EMOTION MATCHING RULES:
 49 | - Sales/Discounts (40% OFF, 60% OFF, Buy 2 Get 1) → [excited]
 50 | - Hidden gems (rooftop garden, free music) → [surprised]
 51 | - Closed stores (Indian Spice Junction) → [sad]
 52 | - Directions/locations → [helpful]
 53 | - Greetings → [welcoming]
 54 | - Standard info → [neutral]
 55 | 
 56 | Example responses with STRONG emotional language:
 57 | - "[excited] WOW! The electronics store has an AMAZING 40% OFF sale on mobile phones right now!"
 58 | - "[surprised] You won't believe this - there's a SECRET rooftop garden on the 4th floor that most visitors don't know about!"
 59 | - "[sad] Unfortunately, Indian Spice Junction is temporarily closed for renovations. I apologize for the inconvenience."
 60 | - "[helpful] The washroom is on the second floor near the escalators - easy to find!"
 61 | - "[welcoming] Hello! Welcome to Central City Mall - I'm so glad you're here!"
 62 | 
 63 | Rules:
 64 | 1. ALWAYS start with [emotion] - no exceptions
 65 | 2. Use EMPHATIC language matching the emotion (WOW!, AMAZING!, Unfortunately, I'm sorry)
 66 | 3. Keep responses under 3 sentences
 67 | 4. Be specific with floor numbers and landmarks
 68 | 5. Match emotion intensity to the content
 69 | 6. Only ONE emotion per response"""
 70 | 
 71 | # ==========================
 72 | # MODELS
 73 | # ==========================
 74 | class ChatMessage(BaseModel):
 75 |     role: str
 76 |     content: str
 77 | 
 78 | class ChatCompletionRequest(BaseModel):
 79 |     model: Optional[str] = "llama-3.3-70b-versatile"
 80 |     messages: List[ChatMessage]
 81 |     stream: bool = True
 82 |     max_tokens: Optional[int] = 1000
 83 |     temperature: Optional[float] = 0.7
 84 | 
 85 | # ==========================
 86 | # RAG FUNCTIONS
 87 | # ==========================
 88 | def load_knowledge_base(file_path: str) -> str:
 89 |     """Load the knowledge base from a text file"""
 90 |     try:
 91 |         if not os.path.exists(file_path):
 92 |             logger.error(f"❌ Knowledge base file not found: {file_path}")
 93 |             return ""
 94 | 
 95 |         with open(file_path, 'r', encoding='utf-8') as f:
 96 |             content = f.read()
 97 | 
 98 |         logger.info(f"✅ Loaded knowledge base: {len(content)} characters")
 99 |         return content
100 |     except Exception as e:
101 |         logger.error(f"❌ Error loading knowledge base: {e}")
102 |         return ""
103 | 
104 | def search_knowledge_base(query: str, knowledge_base: str) -> str:
105 |     """Enhanced keyword-based search"""
106 |     logger.info(f"🔍 Searching for: '{query}'")
107 | 
108 |     if not knowledge_base:
109 |         return ""
110 | 
111 |     query_lower = query.lower()
112 | 
113 |     # Define location keywords
114 |     location_keywords = {
115 |         'coffee': ['coffee', 'café', 'cafe', 'breeze'],
116 |         'chinese': ['chinese', 'dragon', 'wok'],
117 |         'sri lankan': ['sri lankan', 'ceylon', 'spice'],
118 |         'washroom': ['washroom', 'toilet', 'restroom', 'bathroom'],
119 |         'conference': ['conference', 'hall', 'meeting'],
120 |         'subway': ['subway', 'metro', 'train'],
121 |         'parking': ['parking', 'park', 'car'],
122 |         'food': ['food', 'eat', 'restaurant', 'dining'],
123 |         'shop': ['shop', 'store', 'shopping'],
124 |         'offer': ['offer', 'discount', 'sale', 'deal', 'special'],
125 |     }
126 | 
127 |     # Split knowledge base into sections
128 |     sections = knowledge_base.split('\n\n')
129 |     scored_chunks = []
130 | 
131 |     for section in sections:
132 |         if not section.strip():
133 |             continue
134 | 
135 |         section_lower = section.lower()
136 |         score = 0
137 | 
138 |         # Check for keyword matches
139 |         for category, keywords in location_keywords.items():
140 |             if any(keyword in query_lower for keyword in keywords):
141 |                 if any(keyword in section_lower for keyword in keywords):
142 |                     score += 10
143 | 
144 |         # Check for word overlap
145 |         query_words = [w for w in query_lower.split() if len(w) > 3]
146 |         for word in query_words:
147 |             if word in section_lower:
148 |                 score += 1
149 | 
150 |         if score > 0:
151 |             scored_chunks.append((score, section))
152 | 
153 |     # Sort by relevance and take top 4
154 |     scored_chunks.sort(reverse=True, key=lambda x: x[0])
155 |     top_chunks = [chunk for _, chunk in scored_chunks[:4]]
156 | 
157 |     if top_chunks:
158 |         result = "\n\n".join(top_chunks)
159 |         logger.info(f"✅ Found {len(top_chunks)} relevant sections")
160 |         return result
161 | 
162 |     # Fallback to overview
163 |     return "\n\n".join(sections[:3])
164 | 
165 | def create_rag_enhanced_messages(
166 |     original_messages: List[ChatMessage],
167 |     retrieved_context: str
168 | ) -> List[Dict]:
169 |     """Enhance messages with retrieved context and emotion instructions"""
170 |     enhanced_messages = []
171 | 
172 |     # Add system message with emotion instructions and context
173 |     if retrieved_context:
174 |         system_content = f"""{EMOTION_SYSTEM_PROMPT}
175 | 
176 | Based on the following mall information:
177 | 
178 | {retrieved_context}
179 | 
180 | Instructions:
181 | - Answer clearly and concisely (2-3 sentences max)
182 | - Provide specific floor numbers and landmarks
183 | - If information is not available, direct to information desk on ground floor
184 | - Always be welcoming and professional"""
185 | 
186 |         enhanced_messages.append({
187 |             "role": "system",
188 |             "content": system_content
189 |         })
190 |         logger.info(f"✅ Created system message with emotions + {len(retrieved_context)} chars context")
191 |     else:
192 |         enhanced_messages.append({
193 |             "role": "system",
194 |             "content": EMOTION_SYSTEM_PROMPT
195 |         })
196 | 
197 |     # Add original messages (skip system messages)
198 |     for msg in original_messages:
199 |         if msg.role != "system":
200 |             enhanced_messages.append({
201 |                 "role": msg.role,
202 |                 "content": msg.content
203 |             })
204 | 
205 |     return enhanced_messages
206 | 
207 | # ==========================
208 | # WAITING MESSAGES WITH EMOTIONS
209 | # ==========================
210 | WAITING_MESSAGES = [
211 |     "[thinking] Just a moment, checking the mall directory...",
212 |     "[thinking] Let me look that up for you...",
213 |     "[thinking] Good question, finding the information...",
214 | ]
215 | 
216 | # ==========================
217 | # ENDPOINTS
218 | # ==========================
219 | @app.get("/")
220 | async def root():
221 |     return {
222 |         "message": "RAG Server with Emotion Support for Agora AI",
223 |         "endpoints": {
224 |             "/rag/chat/completions": "RAG-enhanced chat with emotions",
225 |             "/health": "Health check"
226 |         },
227 |         "status": "running"
228 |     }
229 | 
230 | @app.get("/health")
231 | async def health():
232 |     kb_exists = os.path.exists(KNOWLEDGE_BASE_PATH)
233 |     kb_size = os.path.getsize(KNOWLEDGE_BASE_PATH) if kb_exists else 0
234 |     return {
235 |         "status": "healthy",
236 |         "knowledge_base_loaded": kb_exists,
237 |         "knowledge_base_size": kb_size,
238 |         "groq_api_configured": bool(GROQ_API_KEY)
239 |     }
240 | 
241 | @app.post("/rag/chat/completions")
242 | async def rag_chat_completions(request: ChatCompletionRequest):
243 |     """RAG-enhanced chat completions with emotion labeling"""
244 |     try:
245 |         logger.info("=" * 60)
246 |         logger.info("📨 NEW RAG CHAT COMPLETION REQUEST WITH EMOTIONS")
247 |         logger.info("=" * 60)
248 | 
249 |         if not request.stream:
250 |             raise HTTPException(status_code=400, detail="Streaming required")
251 | 
252 |         async def generate():
253 |             try:
254 |                 # Load knowledge base
255 |                 knowledge_base = load_knowledge_base(KNOWLEDGE_BASE_PATH)
256 |                 if not knowledge_base:
257 |                     logger.error("❌ Knowledge base empty!")
258 |                     raise Exception("Knowledge base could not be loaded")
259 | 
260 |                 # Get user query
261 |                 user_messages = [msg for msg in request.messages if msg.role == "user"]
262 |                 last_user_message = user_messages[-1].content if user_messages else ""
263 |                 logger.info(f"User query: '{last_user_message}'")
264 | 
265 |                 # Search knowledge base
266 |                 retrieved_context = search_knowledge_base(last_user_message, knowledge_base)
267 |                 if not retrieved_context:
268 |                     retrieved_context = knowledge_base[:500]
269 | 
270 |                 # Create enhanced messages with emotion instructions
271 |                 enhanced_messages = create_rag_enhanced_messages(request.messages, retrieved_context)
272 | 
273 |                 # Call LLM
274 |                 logger.info("🤖 Calling Groq API...")
275 |                 client = AsyncOpenAI(
276 |                     api_key=GROQ_API_KEY,
277 |                     base_url="https://api.groq.com/openai/v1"
278 |                 )
279 | 
280 |                 response = await client.chat.completions.create(
281 |                     model=request.model,
282 |                     messages=enhanced_messages,
283 |                     stream=True,
284 |                     max_tokens=request.max_tokens,
285 |                     temperature=request.temperature
286 |                 )
287 | 
288 |                 # Stream the response
289 |                 chunk_count = 0
290 |                 async for chunk in response:
291 |                     chunk_count += 1
292 |                     yield f"data: {json.dumps(chunk.model_dump())}\n\n"
293 | 
294 |                 logger.info(f"✅ Streamed {chunk_count} chunks successfully")
295 |                 yield "data: [DONE]\n\n"
296 | 
297 |             except Exception as e:
298 |                 logger.error(f"❌ ERROR in RAG pipeline: {str(e)}", exc_info=True)
299 |                 error_msg = {
300 |                     "id": "error_msg",
301 |                     "object": "chat.completion.chunk",
302 |                     "created": 1234567890,
303 |                     "model": request.model,
304 |                     "choices": [{
305 |                         "index": 0,
306 |                         "delta": {
307 |                             "role": "assistant",
308 |                             "content": "[sad] I apologize, I'm having trouble right now. Please ask at the information desk on the ground floor."
309 |                         },
310 |                         "finish_reason": "stop"
311 |                     }]
312 |                 }
313 |                 yield f"data: {json.dumps(error_msg)}\n\n"
314 |                 yield "data: [DONE]\n\n"
315 | 
316 |         return StreamingResponse(generate(), media_type="text/event-stream")
317 | 
318 |     except Exception as e:
319 |         logger.error(f"❌ RAG error: {e}", exc_info=True)
320 |         raise HTTPException(status_code=500, detail=str(e))
321 | 
322 | # ==========================
323 | # RUN SERVER
324 | # ==========================
325 | if __name__ == "__main__":
326 |     import uvicorn
327 | 
328 |     print("=" * 60)
329 |     print("🚀 Starting RAG Server with Emotion Support")
330 |     print("=" * 60)
331 |     print(f"📚 Knowledge Base: {KNOWLEDGE_BASE_PATH}")
332 | 
333 |     if os.path.exists(KNOWLEDGE_BASE_PATH):
334 |         size = os.path.getsize(KNOWLEDGE_BASE_PATH)
335 |         print(f"✅ Knowledge base found: {size} bytes")
336 |     else:
337 |         print(f"❌ WARNING: Knowledge base not found!")
338 | 
339 |     print("🌐 Endpoints:")
340 |     print("   - http://localhost:8000")
341 |     print("   - http://localhost:8000/health")
342 |     print("   - http://localhost:8000/rag/chat/completions")
343 |     print("=" * 60)
344 | 
345 |     uvicorn.run(app, host="0.0.0.0", port=8000, log_level="info")
346 | 


--------------------------------------------------------------------------------
/emotion_led_service.py:
--------------------------------------------------------------------------------
  1 | # emotion_led_service.py
  2 | """
  3 | Emotion LED Control Service for ReSpeaker
  4 | Receives emotion data and controls LED animations
  5 | """
  6 | 
  7 | from fastapi import FastAPI, HTTPException
  8 | from fastapi.middleware.cors import CORSMiddleware
  9 | from pydantic import BaseModel
 10 | import sys
 11 | import struct
 12 | import time
 13 | import threading
 14 | from typing import Optional
 15 | 
 16 | # Try to import USB libraries
 17 | try:
 18 |     import usb.core
 19 |     import usb.util
 20 | 
 21 |     USB_AVAILABLE = True
 22 | except ImportError:
 23 |     USB_AVAILABLE = False
 24 |     print("⚠️  PyUSB not installed. Install with: pip install pyusb")
 25 | 
 26 | try:
 27 |     import libusb_package
 28 | 
 29 |     LIBUSB_AVAILABLE = True
 30 | except ImportError:
 31 |     LIBUSB_AVAILABLE = False
 32 | 
 33 | app = FastAPI()
 34 | 
 35 | # Enable CORS for frontend
 36 | app.add_middleware(
 37 |     CORSMiddleware,
 38 |     allow_origins=["*"],
 39 |     allow_credentials=True,
 40 |     allow_methods=["*"],
 41 |     allow_headers=["*"],
 42 | )
 43 | 
 44 | # ==========================
 45 | # ReSpeaker LED Control
 46 | # ==========================
 47 | 
 48 | PARAMETERS = {
 49 |     "LED_EFFECT": (20, 12, 1, "rw", "uint8"),
 50 |     "LED_BRIGHTNESS": (20, 13, 1, "rw", "uint8"),
 51 |     "LED_GAMMIFY": (20, 14, 1, "rw", "uint8"),
 52 |     "LED_SPEED": (20, 15, 1, "rw", "uint8"),
 53 |     "LED_COLOR": (20, 16, 1, "rw", "uint32"),
 54 | }
 55 | 
 56 | 
 57 | class ReSpeaker:
 58 |     """ReSpeaker USB device controller"""
 59 |     TIMEOUT = 100000
 60 | 
 61 |     def __init__(self, dev):
 62 |         self.dev = dev
 63 | 
 64 |     def write(self, name, data_list):
 65 |         """Write parameter to device"""
 66 |         if name not in PARAMETERS:
 67 |             raise ValueError(f"Unknown parameter: {name}")
 68 | 
 69 |         data = PARAMETERS[name]
 70 |         if len(data_list) != data[2]:
 71 |             raise ValueError('{} value count is not {}'.format(name, data[2]))
 72 | 
 73 |         windex = data[0]
 74 |         wvalue = data[1]
 75 |         data_cnt = data[2]
 76 |         data_type = data[4]
 77 |         payload = []
 78 | 
 79 |         if data_type == 'uint8':
 80 |             for i in range(data_cnt):
 81 |                 payload += data_list[i].to_bytes(1, byteorder='little')
 82 |         elif data_type == 'uint32':
 83 |             for i in range(data_cnt):
 84 |                 payload += struct.pack(b'I', data_list[i])
 85 | 
 86 |         self.dev.ctrl_transfer(
 87 |             usb.util.CTRL_OUT | usb.util.CTRL_TYPE_VENDOR | usb.util.CTRL_RECIPIENT_DEVICE,
 88 |             0, wvalue, windex, payload, self.TIMEOUT)
 89 | 
 90 |     def close(self):
 91 |         """Release USB device"""
 92 |         try:
 93 |             usb.util.dispose_resources(self.dev)
 94 |         except:
 95 |             pass
 96 | 
 97 | 
 98 | def find_device(vid=0x2886, pid=0x001A):
 99 |     """Find ReSpeaker USB device"""
100 |     if not USB_AVAILABLE:
101 |         return None
102 | 
103 |     try:
104 |         if sys.platform.startswith('win') and LIBUSB_AVAILABLE:
105 |             dev = libusb_package.find(idVendor=vid, idProduct=pid)
106 |         else:
107 |             dev = usb.core.find(idVendor=vid, idProduct=pid)
108 | 
109 |         if not dev:
110 |             return None
111 | 
112 |         return ReSpeaker(dev)
113 |     except Exception as e:
114 |         print(f"Error finding device: {e}")
115 |         return None
116 | 
117 | 
118 | # ==========================
119 | # Emotion to Color Mapping
120 | # ==========================
121 | 
122 | EMOTION_COLORS = {
123 |     "happy": 0xFFFF00,  # Yellow - Bright and cheerful
124 |     "excited": 0xFF00FF,  # Magenta - Vibrant and energetic
125 |     "surprised": 0xFF8800,  # Orange - Bright surprise
126 |     "thinking": 0x00FFFF,  # Cyan - Cool contemplation
127 |     "helpful": 0x00FF00,  # Green - Friendly assistance
128 |     "neutral": 0x8888FF,  # Light blue - Calm neutral
129 |     "sad": 0x0000FF,  # Blue - Somber
130 |     "welcoming": 0xFF69B4,  # Pink - Warm welcome
131 |     "loving": 0xFF1493,  # Deep pink - Affectionate
132 |     "curious": 0x9932CC,  # Purple - Intrigued
133 |     "angry": 0xFF0000,  # Red - Intense
134 |     "sleepy": 0x4B0082,  # Indigo - Drowsy
135 | }
136 | 
137 | # LED animation lock to prevent conflicts
138 | led_lock = threading.Lock()
139 | 
140 | 
141 | def play_emotion_animation(emotion: str, duration: float = 1.0):
142 |     """
143 |     Play LED animation for detected emotion
144 | 
145 |     Args:
146 |         emotion: Emotion name (e.g., "happy", "excited")
147 |         duration: Animation duration in seconds (default: 1.0)
148 | 
149 |     Returns:
150 |         bool: True if successful
151 |     """
152 |     if not USB_AVAILABLE:
153 |         print("⚠️  USB not available")
154 |         return False
155 | 
156 |     with led_lock:  # Ensure only one animation at a time
157 |         dev = find_device()
158 |         if not dev:
159 |             print("❌ ReSpeaker device not found")
160 |             return False
161 | 
162 |         try:
163 |             # Get color for emotion (default to cyan if unknown)
164 |             color = EMOTION_COLORS.get(emotion.lower(), 0x00FFFF)
165 | 
166 |             print(f"🎭 Playing animation for emotion: {emotion} (color: {hex(color)})")
167 | 
168 |             # Set breathing effect with emotion color
169 |             dev.write("LED_GAMMIFY", [1])  # Smooth effect
170 |             dev.write("LED_EFFECT", [1])  # Breath effect (1 = breathing)
171 |             dev.write("LED_COLOR", [color])  # Emotion color
172 |             dev.write("LED_SPEED", [8])  # Speed (1-20, higher = faster)
173 |             dev.write("LED_BRIGHTNESS", [200])  # Bright
174 | 
175 |             # Hold the animation
176 |             time.sleep(duration)
177 | 
178 |             # Return to DoA (Direction of Arrival) mode
179 |             print("🔄 Returning to DoA mode...")
180 |             dev.write("LED_EFFECT", [4])  # DoA mode
181 |             dev.write("LED_BRIGHTNESS", [128])  # Medium brightness
182 | 
183 |             return True
184 | 
185 |         except Exception as e:
186 |             print(f"❌ ReSpeaker LED error: {e}")
187 |             return False
188 |         finally:
189 |             dev.close()
190 | 
191 | 
192 | # ==========================
193 | # API Models
194 | # ==========================
195 | 
196 | class EmotionRequest(BaseModel):
197 |     emotion: str
198 |     duration: Optional[float] = 1.0
199 |     text: Optional[str] = None
200 | 
201 | 
202 | class StatusResponse(BaseModel):
203 |     status: str
204 |     message: str
205 |     usb_available: bool
206 |     device_found: bool
207 | 
208 | 
209 | # ==========================
210 | # API Endpoints
211 | # ==========================
212 | 
213 | @app.get("/")
214 | async def root():
215 |     """Root endpoint with service info"""
216 |     return {
217 |         "service": "Emotion LED Control Service",
218 |         "status": "running",
219 |         "usb_available": USB_AVAILABLE,
220 |         "endpoints": {
221 |             "/emotion": "POST - Trigger emotion LED animation",
222 |             "/test/{color}": "GET - Test LED with specific color",
223 |             "/status": "GET - Check device status",
224 |             "/doa": "POST - Return to DoA mode"
225 |         }
226 |     }
227 | 
228 | 
229 | @app.get("/status")
230 | async def get_status():
231 |     """Check ReSpeaker device status"""
232 |     device_found = False
233 | 
234 |     if USB_AVAILABLE:
235 |         dev = find_device()
236 |         if dev:
237 |             device_found = True
238 |             dev.close()
239 | 
240 |     return StatusResponse(
241 |         status="ok" if device_found else "no_device",
242 |         message="Device ready" if device_found else "Device not found",
243 |         usb_available=USB_AVAILABLE,
244 |         device_found=device_found
245 |     )
246 | 
247 | 
248 | @app.post("/emotion")
249 | async def trigger_emotion(request: EmotionRequest):
250 |     """
251 |     Trigger LED animation for detected emotion
252 | 
253 |     Example:
254 |         POST /emotion
255 |         {
256 |             "emotion": "excited",
257 |             "duration": 1.0,
258 |             "text": "WOW! 40% OFF on phones!"
259 |         }
260 |     """
261 |     if not USB_AVAILABLE:
262 |         raise HTTPException(
263 |             status_code=503,
264 |             detail="USB libraries not available. Install pyusb."
265 |         )
266 | 
267 |     print(f"\n{'=' * 60}")
268 |     print(f"🎭 Emotion Detected: {request.emotion}")
269 |     if request.text:
270 |         print(f"📝 Text: {request.text}")
271 |     print(f"⏱️  Duration: {request.duration}s")
272 |     print(f"{'=' * 60}\n")
273 | 
274 |     # Play animation in background thread to avoid blocking
275 |     thread = threading.Thread(
276 |         target=play_emotion_animation,
277 |         args=(request.emotion, request.duration)
278 |     )
279 |     thread.daemon = True
280 |     thread.start()
281 | 
282 |     return {
283 |         "status": "success",
284 |         "emotion": request.emotion,
285 |         "message": f"Playing {request.emotion} animation"
286 |     }
287 | 
288 | 
289 | @app.post("/doa")
290 | async def return_to_doa():
291 |     """Return LED to DoA (Direction of Arrival) mode"""
292 |     if not USB_AVAILABLE:
293 |         raise HTTPException(status_code=503, detail="USB not available")
294 | 
295 |     with led_lock:
296 |         dev = find_device()
297 |         if not dev:
298 |             raise HTTPException(status_code=404, detail="Device not found")
299 | 
300 |         try:
301 |             dev.write("LED_EFFECT", [4])  # DoA mode
302 |             dev.write("LED_BRIGHTNESS", [128])
303 |             return {"status": "success", "message": "Returned to DoA mode"}
304 |         finally:
305 |             dev.close()
306 | 
307 | 
308 | @app.get("/test/{color}")
309 | async def test_color(color: str):
310 |     """
311 |     Test LED with specific color
312 | 
313 |     Available colors: red, green, blue, yellow, purple, cyan,
314 |                      magenta, orange, pink, white
315 |     """
316 |     color_map = {
317 |         "red": 0xFF0000,
318 |         "green": 0x00FF00,
319 |         "blue": 0x0000FF,
320 |         "yellow": 0xFFFF00,
321 |         "purple": 0x9932CC,
322 |         "cyan": 0x00FFFF,
323 |         "magenta": 0xFF00FF,
324 |         "orange": 0xFF8800,
325 |         "pink": 0xFF69B4,
326 |         "white": 0xFFFFFF,
327 |     }
328 | 
329 |     if color.lower() not in color_map:
330 |         raise HTTPException(
331 |             status_code=400,
332 |             detail=f"Unknown color. Available: {list(color_map.keys())}"
333 |         )
334 | 
335 |     if not USB_AVAILABLE:
336 |         raise HTTPException(status_code=503, detail="USB not available")
337 | 
338 |     with led_lock:
339 |         dev = find_device()
340 |         if not dev:
341 |             raise HTTPException(status_code=404, detail="Device not found")
342 | 
343 |         try:
344 |             color_hex = color_map[color.lower()]
345 |             dev.write("LED_EFFECT", [1])  # Breathing
346 |             dev.write("LED_COLOR", [color_hex])
347 |             dev.write("LED_BRIGHTNESS", [200])
348 |             dev.write("LED_SPEED", [8])
349 | 
350 |             time.sleep(1.0)
351 | 
352 |             dev.write("LED_EFFECT", [4])  # Back to DoA
353 |             dev.write("LED_BRIGHTNESS", [128])
354 | 
355 |             return {
356 |                 "status": "success",
357 |                 "color": color,
358 |                 "hex": hex(color_hex)
359 |             }
360 |         finally:
361 |             dev.close()
362 | 
363 | 
364 | # ==========================
365 | # Startup Check
366 | # ==========================
367 | 
368 | @app.on_event("startup")
369 | async def startup_event():
370 |     """Check device on startup"""
371 |     print("\n" + "=" * 60)
372 |     print("🚀 Emotion LED Control Service Starting...")
373 |     print("=" * 60)
374 | 
375 |     if not USB_AVAILABLE:
376 |         print("⚠️  WARNING: PyUSB not installed!")
377 |         print("   Install with: pip install pyusb")
378 |         print("   LED control will not work without it.")
379 |     else:
380 |         print("✅ PyUSB available")
381 | 
382 |         dev = find_device()
383 |         if dev:
384 |             print("✅ ReSpeaker device found!")
385 |             try:
386 |                 # Set to DoA mode on startup
387 |                 dev.write("LED_EFFECT", [4])
388 |                 dev.write("LED_BRIGHTNESS", [128])
389 |                 print("✅ Device initialized in DoA mode")
390 |             except Exception as e:
391 |                 print(f"⚠️  Device initialization error: {e}")
392 |             finally:
393 |                 dev.close()
394 |         else:
395 |             print("⚠️  ReSpeaker device not found")
396 |             print("   Make sure device is connected")
397 | 
398 |     print("\n🎨 Emotion Color Mapping:")
399 |     for emotion, color in EMOTION_COLORS.items():
400 |         print(f"   {emotion:12s} → {hex(color)}")
401 | 
402 |     print("\n🌐 Service running on http://localhost:5000")
403 |     print("=" * 60 + "\n")
404 | 
405 | 
406 | # ==========================
407 | # Run Server
408 | # ==========================
409 | 
410 | if __name__ == "__main__":
411 |     import uvicorn
412 | 
413 |     print("\n" + "=" * 60)
414 |     print("🎭 Starting Emotion LED Control Service")
415 |     print("=" * 60)
416 |     print("📡 Port: 5000")
417 |     print("🎨 Endpoints:")
418 |     print("   POST /emotion - Trigger emotion animation")
419 |     print("   GET  /status - Check device status")
420 |     print("   POST /doa - Return to DoA mode")
421 |     print("   GET  /test/{color} - Test LED colors")
422 |     print("=" * 60 + "\n")
423 | 
424 |     uvicorn.run(app, host="0.0.0.0", port=5000, log_level="info")


--------------------------------------------------------------------------------
/my_city_info.txt:
--------------------------------------------------------------------------------
  1 | Central City Mall - Complete Guide with Exciting Updates
  2 | Overview
  3 | Welcome to Central City Mall, one of the most vibrant and diverse shopping centers in the region. As your tour guide today, I will help you explore everything this mall has to offer—from food courts and entertainment zones to conference facilities and easy transportation access. Whether you are here to shop, relax, or attend an event, Central City Mall provides a complete experience for visitors of all ages.
  4 | 
  5 | 🎉 SPECIAL OFFERS AND PROMOTIONS (EXCITING!)
  6 | Electronics Mega Sale - Limited Time!
  7 | AMAZING DEALS ON TECH! The electronics store on the second floor is having an incredible sale:
  8 | 
  9 | Mobile phones: Up to 40% OFF on Samsung, iPhone, and OnePlus
 10 | Laptops: Special discounts of 30% on gaming laptops and ultrabooks
 11 | Gaming consoles: PlayStation 5 bundles with 25% discount
 12 | Smart watches and accessories: Buy 1 Get 1 FREE
 13 | Headphones: Premium brands at 50% OFF
 14 | Dates: Valid until end of month
 15 | Location: Electronics Mega Store, Second Floor, East Wing
 16 | 
 17 | Fashion Flash Sale - Unbelievable Prices!
 18 | The fashion boutiques are offering seasonal clearance:
 19 | 
 20 | Designer clothing: 60% OFF on winter collection
 21 | Shoes: Buy 2 pairs, get 1 FREE
 22 | Jewelry: Gold and silver at special festival prices
 23 | Location: Multiple stores, Second Floor
 24 | 
 25 | Food Court Happy Hours - Delicious Deals!
 26 | 
 27 | All restaurants: 20% OFF between 2:00 PM - 4:00 PM daily
 28 | Free dessert with any main course at selected outlets
 29 | Kids eat FREE on weekends (with adult meal purchase)
 30 | 
 31 | 
 32 | ❌ TEMPORARY CLOSURES (Unfortunately)
 33 | Indian Spice Junction - Temporarily Closed
 34 | We're sorry to inform you that Indian Spice Junction on the second floor is temporarily closed for renovations. They are expected to reopen next month with an exciting new menu and modern interior design.
 35 | 
 36 | Alternative: Try the Indian cuisine section at the main food court
 37 | Expected reopening: Early next month
 38 | Location: Second floor, food court area (currently under renovation)
 39 | 
 40 | Kids' Soft Play Area - Maintenance
 41 | The kids' soft play area on the third floor is closed for safety maintenance this week. We apologize for any inconvenience.
 42 | 
 43 | Reopening: Next Monday
 44 | Alternative: Visit the arcade and gaming center nearby
 45 | 
 46 | 
 47 | 😲 SURPRISING & UNIQUE FEATURES (You Won't Believe This!)
 48 | Rooftop Garden - Hidden Gem!
 49 | Did you know? Central City Mall has a SECRET rooftop garden on the fourth floor! Most visitors don't know about this peaceful oasis:
 50 | 
 51 | Beautiful indoor plants and vertical gardens
 52 | Quiet seating areas with comfortable benches
 53 | Natural lighting through skylights
 54 | Perfect for relaxation or phone calls
 55 | Access: Take the elevator to 4th floor, follow signs to "Sky Garden"
 56 | 
 57 | Live Music Performances - Free Entertainment!
 58 | Surprise! Every Friday and Saturday evening (6:00 PM - 8:00 PM), we host FREE live music performances in the atrium:
 59 | 
 60 | Local bands and musicians
 61 | Various genres: jazz, pop, acoustic
 62 | Comfortable seating area provided
 63 | Location: Ground floor atrium, near main entrance
 64 | 
 65 | Art Gallery - Unexpected Cultural Space
 66 | Surprisingly, we have a rotating art gallery on the third floor featuring local artists:
 67 | 
 68 | Free admission
 69 | New exhibitions every month
 70 | Paintings, sculptures, and photography
 71 | Some artworks are available for purchase
 72 | Location: Third floor, next to exhibition center
 73 | 
 74 | Charging Stations Everywhere!
 75 | Amazing feature: Free phone and laptop charging stations on every floor! No need to worry about battery life while shopping.
 76 | Pet-Friendly Zone - Bring Your Furry Friends!
 77 | Unexpected but true! The outdoor seating area and rooftop garden are pet-friendly zones:
 78 | 
 79 | Dogs and cats welcome (on leash)
 80 | Water bowls provided
 81 | Pet waste disposal stations available
 82 | Location: Outdoor terrace, accessible from ground floor
 83 | 
 84 | Book Exchange Corner
 85 | Unique! We have a free community book exchange shelf on the ground floor near the coffee shop. Take a book, leave a book!
 86 | 
 87 | Ground Floor
 88 | Main Entrance
 89 | The main entrance welcomes visitors with wide automatic doors and accessible ramps. Security and information desks are located immediately inside.
 90 | Coffee Breeze Café ☕
 91 | Location: Ground floor, to the left of the main entrance
 92 | Specialty: Premium coffee, pastries, and light breakfast items
 93 | Hours: 7:00 AM - 10:00 PM daily
 94 | Special: Try their signature "Mall Mocha" - customer favorite!
 95 | The café offers a cozy seating area perfect for meetings or casual relaxation.
 96 | Supermarket
 97 | A full-service supermarket is located on the ground floor. It provides groceries, fresh produce, household items, and daily essentials. The supermarket connects to a hallway leading to the subway entrance.
 98 | New: Now stocking international products from Asia, Europe, and the Americas!
 99 | Washrooms
100 | Washrooms are located near the escalators on the ground floor. They are well-maintained, accessible, and include facilities for people with disabilities.
101 | 
102 | Second Floor
103 | Food Court
104 | The second floor hosts the main food court with diverse dining options. Seating areas accommodate large groups and families.
105 | Dragon Wok Restaurant 🥢
106 | Cuisine: Chinese dishes
107 | Specialties: Fried rice, noodles, dim sum, and authentic Chinese cuisine
108 | Special Recommendation: Their Kung Pao Chicken is the BEST in the city!
109 | Location: Food court, second floor
110 | Hours: 10:00 AM - 9:00 PM
111 | Ceylon Spice Kitchen 🌶️
112 | Cuisine: Sri Lankan food
113 | Specialties: Rice and curry, kottu roti, hoppers, and traditional Sri Lankan dishes
114 | Must-try: Authentic kottu roti - incredibly delicious!
115 | Location: Near the main food court area, second floor
116 | Hours: 10:00 AM - 9:00 PM
117 | Other Dining Options
118 | The food court also includes pizza, burgers, Italian cuisine, Japanese sushi, and dessert shops.
119 | New Addition: Korean BBQ restaurant opening soon!
120 | Washrooms
121 | Main washroom facilities are located near the escalators on the second floor. They are larger and include family restrooms.
122 | Shopping - East Wing
123 | The second floor features:
124 | 
125 | Electronics Mega Store - Currently having HUGE SALE! (40% OFF mobile phones!)
126 | Fashion boutiques - Designer brands at discount prices
127 | Shoe stores - Buy 2 Get 1 FREE promotion
128 | Jewelry stores - Festival special prices
129 | Toy stores - Educational toys and games
130 | 
131 | 
132 | Third Floor
133 | Conference Hall 📊
134 | Location: Third floor, next to the exhibition center
135 | Capacity: Seats up to 200 people
136 | Facilities: Audio-visual equipment, Wi-Fi, catering services available
137 | Special: Now with virtual meeting support and live streaming equipment
138 | Booking: Contact mall management at info@centralcitymall.com
139 | Exhibition Center
140 | A flexible space for art exhibitions, product launches, and community events. Adjacent to the conference hall.
141 | Current Exhibition: Local photography showcase (FREE entry!)
142 | Entertainment Zone 🎮
143 | 
144 | Movie Theater with multiple screens and comfortable recliners
145 | Arcade and Gaming Center with latest games and VR experiences
146 | Kids' Play Area with supervised activities (temporarily closed for maintenance)
147 | Bowling Alley - Recently opened! 6 lanes available
148 | 
149 | Washrooms
150 | Washrooms are located near the entertainment zone on the third floor.
151 | 
152 | Fourth Floor (Secret Level!)
153 | Sky Garden - Rooftop Paradise 🌿
154 | Surprise location! Most people don't know this exists:
155 | 
156 | Peaceful indoor garden with natural lighting
157 | Comfortable seating and quiet atmosphere
158 | Perfect for breaks, reading, or phone calls
159 | Free Wi-Fi available
160 | Open during mall hours
161 | 
162 | 
163 | Transportation and Access
164 | Subway Connection 🚇
165 | From the ground floor, walk toward the supermarket. A clearly marked hallway leads directly to the subway entrance. The subway station connects to major city routes.
166 | Convenient: Only 2-minute walk from mall entrance to subway platform!
167 | Parking 🚗
168 | Multi-level parking is available with spaces for over 500 vehicles. Accessible parking spots are near all elevator entrances.
169 | Tip: Parking on basement level 2 is usually less crowded!
170 | Bus Access 🚌
171 | Bus stops are located at the front and rear entrances of the mall. Multiple bus routes serve the mall throughout the day.
172 | Taxi and Ride-Share 🚕
173 | Designated pick-up and drop-off zones are at the main entrance for taxis and ride-share services.
174 | 
175 | Facilities and Services
176 | Information Desk ℹ️
177 | Location: Main entrance on the ground floor
178 | Staff can provide directions, answer questions, and assist with inquiries.
179 | New Service: Digital maps available on tablets!
180 | ATM Machines 💰
181 | ATMs are available on every floor near the escalators.
182 | Free Wi-Fi 📶
183 | The mall offers free Wi-Fi throughout all floors. Connect to "CentralCityMall_Free" network.
184 | Speed: High-speed connection suitable for video calls!
185 | Lost and Found 🔍
186 | Located at the security office on the ground floor. Contact security staff if you lose or find items.
187 | Accessibility ♿
188 | Wheelchair ramps and elevators provide access to all floors. Accessible washrooms are available on every floor. Reserved parking is near all entrances.
189 | Phone & Laptop Charging Stations 🔋
190 | Amazing: Free charging stations on every floor - never run out of battery!
191 | Nursing Room 👶
192 | Private nursing and baby-care room on the second floor near family washrooms.
193 | 
194 | Customer Service
195 | For assistance, contact mall management:
196 | 
197 | Phone: +94 11 234 5678
198 | Email: info@centralcitymall.com
199 | Hours: 9:00 AM - 9:00 PM daily
200 | WhatsApp Support: Available for quick questions
201 | 
202 | 
203 | Frequently Asked Questions
204 | Where is the coffee shop?
205 | The coffee shop, Coffee Breeze Café, is on the ground floor to the left of the main entrance. Try their signature Mall Mocha!
206 | Where can I get Chinese dishes?
207 | On the second floor at the food court—look for Dragon Wok Restaurant. Their Kung Pao Chicken is amazing!
208 | Where can I find Sri Lankan food?
209 | Also on the second floor, at Ceylon Spice Kitchen, near the main food court area. Don't miss their authentic kottu roti!
210 | Is there Indian food available?
211 | Unfortunately, Indian Spice Junction is temporarily closed for renovations. However, Indian cuisine is still available at the main food court section.
212 | Where is the washroom?
213 | Washrooms are on every floor near the escalators. The main one is on the second floor and includes family facilities.
214 | Where is the conference hall?
215 | It is on the third floor, next to the exhibition center. Recently upgraded with virtual meeting equipment!
216 | How do I go to the subway?
217 | From the ground floor, walk toward the supermarket. A marked hallway leads directly to the subway entrance—only 2 minutes away!
218 | What are the special offers right now?
219 | EXCITING! Electronics store has 40% OFF on mobile phones, fashion stores have 60% OFF winter collection, and the food court offers 20% OFF during happy hours (2-4 PM)!
220 | What are some hidden or surprising features of this mall?
221 | WOW! Most people don't know about: (1) The rooftop Sky Garden on the 4th floor, (2) FREE live music every Friday & Saturday evening, (3) Pet-friendly outdoor areas, (4) Free charging stations everywhere, (5) Community book exchange corner!
222 | What are the mall hours?
223 | The mall is open from 9:00 AM to 9:00 PM daily. Individual stores and restaurants may have different hours.
224 | Is there parking available?
225 | Yes, multi-level parking with over 500 spaces is available. Parking fees apply. Tip: Basement level 2 is usually less crowded!
226 | Does the mall have Wi-Fi?
227 | Yes, free HIGH-SPEED Wi-Fi is available throughout the mall. Connect to "CentralCityMall_Free."
228 | Can I book the conference hall?
229 | Yes, contact mall management at info@centralcitymall.com or call +94 11 234 5678 to book.
230 | Are there facilities for people with disabilities?
231 | Yes, wheelchair ramps, elevators, accessible washrooms, and reserved parking are available on all floors.
232 | Can I bring my pet?
233 | Surprisingly, YES! The outdoor terrace and rooftop garden are pet-friendly zones. Dogs and cats welcome (on leash).
234 | 
235 | Shopping Categories
236 | Fashion and Apparel (Currently 60% OFF!)
237 | 
238 | Clothing stores for men, women, and children
239 | Shoe stores (Buy 2 Get 1 FREE!)
240 | Jewelry and accessories (Festival prices!)
241 | Sports and activewear
242 | 
243 | Electronics (MASSIVE SALE - 40% OFF!)
244 | 
245 | Mobile phones and accessories (Samsung, iPhone, OnePlus)
246 | Computers and laptops (Gaming laptops 30% OFF!)
247 | Gaming consoles (PS5 bundles available!)
248 | Smart watches (Buy 1 Get 1 FREE!)
249 | Headphones (50% OFF on premium brands!)
250 | 
251 | Home and Lifestyle
252 | 
253 | Furniture and home décor
254 | Kitchenware
255 | Bedding and bath
256 | Plants and gardening supplies
257 | 
258 | Entertainment
259 | 
260 | Movie theater (with recliner seats!)
261 | Arcade and gaming center (VR experiences!)
262 | Bowling alley (NEW!)
263 | Bookstores
264 | Music and hobby shops
265 | 
266 | 
267 | Events and Promotions
268 | Current Events
269 | 
270 | Live Music: Every Friday & Saturday, 6-8 PM, ground floor atrium (FREE!)
271 | Art Exhibition: Local photography showcase, third floor (FREE entry!)
272 | Electronics Mega Sale: Up to 40% OFF - Limited time only!
273 | Fashion Clearance: 60% OFF winter collection!
274 | 
275 | Upcoming Events
276 | 
277 | Korean BBQ restaurant opening next month (second floor)
278 | Indian Spice Junction reopening with new menu
279 | Kids' play area reopening next Monday
280 | 
281 | 
282 | Safety and Security
283 | Security personnel are on duty 24/7. CCTV cameras monitor all public areas. In case of emergency, follow evacuation signs and instructions from staff. Emergency exits are clearly marked on all floors.
284 | Emergency Contact: Available at all information desks
285 | 
286 | Pro Tips for Visitors
287 | 
288 | Visit the Secret Rooftop Garden on the 4th floor - most people miss this!
289 | Come during Happy Hours (2-4 PM) for 20% OFF at food court
290 | Check out the electronics sale - 40% OFF won't last long!
291 | Free live music Friday & Saturday evenings - great entertainment!
292 | Park on basement level 2 - usually less crowded
293 | Use free charging stations - available on every floor
294 | Bring your pet to the outdoor terrace on nice days
295 | Book exchange corner - free books near the coffee shop!
296 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # 🎭 AI Voice Assistant with Emotion Detection & LED Control
  2 | 
  3 | A real-time conversational AI assistant powered by Agora, with RAG (Retrieval-Augmented Generation), emotion detection, live transcripts, and ReSpeaker LED visualization.
  4 | 
  5 | ![Status](https://img.shields.io/badge/status-active-success.svg)
  6 | ![Python](https://img.shields.io/badge/python-3.7+-blue.svg)
  7 | ![License](https://img.shields.io/badge/license-MIT-blue.svg)
  8 | 
  9 | ## ✨ Features
 10 | 
 11 | - 🎙️ **Real-time Voice Conversation** - Talk naturally with AI assistant
 12 | - 🤖 **RAG-Powered Responses** - Custom knowledge base for accurate answers
 13 | - 🎭 **Emotion Detection** - AI responses include emotional context
 14 | - 💡 **LED Visualization** - reSpeaker lights up with emotion colors
 15 | - 📝 **Live Transcripts** - Real-time conversation transcription
 16 | - 🌐 **Web Interface** - Beautiful, animated UI
 17 | - 🔊 **Voice Synthesis** - Natural-sounding TTS responses
 18 | 
 19 | ![Demo Image](img/img_v1.png)
 20 | 
 21 | ## 📋 Table of Contents
 22 | 
 23 | - [Prerequisites](#prerequisites)
 24 | - [Setup Instructions](#setup-instructions)
 25 |   - [1. Agora Account Setup](#1-agora-account-setup)
 26 |   - [2. Third-Party API Keys](#2-third-party-api-keys)
 27 |   - [3. Generate Tokens](#3-generate-tokens)
 28 |   - [4. Configure Project](#4-configure-project)
 29 |   - [5. Setup RAG Server](#5-setup-rag-server)
 30 |   - [6. Setup LED Control (Optional)](#6-setup-led-control-optional)
 31 | - [Running the Application](#running-the-application)
 32 | - [Project Structure](#project-structure)
 33 | - [Troubleshooting](#troubleshooting)
 34 | - [API Reference](#api-reference)
 35 | 
 36 | ---
 37 | 
 38 | ## 🔧 Prerequisites
 39 | 
 40 | ### Required Hardware
 41 | - Computer with reSpeaker (built-in or external)
 42 | - Internet connection
 43 | 
 44 | 
 45 | ### Required Software
 46 | - Python 3.7 or higher
 47 | - pip (Python package manager)
 48 | - Modern web browser (Chrome, Firefox, Edge)
 49 | 
 50 | ### Required Accounts
 51 | - [Agora Account](https://console.agora.io/)
 52 | - [AssemblyAI Account](https://www.assemblyai.com/) (for speech recognition)
 53 | - [Groq Account](https://groq.com/) (for LLM and TTS)
 54 | 
 55 | ---
 56 | 
 57 | ## 🚀 Setup Instructions
 58 | 
 59 | ### 1. Agora Account Setup
 60 | 
 61 | #### Step 1.1: Create Agora Account
 62 | 1. Go to [Agora Console](https://console.agora.io/)
 63 | 2. Sign up for a free account
 64 | 3. Verify your email
 65 | 
 66 | #### Step 1.2: Create a Project
 67 | 1. In the Agora Console, click **"Create Project"**
 68 | 2. Enter a project name (e.g., "AI Voice Assistant")
 69 | 3. Choose **"Secured mode: APP ID + Token"**
 70 | 4. Click **"Create"**
 71 | 
 72 | #### Step 1.3: Get Your Credentials
 73 | After creating the project, you'll see:
 74 | - **APP ID** - Copy this (looks like: `550749b706214846a1a2eef3612a8cd3`)
 75 | - Click **"Configure"** next to your project
 76 | - Find **"Primary Certificate"** - Copy this
 77 | 
 78 | #### Step 1.4: Get Customer Key & Customer Secret
 79 | 1. In Agora Console, go to [RESTful API](https://console.agora.io/restful)
 80 | 2. Click **"Add a secret"** or view existing secrets
 81 | 3. Copy:
 82 |    - **Customer Key** (looks like: `8a598f4690f740c9a8760a10e28cae9d`)
 83 |    - **Customer Secret** (looks like: `0706c45e30b74b7fa4b3c71eae2c2924`)
 84 | 
 85 | 📚 **Reference**: [Agora RESTful Authentication Guide](https://docs.agora.io/en/conversational-ai/rest-api/restful-authentication)
 86 | 
 87 | ---
 88 | 
 89 | ### 2. Third-Party API Keys
 90 | 
 91 | #### Step 2.1: Get AssemblyAI API Key
 92 | 1. Go to [AssemblyAI](https://www.assemblyai.com/)
 93 | 2. Sign up for a free account
 94 | 3. Go to your [Dashboard](https://www.assemblyai.com/app)
 95 | 4. Copy your **API Key**
 96 | 
 97 | #### Step 2.2: Get Groq API Keys
 98 | 1. Go to [Groq Console](https://console.groq.com/)
 99 | 2. Sign up for a free account
100 | 3. Navigate to [API Keys](https://console.groq.com/keys)
101 | 4. Create **two API keys**:
102 |    - One for LLM (text generation)
103 |    - One for TTS (text-to-speech)
104 | 5. Copy both keys
105 | 
106 | ---
107 | 
108 | ### 3. Generate Tokens
109 | 
110 | #### Step 3.1: Clone Token Generator
111 | ```bash
112 | git clone https://github.com/KasunThushara/RTM_RTC_TokenGenerator.git
113 | cd RTM_RTC_TokenGenerator
114 | ```
115 | 
116 | #### Step 3.2: Configure Token Generator
117 | Edit the token generator configuration with your Agora credentials:
118 | ```python
119 | # In the token generator script
120 | APP_ID = "your_app_id_from_step_1.3"
121 | APP_CERTIFICATE = "your_primary_certificate_from_step_1.3"
122 | ```
123 | 
124 | #### Step 3.3: Generate Token for Agent (UID: 1001)
125 | ```bash
126 | python generate_rtc_rtm_token.py --account 1001
127 | ```
128 | 
129 | **Copy the generated token** - This is for the AI Agent
130 | ```
131 | Token: 007eJxTYHhx+deOGjf+P58sJG4e...
132 | ```
133 | 
134 | #### Step 3.4: Generate Token for User (UID: 1002)
135 | ```bash
136 | python generate_rtc_rtm_token.py --account 1002
137 | ```
138 | 
139 | **Copy the generated token** - This is for the Web User
140 | 
141 | ⚠️ **Important**: Keep both tokens safe. You'll need them in the next steps.
142 | 
143 | ---
144 | 
145 | ### 4. Configure Project
146 | 
147 | #### Step 4.1: Clone This Repository
148 | ```bash
149 | git clone https://github.com/KasunThushara/Agora_Convo_AI_reSpeaker.git
150 | cd ai-voice-assistant
151 | ```
152 | 
153 | #### Step 4.2: Install Python Dependencies
154 | ```bash
155 | pip install -r requirements.txt
156 | ```
157 | 
158 | If you don't have a `requirements.txt`, install manually:
159 | ```bash
160 | pip install fastapi uvicorn requests openai pydantic
161 | ```
162 | 
163 | #### Step 4.3: Configure `config.py`
164 | Create or edit `config.py` with your credentials:
165 | 
166 | ```python
167 | # config.py
168 | # Central configuration file for Agora AI Voice Chat
169 | 
170 | # ==========================
171 | # AGORA CREDENTIALS
172 | # ==========================
173 | CUSTOMER_KEY = "your_customer_key_from_step_1.4"
174 | CUSTOMER_SECRET = "your_customer_secret_from_step_1.4"
175 | APP_ID = "your_app_id_from_step_1.3"
176 | 
177 | # ==========================
178 | # CHANNEL SETTINGS
179 | # ==========================
180 | CHANNEL_NAME = "test"
181 | AGORA_TEMP_TOKEN = "your_agent_token_from_step_3.3_uid_1001"
182 | 
183 | # Agent and User UIDs
184 | AGENT_RTC_UID = "1001"
185 | USER_RTC_UID = "1002"
186 | 
187 | # ==========================
188 | # 3RD PARTY SERVICES
189 | # ==========================
190 | ASSEMBLY_AI_KEY = "your_assemblyai_key_from_step_2.1"
191 | GROQ_KEY = "your_groq_llm_key_from_step_2.2"
192 | TTS_GROQ_KEY = "your_groq_tts_key_from_step_2.2"
193 | 
194 | # ==========================
195 | # AGENT SETTINGS
196 | # ==========================
197 | IDLE_TIMEOUT = 120
198 | MAX_HISTORY = 32
199 | 
200 | SYSTEM_PROMPT = "You are a helpful chatbot."
201 | GREETING_MESSAGE = "Hello, how can I assist you?"
202 | FAILURE_MESSAGE = "Please hold on a second."
203 | 
204 | LLM_MODEL = "llama-3.3-70b-versatile"
205 | TTS_MODEL = "playai-tts"
206 | TTS_VOICE = "Arista-PlayAI"
207 | ASR_LANGUAGE = "en-US"
208 | ```
209 | 
210 | #### Step 4.4: Configure Web Interface
211 | Edit `index_v5.html` (or your HTML file) in **two places**:
212 | 
213 | **Location 1: RTM Login Token (around line 950)**
214 | ```javascript
215 | // Find this line:
216 | await rtmClient.login();
217 | 
218 | // Replace with:
219 | await rtmClient.login({token: 'your_user_token_from_step_3.4_uid_1002'});
220 | ```
221 | 
222 | **Location 2: Configuration Panel Inputs**
223 | Update the default values in the HTML:
224 | ```html
225 | <!-- App ID -->
226 | <input type="text" class="config-input" id="appId" value="your_app_id">
227 | 
228 | <!-- Token -->
229 | <input type="text" class="config-input" id="token" value="your_user_token_uid_1002">
230 | ```
231 | 
232 | ---
233 | 
234 | ### 5. Setup RAG Server
235 | 
236 | #### Step 5.1: Customize Knowledge Base
237 | Edit `my_city_info.txt` with your own information:
238 | 
239 | ```text
240 | # Example: Replace with your use case
241 | Your Company/Location Information
242 | 
243 | Ground Floor
244 | - Main entrance and reception
245 | - Coffee shop location
246 | - Facilities
247 | 
248 | ... (customize with your data)
249 | ```
250 | 
251 | 💡 **Use Cases**:
252 | - Shopping mall guide
253 | - Office building directory
254 | - Museum tour guide
255 | - Hotel concierge
256 | - Campus navigation
257 | 
258 | #### Step 5.2: Test RAG Server Locally
259 | ```bash
260 | python rag_server.py
261 | ```
262 | 
263 | You should see:
264 | ```
265 | 🚀 Starting RAG Server with Emotion Support
266 | ✅ Knowledge base found: X bytes
267 | 🌐 Service running on http://localhost:8000
268 | ```
269 | 
270 | Test it:
271 | ```bash
272 | curl http://localhost:8000/health
273 | ```
274 | 
275 | #### Step 5.3: Setup ngrok (For Cloud Connectivity)
276 | 
277 | **Why ngrok?** Agora's servers need to reach your RAG server. ngrok creates a public URL.
278 | 
279 | 1. **Install ngrok**:
280 |    - Download from [ngrok.com](https://ngrok.com/download)
281 |    - Or: `brew install ngrok` (Mac) / `choco install ngrok` (Windows)
282 | 
283 | 2. **Sign up and authenticate**:
284 |    ```bash
285 |    ngrok config add-authtoken <your-auth-token>
286 |    ```
287 | 
288 | 3. **Start ngrok tunnel**:
289 |    ```bash
290 |    ngrok http 8000
291 |    ```
292 | 
293 | 4. **Copy the public URL**:
294 |    ```
295 |    Forwarding   https://abcd1234.ngrok-free.app -> http://localhost:8000
296 |    ```
297 | 
298 | 5. **Update `join_api.py`**:
299 |    ```python
300 |    RAG_SERVER_URL = "https://your-ngrok-url.ngrok-free.app/rag/chat/completions"
301 |    USE_RAG = True
302 |    ```
303 | 
304 | ⚠️ **Note**: Free ngrok URLs change each restart. Use a static domain with paid plans.
305 | 
306 | ---
307 | 
308 | ### 6. Setup LED Control (Optional)
309 | 
310 | Only needed if you have a reSpeaker USB Microphone.
311 | 
312 | #### Step 6.1: Install USB Libraries
313 | 
314 | **Windows**:
315 | ```bash
316 | pip install pyusb libusb-package
317 | ```
318 | 
319 | **macOS**:
320 | ```bash
321 | brew install libusb
322 | pip install pyusb
323 | ```
324 | 
325 | **Linux**:
326 | ```bash
327 | sudo apt-get install libusb-1.0-0-dev
328 | pip install pyusb
329 | ```
330 | 
331 | #### Step 6.2: Test Device Connection
332 | ```bash
333 | python test_respeaker.py
334 | ```
335 | 
336 | Expected output:
337 | ```
338 | ✅ ReSpeaker device found!
339 |    Vendor ID: 0x2886
340 |    Product ID: 0x001a
341 | ```
342 | 
343 | #### Step 6.3: Linux USB Permissions (if needed)
344 | ```bash
345 | sudo nano /etc/udev/rules.d/99-respeaker.rules
346 | ```
347 | 
348 | Add this line:
349 | ```
350 | SUBSYSTEM=="usb", ATTR{idVendor}=="2886", ATTR{idProduct}=="001a", MODE="0666"
351 | ```
352 | 
353 | Reload rules:
354 | ```bash
355 | sudo udevadm control --reload-rules
356 | sudo udevadm trigger
357 | ```
358 | 
359 | #### Step 6.4: Test LED Service
360 | ```bash
361 | python emotion_led_service.py
362 | ```
363 | 
364 | Test all emotions:
365 | ```bash
366 | python test_led_emotions.py
367 | ```
368 | 
369 | ---
370 | 
371 | ## ▶️ Running the Application
372 | 
373 | ### Terminal Setup
374 | 
375 | You'll need **4 terminals** (or 3 if skipping LED):
376 | 
377 | #### Terminal 1: LED Service (Optional)
378 | ```bash
379 | python emotion_led_service.py
380 | ```
381 | 
382 | Wait for:
383 | ```
384 | ✅ reSpeaker device found!
385 | ✅ Device initialized in DoA mode
386 | 🌐 Service running on http://localhost:5000
387 | ```
388 | 
389 | #### Terminal 2: RAG Server (with ngrok)
390 | ```bash
391 | # Terminal 2a: Start RAG Server
392 | python rag_server.py
393 | ```
394 | 
395 | ```bash
396 | # Terminal 2b: Start ngrok (separate terminal/tab)
397 | ngrok http 8000
398 | ```
399 | 
400 | Copy the ngrok URL and update `join_api.py`.
401 | 
402 | #### Terminal 3: Agora AI Agent
403 | ```bash
404 | python join_api.py
405 | ```
406 | 
407 | Wait for:
408 | ```
409 | ✅ SUCCESS!
410 | Agent ID: A42AA74LL69CF58MN33AE74ME57KJ86K
411 | ⚠️  SAVE THIS AGENT ID FOR STOPPING
412 | ```
413 | 
414 | **⚠️ Important**: Copy the Agent ID - you'll need it to stop the agent later.
415 | 
416 | #### Terminal 4: Open Web Interface
417 | Simply open `index_v5.html` in your web browser.
418 | 
419 | Or use a local server:
420 | ```bash
421 | python -m http.server 8080
422 | # Then visit: http://localhost:8080/index_v5.html
423 | ```
424 | 
425 | ### Using the Application
426 | 
427 | 1. **Click "▶ Start Conversation"**
428 | 2. **Allow microphone access** when prompted
429 | 3. **Start talking!** Try:
430 |    - "Hello!"
431 |    - "Are there any special offers?"
432 |    - "Where is the washroom?"
433 |    - "What are some hidden features?"
434 | 
435 | 4. **Watch the magic happen**:
436 |    - 🎙️ Your speech is transcribed
437 |    - 🤖 AI responds with emotion
438 |    - 📝 Transcripts appear in left panel
439 |    - 🎭 Emoji displays at top
440 |    - 💡 reSpeaker LEDs light up (if connected)
441 | 
442 | ### Stopping the Application
443 | 
444 | 1. **Stop the conversation**: Click "⏹ Stop Conversation" in web UI
445 | 
446 | 2. **Stop the Agora Agent**:
447 |    ```bash
448 |    # Edit stop_api.py with your Agent ID
449 |    AGENT_ID = "your_agent_id_from_terminal_3"
450 |    
451 |    # Then run:
452 |    python stop_api.py
453 |    ```
454 | 
455 | 3. **Stop other services**: Press `Ctrl+C` in each terminal
456 | 
457 | ---
458 | 
459 | ## 📁 Project Structure
460 | 
461 | ```
462 | ai-voice-assistant/
463 | ├── config.py                    # Main configuration file
464 | ├── join_api.py                  # Starts Agora AI agent
465 | ├── stop_api.py                  # Stops Agora AI agent
466 | ├── rag_server.py                # RAG server with emotions
467 | ├── emotion_led_service.py       # LED control service
468 | ├── my_city_info.txt            # Your knowledge base
469 | ├── index.html               # Web interface
470 | ├── agora-rtm-2.2.3.min.js      # Agora RTM SDK
471 | ├── test_respeaker.py           # Device connection test
472 | ├── utils
473 | └── requirements.txt            # Python dependencies
474 | └── index.ts
475 | └── type.ts
476 | ```
477 | 
478 | ---
479 | 
480 | ## 🧪 Testing
481 | 
482 | ### Test RAG Server
483 | ```bash
484 | # Health check
485 | curl http://localhost:8000/health
486 | 
487 | # Test query
488 | curl -X POST http://localhost:8000/rag/chat/completions \
489 |   -H "Content-Type: application/json" \
490 |   -d '{
491 |     "model": "llama-3.3-70b-versatile",
492 |     "messages": [{"role": "user", "content": "Where is the coffee shop?"}],
493 |     "stream": false
494 |   }'
495 | ```
496 | 
497 | ### Test LED Control
498 | ```bash
499 | # Check device status
500 | curl http://localhost:5000/status
501 | 
502 | # Test emotion
503 | curl -X POST http://localhost:5000/emotion \
504 |   -H "Content-Type: application/json" \
505 |   -d '{"emotion": "excited", "duration": 1.0}'
506 | 
507 | # Test color
508 | curl http://localhost:5000/test/yellow
509 | ```
510 | 
511 | ### Test End-to-End
512 | 1. Start all services
513 | 2. Open web interface
514 | 3. Start conversation
515 | 4. Say: "Are there any special offers?"
516 | 5. Verify:
517 |    - ✅ Transcript appears
518 |    - ✅ Emotion emoji shows
519 |    - ✅ LED lights up (if connected)
520 | 
521 | ---
522 | 
523 | ## 🐛 Troubleshooting
524 | 
525 | ### Issue: "Device Not Found"
526 | 
527 | **reSpeaker LED**:
528 | ```bash
529 | # Check device connection
530 | lsusb | grep 2886  # Linux/Mac
531 | # or check Device Manager (Windows)
532 | 
533 | # Verify with test script
534 | python test_respeaker.py
535 | ```
536 | 
537 | ### Issue: "Agent Join Failed"
538 | 
539 | **Check**:
540 | 1. Verify all credentials in `config.py`
541 | 2. Ensure tokens are not expired (regenerate if needed)
542 | 3. Check Agora Console for account status
543 | 4. Verify network connectivity
544 | 
545 | **Debug**:
546 | ```bash
547 | python join_api.py
548 | # Check the error message in output
549 | ```
550 | 
551 | ### Issue: "RAG Server Connection Failed"
552 | 
553 | **Check**:
554 | 1. Is `rag_server.py` running? Check Terminal 2
555 | 2. Is ngrok running? Check the public URL
556 | 3. Did you update `join_api.py` with ngrok URL?
557 | 
558 | **Test**:
559 | ```bash
560 | # Test local
561 | curl http://localhost:8000/health
562 | 
563 | # Test ngrok
564 | curl https://your-ngrok-url.ngrok-free.app/health
565 | ```
566 | 
567 | ### Issue: "No Transcripts Appearing"
568 | 
569 | **Check**:
570 | 1. Open browser console (F12)
571 | 2. Look for RTM connection messages
572 | 3. Verify token in `index_v5.html` (UID 1002)
573 | 4. Check if `enable_rtm: True` in `join_api.py`
574 | 
575 | ### Issue: "Emotions Not Detected"
576 | 
577 | **Check**:
578 | 1. System prompt includes emotion instructions
579 | 2. RAG server has `EMOTION_SYSTEM_PROMPT`
580 | 3. Look for `[emotion]` labels in transcripts
581 | 4. Check browser console for emotion detection logs
582 | 
583 | ### Issue: "Port Already in Use"
584 | 
585 | ```bash
586 | # Find process using port
587 | lsof -i :5000  # LED service
588 | lsof -i :8000  # RAG server
589 | 
590 | # Kill process
591 | kill -9 <PID>
592 | ```
593 | 
594 | ### Issue: "LED Not Responding"
595 | 
596 | 1. Unplug and replug reSpeaker
597 | 2. Restart LED service
598 | 3. Manual reset:
599 |    ```bash
600 |    curl -X POST http://localhost:5000/doa
601 |    ```
602 | 
603 | ---
604 | 
605 | ![Demo Image](img/img_debug.png)
606 | 
607 | ## 🎨 Emotion Color Reference
608 | 
609 | | Emotion | Color | Hex | Use Case |
610 | |---------|-------|-----|----------|
611 | | 😊 happy | Yellow | `0xFFFF00` | Good news, positive responses |
612 | | 🎉 excited | Magenta | `0xFF00FF` | Sales, special offers, amazing deals |
613 | | 😲 surprised | Orange | `0xFF8800` | Unexpected facts, hidden features |
614 | | 🤔 thinking | Cyan | `0x00FFFF` | Processing, searching information |
615 | | 🙋 helpful | Green | `0x00FF00` | Giving directions, assistance |
616 | | 😐 neutral | Light Blue | `0x8888FF` | Standard information, facts |
617 | | 😔 sad | Blue | `0x0000FF` | Apologies, closures, bad news |
618 | | 👋 welcoming | Pink | `0xFF69B4` | Greetings, warm welcomes |
619 | 
620 | ---
621 | 
622 | ## 📊 API Reference
623 | 
624 | ### RAG Server (`http://localhost:8000`)
625 | 
626 | #### `POST /rag/chat/completions`
627 | Generate AI response with RAG
628 | 
629 | **Request**:
630 | ```json
631 | {
632 |   "model": "llama-3.3-70b-versatile",
633 |   "messages": [
634 |     {"role": "user", "content": "Where is the coffee shop?"}
635 |   ],
636 |   "stream": true
637 | }
638 | ```
639 | 
640 | #### `GET /health`
641 | Check server health
642 | 
643 | **Response**:
644 | ```json
645 | {
646 |   "status": "healthy",
647 |   "knowledge_base_loaded": true,
648 |   "knowledge_base_size": 12345
649 | }
650 | ```
651 | 
652 | ### LED Service (`http://localhost:5000`)
653 | 
654 | #### `POST /emotion`
655 | Trigger emotion LED animation
656 | 
657 | **Request**:
658 | ```json
659 | {
660 |   "emotion": "excited",
661 |   "duration": 1.0,
662 |   "text": "Optional transcript"
663 | }
664 | ```
665 | 
666 | #### `GET /status`
667 | Check device status
668 | 
669 | #### `POST /doa`
670 | Return to Direction of Arrival mode
671 | 
672 | #### `GET /test/{color}`
673 | Test specific color (red, green, blue, yellow, etc.)
674 | 
675 | ---
676 | 
677 | ## 🔐 Security Notes
678 | 
679 | ⚠️ **Important Security Considerations**:
680 | 
681 | 1. **Never commit credentials to Git**:
682 |    ```bash
683 |    # Add to .gitignore
684 |    config.py
685 |    .env
686 |    *.key
687 |    ```
688 | 
689 | 2. **Use environment variables**:
690 |    ```python
691 |    import os
692 |    GROQ_KEY = os.getenv('GROQ_API_KEY')
693 |    ```
694 | 
695 | 3. **Rotate tokens regularly**: Agora tokens expire after 24 hours by default
696 | 
697 | 4. **Secure ngrok tunnels**: Use authentication for production
698 | 
699 | 5. **Keep dependencies updated**:
700 |    ```bash
701 |    pip install --upgrade -r requirements.txt
702 |    ```
703 | 
704 | ---
705 | 
706 | ## 🤝 Contributing
707 | 
708 | Contributions are welcome! Please:
709 | 
710 | 1. Fork the repository
711 | 2. Create a feature branch
712 | 3. Make your changes
713 | 4. Submit a pull request
714 | 
715 | ---
716 | 
717 | ## 📄 License
718 | 
719 | This project is licensed under the MIT License.
720 | 
721 | ---
722 | 
723 | ## 🙏 Acknowledgments
724 | 
725 | - [Agora](https://www.agora.io/) - Real-time communication platform
726 | - [Groq](https://groq.com/) - Fast LLM inference
727 | - [AssemblyAI](https://www.assemblyai.com/) - Speech recognition
728 | - [FastAPI](https://fastapi.tiangolo.com/) - Web framework
729 | - [reSpeaker](https://www.seeedstudio.com/) - Smart microphone
730 | 
731 | ---
732 | 
733 | ## 📞 Support
734 | 
735 | If you encounter issues:
736 | 
737 | 1. Check the [Troubleshooting](#troubleshooting) section
738 | 2. Review console logs from all services
739 | 3. Verify all credentials are correct
740 | 4. Check that all services are running
741 | 5. Open an issue on GitHub with:
742 |    - Error messages
743 |    - Steps to reproduce
744 |    - System information
745 | 
746 | ---
747 | 
748 | ## 🎯 Quick Start Checklist
749 | 
750 | - [ ] Agora account created
751 | - [ ] APP_ID and certificates obtained
752 | - [ ] Customer Key & Secret obtained
753 | - [ ] AssemblyAI API key obtained
754 | - [ ] Groq API keys obtained (LLM + TTS)
755 | - [ ] Tokens generated (UID 1001 & 1002)
756 | - [ ] `config.py` configured
757 | - [ ] `index_v5.html` token updated
758 | - [ ] `my_city_info.txt` customized
759 | - [ ] Python dependencies installed
760 | - [ ] ngrok installed and authenticated
761 | - [ ] All services started
762 | - [ ] Web interface opened
763 | - [ ] Test conversation successful
764 | 
765 | ---
766 | 
767 | **Ready to go? Start with [Setup Instructions](#setup-instructions)!** 🚀
768 | 


--------------------------------------------------------------------------------
/type.ts:
--------------------------------------------------------------------------------
  1 | import type {
  2 |   ConnectionDisconnectedReason,
  3 |   ConnectionState,
  4 |   IAgoraRTCRemoteUser,
  5 |   ICameraVideoTrack,
  6 |   IMicrophoneAudioTrack,
  7 |   NetworkQuality,
  8 |   UID
  9 | } from 'agora-rtc-sdk-ng'
 10 | import type { RTMEvents } from 'agora-rtm'
 11 | 
 12 | /**
 13 |  * Transcript modes for the Conversational AI API
 14 |  *
 15 |  * @description
 16 |  * Defines the different modes available for transcript processing and display.
 17 |  * Used to determine how transcription data should be handled and presented.
 18 |  *
 19 |  * @remarks
 20 |  * - All modes are string literals for easy serialization
 21 |  * - Modes determine the granularity of transcript processing
 22 |  * - TEXT should be used as a fallback for unrecognized modes
 23 |  *
 24 |  * Values include:
 25 |  * - TEXT: Plain text mode for simple transcript display
 26 |  * - WORD: Word-level processing with individual word tracking
 27 |  * - CHUNK: Chunk-based processing for grouped text segments
 28 |  * - UNKNOWN: Initial mode for unrecognized transcript types
 29 |  *
 30 |  * @since 1.6.0
 31 |  */
 32 | export enum ETranscriptHelperMode {
 33 |   TEXT = 'text',
 34 |   WORD = 'word',
 35 |   CHUNK = 'chunk',
 36 |   UNKNOWN = 'unknown'
 37 | }
 38 | 
 39 | export enum EMessageType {
 40 |   USER_TRANSCRIPTION = 'user.transcription',
 41 |   AGENT_TRANSCRIPTION = 'assistant.transcription',
 42 |   MSG_INTERRUPTED = 'message.interrupt',
 43 |   MSG_METRICS = 'message.metrics',
 44 |   MSG_ERROR = 'message.error',
 45 |   /** @deprecated */
 46 |   MSG_STATE = 'message.state',
 47 |   IMAGE_UPLOAD = 'image.upload',
 48 |   MESSAGE_INFO = 'message.info',
 49 |   MESSAGE_SAL_STATUS = 'message.sal_status'
 50 | }
 51 | 
 52 | export enum ERTMEvents {
 53 |   MESSAGE = 'message',
 54 |   PRESENCE = 'presence',
 55 |   // TOPIC = 'topic',
 56 |   // STORAGE = 'storage',
 57 |   // LOCK = 'lock',
 58 |   STATUS = 'status'
 59 |   // LINK_STATE = 'linkState',
 60 |   // TOKEN_PRIVILEGE_WILL_EXPIRE = 'tokenPrivilegeWillExpire',
 61 | }
 62 | 
 63 | export enum ERTCEvents {
 64 |   NETWORK_QUALITY = 'network-quality',
 65 |   USER_PUBLISHED = 'user-published',
 66 |   USER_UNPUBLISHED = 'user-unpublished',
 67 |   STREAM_MESSAGE = 'stream-message',
 68 |   USER_JOINED = 'user-joined',
 69 |   USER_LEFT = 'user-left',
 70 |   CONNECTION_STATE_CHANGE = 'connection-state-change',
 71 |   // AUDIO_METADATA = 'audio-metadata',
 72 |   AUDIO_PTS = 'audio-pts'
 73 | }
 74 | 
 75 | export enum ERTCCustomEvents {
 76 |   MICROPHONE_CHANGED = 'microphone-changed',
 77 |   REMOTE_USER_CHANGED = 'remote-user-changed',
 78 |   REMOTE_USER_JOINED = 'remote-user-joined',
 79 |   REMOTE_USER_LEFT = 'remote-user-left',
 80 |   LOCAL_TRACKS_CHANGED = 'local-tracks-changed'
 81 | }
 82 | 
 83 | /**
 84 |  * Event types for the Conversational AI API
 85 |  *
 86 |  * @description
 87 |  * Defines the event types that can be emitted by the Conversational AI API.
 88 |  * Contains events for agent state changes, interruptions, metrics, errors, transcription updates, debug logs, and message receipt updates.
 89 |  *
 90 |  * @remarks
 91 |  * - All events are string literals and can be used with event listeners
 92 |  * - Events are case-sensitive
 93 |  *
 94 |  * Values include:
 95 |  * - AGENT_STATE_CHANGED: Agent state change events
 96 |  * - AGENT_INTERRUPTED: Agent interruption events
 97 |  * - AGENT_METRICS: Agent performance metrics
 98 |  * - AGENT_ERROR: Agent error events
 99 |  * - TRANSCRIPT_UPDATED: Transcription update events
100 |  * - DEBUG_LOG: Debug logging events
101 |  * - MESSAGE_RECEIPT_UPDATED: Message receipt update events
102 |  * - MESSAGE_ERROR: Message error events
103 |  *
104 |  * @since 1.6.0
105 |  */
106 | export enum EConversationalAIAPIEvents {
107 |   AGENT_STATE_CHANGED = 'agent-state-changed',
108 |   AGENT_INTERRUPTED = 'agent-interrupted',
109 |   AGENT_METRICS = 'agent-metrics',
110 |   AGENT_ERROR = 'agent-error',
111 |   TRANSCRIPT_UPDATED = 'transcript-updated',
112 |   DEBUG_LOG = 'debug-log',
113 |   MESSAGE_RECEIPT_UPDATED = 'message-receipt-updated',
114 |   MESSAGE_ERROR = 'message-error',
115 |   MESSAGE_SAL_STATUS = 'message-sal-status'
116 | }
117 | 
118 | /**
119 |  * Module type enumeration for AI capabilities
120 |  *
121 |  * Defines the different types of AI modules available in the system, including language models and text-to-speech
122 |  *
123 |  * @remarks
124 |  * - Each enum value represents a distinct AI capability module
125 |  * - Use these values to specify module type in API calls
126 |  *
127 |  * Values include:
128 |  * - LLM: Language Learning Model
129 |  * - MLLM: Multimodal Language Learning Model
130 |  * - TTS: Text-to-Speech
131 |  * - CONTEXT: Context management module
132 |  * - UNKNOWN: Unknown module type
133 |  *
134 |  * @since 1.6.0
135 |  */
136 | export enum EModuleType {
137 |   LLM = 'llm',
138 |   MLLM = 'mllm',
139 |   TTS = 'tts',
140 |   CONTEXT = 'context',
141 |   UNKNOWN = 'unknown'
142 | }
143 | 
144 | /**
145 |  * Agent metrics statistics data type definition
146 |  *
147 |  * @description
148 |  * Used to store metric data during AI agent runtime, including type, name, value and timestamp
149 |  *
150 |  * @param type - Metric module type {@link EModuleType}
151 |  * @param name - Metric name
152 |  * @param value - Metric value
153 |  * @param timestamp - Data collection timestamp (milliseconds)
154 |  *
155 |  * @since 1.6.0
156 |  */
157 | export type TAgentMetric = {
158 |   type: EModuleType
159 |   name: string
160 |   value: number
161 |   timestamp: number
162 | }
163 | 
164 | /**
165 |  * Message receipt type definition
166 |  *
167 |  * @description
168 |  * Represents a message receipt from the AI module, including type, message content and turn ID
169 |  *
170 |  * @param moduleType - The module type that sent the message {@link EModuleType}
171 |  * @param messageType - The type of the message {@link EChatMessageType}
172 |  * @param message - The content of the message
173 |  * @param turnId - Unique identifier for the conversation turn
174 |  *
175 |  * @since 1.7.0
176 |  */
177 | export type TMessageReceipt = {
178 |   moduleType: EModuleType
179 |   messageType: EChatMessageType
180 |   message: string
181 |   turnId: number
182 | }
183 | 
184 | /**
185 |  * Module error type definition
186 |  *
187 |  * @description
188 |  * Represents error information from different AI modules including error type, code,
189 |  * message and timestamp. Used for error handling and debugging.
190 |  *
191 |  * @remarks
192 |  * - Error codes are module-specific and should be documented by each module
193 |  * - Timestamp is in Unix milliseconds format
194 |  * - Error messages should be human readable and provide actionable information
195 |  *
196 |  * @param type - The module type where error occurred {@link EModuleType}
197 |  * @param code - Error code specific to the module
198 |  * @param message - Human readable error description
199 |  * @param timestamp - Unix timestamp in milliseconds when error occurred
200 |  *
201 |  * @since 1.6.0
202 |  */
203 | export type TModuleError = {
204 |   type: EModuleType
205 |   code: number
206 |   message: string
207 |   timestamp: number
208 | }
209 | 
210 | /**
211 |  * Type definition for state change event
212 |  *
213 |  * Used to describe the information related to voice agent state changes, including current state, turn ID, timestamp and reason
214 |  *
215 |  * @param state Current state of the voice agent. See {@link EAgentState}
216 |  * @param turnID Unique identifier for the current conversation turn
217 |  * @param timestamp Timestamp when the state change occurred (in milliseconds)
218 |  * @param reason Reason description for the state change
219 |  *
220 |  * @since 1.6.0
221 |  *
222 |  * @remarks
223 |  * - State change events are triggered when the voice agent's state changes
224 |  * - timestamp uses UNIX timestamp (in milliseconds)
225 |  */
226 | export type TStateChangeEvent = {
227 |   state: EAgentState
228 |   turnID: number
229 |   timestamp: number
230 |   reason: string
231 | }
232 | 
233 | /**
234 |  * Event handlers interface for the Conversational AI API module.
235 |  *
236 |  * @since 1.6.0
237 |  *
238 |  * Defines a set of event handlers that can be implemented to respond to various
239 |  * events emitted by the Conversational AI system, including agent state changes,
240 |  * interruptions, metrics, errors, and transcription updates.
241 |  *
242 |  * @remarks
243 |  * - All handlers are required to be implemented when using this interface
244 |  * - Events are emitted asynchronously and should be handled accordingly
245 |  * - Event handlers should be lightweight to avoid blocking the event loop
246 |  * - Error handling should be implemented within each handler to prevent crashes
247 |  *
248 |  * @example
249 |  * ```typescript
250 |  * const handlers: IConversationalAIAPIEventHandlers = {
251 |  *   [EConversationalAIAPIEvents.AGENT_STATE_CHANGED]: (agentUserId, event) => {
252 |  *     console.log(`Agent ${agentUserId} state changed:`, event);
253 |  *   },
254 |  *   // ... implement other handlers
255 |  * };
256 |  * ```
257 |  *
258 |  * @param agentUserId - The unique identifier of the AI agent
259 |  * @param event - Event data specific to each event type
260 |  * @param metrics - Performance metrics data for the agent
261 |  * @param error - Error information when agent encounters issues
262 |  * @param transcription - Array of transcription items containing user and agent dialogue
263 |  * @param message - Debug log message string
264 |  * @param messageReceipt - Updated message receipt information
265 |  * @param messageError - Error information related to a specific message
266 |  *
267 |  * @see {@link EConversationalAIAPIEvents} for all available event types
268 |  * @see {@link TStateChangeEvent} for state change event structure
269 |  * @see {@link TAgentMetric} for agent metrics structure
270 |  * @see {@link TModuleError} for error structure
271 |  * @see {@link ITranscriptHelperItem} for transcription item structure
272 |  * @see {@link TMessageReceipt} for message receipt structure
273 |  * @see {@link EChatMessageType} for message type enumeration
274 |  */
275 | export interface IConversationalAIAPIEventHandlers {
276 |   [EConversationalAIAPIEvents.AGENT_STATE_CHANGED]: (
277 |     agentUserId: string,
278 |     event: TStateChangeEvent
279 |   ) => void
280 |   [EConversationalAIAPIEvents.AGENT_INTERRUPTED]: (
281 |     agentUserId: string,
282 |     event: {
283 |       turnID: number
284 |       timestamp: number
285 |     }
286 |   ) => void
287 |   [EConversationalAIAPIEvents.AGENT_METRICS]: (
288 |     agentUserId: string,
289 |     metrics: TAgentMetric
290 |   ) => void
291 |   [EConversationalAIAPIEvents.AGENT_ERROR]: (
292 |     agentUserId: string,
293 |     error: TModuleError
294 |   ) => void
295 |   [EConversationalAIAPIEvents.TRANSCRIPT_UPDATED]: (
296 |     transcription: ITranscriptHelperItem<
297 |       Partial<IUserTranscription | IAgentTranscription>
298 |     >[]
299 |   ) => void
300 |   [EConversationalAIAPIEvents.DEBUG_LOG]: (message: string) => void
301 |   [EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED]: (
302 |     agentUserId: string,
303 |     messageReceipt: TMessageReceipt
304 |   ) => void
305 |   [EConversationalAIAPIEvents.MESSAGE_ERROR]: (
306 |     agentUserId: string,
307 |     error: {
308 |       type: EChatMessageType
309 |       code: number
310 |       message: string
311 |       timestamp: number
312 |     }
313 |   ) => void
314 |   [EConversationalAIAPIEvents.MESSAGE_SAL_STATUS]: (
315 |     agentUserId: string,
316 |     salStatus: IMessageSalStatus
317 |   ) => void
318 | }
319 | 
320 | // export interface IHelperRTMEvents {
321 | //   [ERTMEvents.MESSAGE]: (message: RTMEvents.MessageEvent) => void
322 | //   [ERTMEvents.PRESENCE]: (message: RTMEvents.PresenceEvent) => void
323 | //   [ERTMEvents.STATUS]: (
324 | //     message: RTMEvents.RTMConnectionStatusChangeEvent
325 | //   ) => void
326 | // }
327 | 
328 | export interface IHelperRTCEvents {
329 |   [ERTCEvents.NETWORK_QUALITY]: (quality: NetworkQuality) => void
330 |   [ERTCEvents.USER_PUBLISHED]: (
331 |     user: IAgoraRTCRemoteUser,
332 |     mediaType: 'audio' | 'video'
333 |   ) => void
334 |   [ERTCEvents.USER_UNPUBLISHED]: (
335 |     user: IAgoraRTCRemoteUser,
336 |     mediaType: 'audio' | 'video'
337 |   ) => void
338 |   [ERTCEvents.USER_JOINED]: (user: IAgoraRTCRemoteUser) => void
339 |   [ERTCEvents.USER_LEFT]: (user: IAgoraRTCRemoteUser, reason?: string) => void
340 |   [ERTCEvents.CONNECTION_STATE_CHANGE]: (data: {
341 |     curState: ConnectionState
342 |     revState: ConnectionState
343 |     reason?: ConnectionDisconnectedReason
344 |     channel: string
345 |   }) => void
346 |   // [ERTCEvents.AUDIO_METADATA]: (metadata: Uint8Array) => void @deprecated
347 |   [ERTCEvents.AUDIO_PTS]: (pts: number) => void
348 |   [ERTCEvents.STREAM_MESSAGE]: (uid: UID, stream: Uint8Array) => void
349 | }
350 | 
351 | export class NotFoundError extends Error {
352 |   constructor(message: string) {
353 |     super(message)
354 |     this.name = 'NotFoundError'
355 |   }
356 | }
357 | 
358 | // --- Message ---
359 | export type TDataChunkMessageWord = {
360 |   word: string
361 |   start_ms: number
362 |   duration_ms: number
363 |   stable: boolean
364 | }
365 | 
366 | export type TTranscriptHelperObjectWord = TDataChunkMessageWord & {
367 |   word_status?: ETurnStatus
368 | }
369 | 
370 | export enum ETurnStatus {
371 |   IN_PROGRESS = 0,
372 |   END = 1,
373 |   INTERRUPTED = 2
374 | }
375 | 
376 | /**
377 |  * Agent state enumeration
378 |  *
379 |  * Represents the different states of a conversational AI agent, including idle, listening, thinking, speaking and silent states
380 |  *
381 |  * Detailed Description:
382 |  * This enum is used to track and manage the current state of an AI agent in a conversational system.
383 |  * The states help coordinate the interaction flow between the user and the AI agent.
384 |  *
385 |  * States include:
386 |  * - IDLE: Agent is ready for new interaction
387 |  * - LISTENING: Agent is receiving user input
388 |  * - THINKING: Agent is processing received input
389 |  * - SPEAKING: Agent is delivering response
390 |  * - SILENT: Agent is intentionally not responding
391 |  *
392 |  * @remarks
393 |  * - State transitions should be handled properly to avoid deadlocks
394 |  * - The SILENT state is different from IDLE as it represents an intentional non-response
395 |  *
396 |  * @since 1.6.0
397 |  */
398 | export enum EAgentState {
399 |   IDLE = 'idle',
400 |   LISTENING = 'listening',
401 |   THINKING = 'thinking',
402 |   SPEAKING = 'speaking',
403 |   SILENT = 'silent'
404 | }
405 | 
406 | export interface ITranscriptionBase {
407 |   object: EMessageType
408 |   text: string
409 |   start_ms: number
410 |   duration_ms: number
411 |   language: string
412 |   turn_id: number
413 |   stream_id: number
414 |   user_id: string
415 |   words: TDataChunkMessageWord[] | null
416 | }
417 | 
418 | export interface IUserTranscription extends ITranscriptionBase {
419 |   object: EMessageType.USER_TRANSCRIPTION // "user.transcription"
420 |   final: boolean
421 | }
422 | 
423 | export interface IAgentTranscription extends ITranscriptionBase {
424 |   object: EMessageType.AGENT_TRANSCRIPTION // "assistant.transcription"
425 |   quiet: boolean
426 |   turn_seq_id: number
427 |   turn_status: ETurnStatus
428 | }
429 | 
430 | export interface IMessageInterrupt {
431 |   object: EMessageType.MSG_INTERRUPTED // "message.interrupt"
432 |   message_id: string
433 |   data_type: 'message'
434 |   turn_id: number
435 |   start_ms: number
436 |   send_ts: number
437 | }
438 | 
439 | export interface IMessageMetrics {
440 |   object: EMessageType.MSG_METRICS // "message.metrics"
441 |   module: EModuleType
442 |   metric_name: string
443 |   turn_id: number
444 |   latency_ms: number
445 |   send_ts: number
446 | }
447 | 
448 | export interface IMessageError {
449 |   object: EMessageType.MSG_ERROR // "message.error"
450 |   module: EModuleType
451 |   code: number
452 |   message: string
453 |   turn_id: number
454 |   send_ts: number
455 |   [x: string]: unknown // Allow additional properties
456 | }
457 | 
458 | export interface IPresenceState
459 |   extends Omit<RTMEvents.PresenceEvent, 'stateChanged'> {
460 |   stateChanged: {
461 |     state: EAgentState
462 |     turn_id: string
463 |   }
464 | }
465 | 
466 | export type TQueueItem = {
467 |   turn_id: number
468 |   text: string
469 |   words: TTranscriptHelperObjectWord[]
470 |   status: ETurnStatus
471 |   stream_id: number
472 |   uid: string
473 | }
474 | 
475 | /**
476 |  * Interface for transcript helper item
477 |  *
478 |  * Defines the data structure for a single transcript item in the transcript system. Contains basic transcript information such as user ID, stream ID, turn ID, timestamp, text content, status, and metadata.
479 |  *
480 |  * @remarks
481 |  * - This interface supports generics, allowing different types of metadata as needed
482 |  * - Status value must be a valid value defined in {@link ETurnStatus}
483 |  *
484 |  * @param T - Type of metadata
485 |  * @param uid - Unique identifier for the user
486 |  * @param stream_id - Stream identifier
487 |  * @param turn_id - Turn identifier in the conversation
488 |  * @param _time - Timestamp of the transcript (in milliseconds)
489 |  * @param text - Transcript text content
490 |  * @param status - Current status of the transcript item
491 |  * @param metadata - Additional metadata information
492 |  *
493 |  * @since 1.6.0
494 |  */
495 | export interface ITranscriptHelperItem<T> {
496 |   uid: string
497 |   stream_id: number
498 |   turn_id: number
499 |   _time: number
500 |   text: string
501 |   status: ETurnStatus
502 |   metadata: T | null
503 | }
504 | 
505 | // --- rtc ---
506 | export interface IUserTracks {
507 |   videoTrack?: ICameraVideoTrack
508 |   audioTrack?: IMicrophoneAudioTrack
509 | }
510 | 
511 | // --- rtm ---
512 | 
513 | /**
514 |  * Enumeration defining chat message priority levels for handling message processing.
515 |  *
516 |  * Specifies how incoming chat messages should be handled when they arrive during
517 |  * ongoing conversation processing, providing control over message queue behavior
518 |  * and user experience during real-time interactions.
519 |  *
520 |  * @remarks
521 |  * Values include:
522 |  * - INTERRUPTED: Interrupt current processing and handle immediately
523 |  * - APPEND: Add to processing queue for sequential handling
524 |  * - IGNORE: Discard the message without processing
525 |  *
526 |  * @enum {string}
527 |  *
528 |  * @since 1.7.0
529 |  */
530 | export enum EChatMessagePriority {
531 |   INTERRUPTED = 'interrupted',
532 |   APPEND = 'append',
533 |   IGNORE = 'ignore'
534 | }
535 | 
536 | /**
537 |  * Enumeration defining the different types of chat messages supported in the conversational AI system.
538 |  *
539 |  * @remarks
540 |  * Values include:
541 |  * - TEXT: Text-based message
542 |  * - IMAGE: Image-based message
543 |  * - UNKNOWN: Unknown message type
544 |  *
545 |  * @enum {string}
546 |  *
547 |  * @since 1.7.0
548 |  */
549 | export enum EChatMessageType {
550 |   TEXT = 'text',
551 |   IMAGE = 'image',
552 |   UNKNOWN = 'unknown'
553 | }
554 | 
555 | /**
556 |  * Base interface for chat messages containing the fundamental message type property.
557 |  * This interface serves as the foundation for all chat message types in the system.
558 |  *
559 |  * @since 1.7.0
560 |  */
561 | export interface IChatMessageBase {
562 |   messageType: EChatMessageType
563 | }
564 | 
565 | /**
566 |  * Represents a text-based chat message with priority and interruption settings.
567 |  *
568 |  * @interface IChatMessageText
569 |  * @extends IChatMessageBase
570 |  *
571 |  * @property messageType - The type of message, must be TEXT
572 |  * @property priority - The priority level of the chat message
573 |  * @property responseInterruptable - Whether the response can be interrupted
574 |  * @property text - The optional text content of the message
575 |  *
576 |  * @since 1.7.0
577 |  */
578 | export interface IChatMessageText extends IChatMessageBase {
579 |   messageType: EChatMessageType.TEXT
580 |   priority: EChatMessagePriority
581 |   responseInterruptable: boolean
582 |   text?: string
583 | }
584 | 
585 | /**
586 |  * Represents an image-based chat message that can contain either a URL or base64 encoded image data.
587 |  *
588 |  * @interface IChatMessageImage
589 |  * @extends IChatMessageBase
590 |  *
591 |  * @property messageType - The type of message, must be IMAGE
592 |  * @property uuid - Unique identifier for the image message
593 |  * @property url - Optional URL pointing to the image resource
594 |  * @property base64 - Optional base64 encoded image data
595 |  */
596 | export interface IChatMessageImage extends IChatMessageBase {
597 |   messageType: EChatMessageType.IMAGE
598 |   uuid: string
599 |   url?: string
600 |   base64?: string
601 | }
602 | 
603 | // --- local ---
604 | export enum ELocalTranscriptStatus {
605 |   PENDING = 'pending',
606 |   SENT = 'sent',
607 |   FAILED = 'failed'
608 | }
609 | 
610 | export interface ILocalTranscriptionBase {
611 |   id: string
612 |   uid: string
613 |   _time: number
614 |   status: ELocalTranscriptStatus
615 | }
616 | 
617 | export interface ILocalImageTranscription extends ILocalTranscriptionBase {
618 |   localImage: File
619 |   imageDimensions: {
620 |     width: number
621 |     height: number
622 |   }
623 |   image_url?: string
624 | }
625 | 
626 | export enum EMessageSalStatus {
627 |   VP_DISABLED = 'VP_DISABLED',
628 |   VP_UNREGISTER = 'VP_UNREGISTER',
629 |   VP_REGISTERING = 'VP_REGISTERING',
630 |   VP_REGISTER_SUCCESS = 'VP_REGISTER_SUCCESS',
631 |   VP_REGISTER_FAIL = 'VP_REGISTER_FAIL',
632 |   VP_REGISTER_DUPLICATE = 'VP_REGISTER_DUPLICATE'
633 | }
634 | export interface IMessageSalStatus {
635 |   object: EMessageType.MESSAGE_SAL_STATUS // "message.sal_status"
636 |   status: EMessageSalStatus
637 |   timestamp: number
638 |   data_type: string
639 |   message_id: string
640 |   send_ts: number
641 | }


--------------------------------------------------------------------------------
/index.ts:
--------------------------------------------------------------------------------
  1 | import { logger as agoraLogger } from '@agora-js/report'
  2 | import type { IAgoraRTCClient } from 'agora-rtc-sdk-ng'
  3 | import type { ChannelType, RTMClient, RTMEvents } from 'agora-rtm'
  4 | 
  5 | import {
  6 |   type EAgentState,
  7 |   EChatMessagePriority,
  8 |   EChatMessageType,
  9 |   EConversationalAIAPIEvents,
 10 |   EMessageType,
 11 |   ERTCEvents,
 12 |   ERTMEvents,
 13 |   ETranscriptHelperMode,
 14 |   type IAgentTranscription,
 15 |   type IChatMessageImage,
 16 |   type IChatMessageText,
 17 |   type IConversationalAIAPIEventHandlers,
 18 |   type IMessageSalStatus,
 19 |   type ITranscriptHelperItem,
 20 |   type IUserTranscription,
 21 |   NotFoundError,
 22 |   type TAgentMetric,
 23 |   type TMessageReceipt,
 24 |   type TModuleError,
 25 |   type TStateChangeEvent
 26 | } from '@/conversational-ai-api/type'
 27 | import { factoryFormatLog } from '@/conversational-ai-api/utils'
 28 | import { EventHelper } from '@/conversational-ai-api/utils/event'
 29 | import { CovSubRenderController } from '@/conversational-ai-api/utils/sub-render'
 30 | import { ELoggerType, logger } from '@/lib/logger'
 31 | import { genTranceID } from '@/lib/utils'
 32 | 
 33 | const TAG = 'ConversationalAIAPI'
 34 | // const CONSOLE_LOG_PREFIX = `[${TAG}]`
 35 | const VERSION = '1.8.0'
 36 | 
 37 | const formatLog = factoryFormatLog({ tag: TAG })
 38 | 
 39 | export interface IConversationalAIAPIConfig {
 40 |   rtcEngine: IAgoraRTCClient
 41 |   rtmEngine: RTMClient
 42 |   renderMode?: ETranscriptHelperMode
 43 |   enableLog?: boolean
 44 | }
 45 | 
 46 | /**
 47 |  * A class that manages conversational AI interactions through Agora's RTC and RTM services.
 48 |  *
 49 |  * Provides functionality to handle real-time communication between users and AI agents,
 50 |  * including message processing, state management, and event handling. It integrates with
 51 |  * Agora's RTC client for audio streaming and RTM client for messaging.
 52 |  *
 53 |  * Key features
 54 |  * - Singleton instance management
 55 |  * - RTC and RTM event handling
 56 |  * - Chat history and transcription management
 57 |  * - Agent state monitoring
 58 |  * - Debug logging
 59 |  * - Event subscription and management through EventHelper
 60 |  *
 61 |  * @remarks
 62 |  * - Must be initialized with {@link IConversationalAIAPIConfig} before use
 63 |  * - Only one instance can exist at a time
 64 |  * - Requires both RTC and RTM engines to be properly configured
 65 |  * - Events are emitted for state changes, transcriptions, and errors
 66 |  * - Extends EventHelper to provide event subscription capabilities
 67 |  *
 68 |  * @example
 69 |  * Basic initialization and usage:
 70 |  * ```typescript
 71 |  * const api = ConversationalAIAPI.init({
 72 |  *   rtcEngine: rtcClient,
 73 |  *   rtmEngine: rtmClient,
 74 |  *   renderMode: ETranscriptHelperMode.REALTIME
 75 |  * });
 76 |  *
 77 |  * // Subscribe to a channel
 78 |  * api.subscribeMessage('channel-id');
 79 |  * ```
 80 |  *
 81 |  * @example
 82 |  * Event handling with EventHelper methods:
 83 |  * ```typescript
 84 |  * const conversationalAIAPI = ConversationalAIAPI.getInstance();
 85 |  *
 86 |  * // Subscribe to all events using on() method
 87 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.AGENT_STATE_CHANGED, (agentUserId, event) => {
 88 |  *   console.log(`Agent ${agentUserId} state changed to:`, event.state);
 89 |  * });
 90 |  *
 91 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.TRANSCRIPT_UPDATED, (transcription) => {
 92 |  *   console.log('Transcription updated:', transcription);
 93 |  * });
 94 |  *
 95 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.AGENT_INTERRUPTED, (agentUserId, event) => {
 96 |  *   console.log(`Agent ${agentUserId} interrupted:`, event);
 97 |  * });
 98 |  *
 99 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.AGENT_METRICS, (agentUserId, metrics) => {
100 |  *   console.log(`Agent ${agentUserId} metrics:`, metrics);
101 |  * });
102 |  *
103 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.AGENT_ERROR, (agentUserId, error) => {
104 |  *   console.error(`Agent ${agentUserId} error:`, error.message);
105 |  * });
106 |  *
107 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.DEBUG_LOG, (message) => {
108 |  *   console.debug('Debug log:', message);
109 |  * });
110 |  *
111 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED, (agentUserId, messageReceipt) => {
112 |  *  console.log(`Message receipt updated for agent ${agentUserId}:`, messageReceipt);
113 |  * });
114 |  *
115 |  * conversationalAIAPI.on(EConversationalAIAPIEvents.MESSAGE_ERROR, (agentUserId, error) => {
116 |  *  console.error(`Message error for agent ${agentUserId}:`, error);
117 |  * });
118 |  *
119 |  * // Unsubscribe from events using off() method
120 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.AGENT_STATE_CHANGED, handleAgentStateChanged);
121 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.TRANSCRIPT_UPDATED, handleTranscriptionUpdated);
122 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.AGENT_INTERRUPTED, handleAgentInterrupted);
123 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.AGENT_METRICS, handleAgentMetrics);
124 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.AGENT_ERROR, handleAgentError);
125 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.DEBUG_LOG, handleDebugLog);
126 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED, handleMessageReceiptUpdated);
127 |  * conversationalAIAPI.off(EConversationalAIAPIEvents.MESSAGE_ERROR, handleMessageError);
128 |  * ```
129 |  *
130 |  * @fires {@link EConversationalAIAPIEvents.TRANSCRIPT_UPDATED} When chat history is updated
131 |  * @fires {@link EConversationalAIAPIEvents.AGENT_STATE_CHANGED} When agent state changes
132 |  * @fires {@link EConversationalAIAPIEvents.AGENT_INTERRUPTED} When agent is interrupted
133 |  * @fires {@link EConversationalAIAPIEvents.AGENT_METRICS} When agent metrics are received
134 |  * @fires {@link EConversationalAIAPIEvents.AGENT_ERROR} When an error occurs
135 |  * @fires {@link EConversationalAIAPIEvents.DEBUG_LOG} When debug logs are generated
136 |  * @fires {@link EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED} When message receipt is updated
137 |  * @fires {@link EConversationalAIAPIEvents.MESSAGE_ERROR} When message error occurs
138 |  *
139 |  * @since 1.7.0
140 |  */
141 | export class ConversationalAIAPI extends EventHelper<IConversationalAIAPIEventHandlers> {
142 |   private static NAME = TAG
143 |   private static VERSION = VERSION
144 |   private static _instance: ConversationalAIAPI | null = null
145 |   private callMessagePrint: (type: ELoggerType, ...args: unknown[]) => void
146 | 
147 |   protected rtcEngine: IAgoraRTCClient | null = null
148 |   protected rtmEngine: RTMClient | null = null
149 |   protected renderMode: ETranscriptHelperMode = ETranscriptHelperMode.UNKNOWN
150 |   protected channel: string | null = null
151 |   protected covSubRenderController: CovSubRenderController
152 |   protected enableLog: boolean = false
153 | 
154 |   constructor() {
155 |     super()
156 | 
157 |     this.callMessagePrint = (
158 |       type: ELoggerType = ELoggerType.debug,
159 |       ...args: unknown[]
160 |     ) => {
161 |       if (!this.enableLog) {
162 |         return
163 |       }
164 |       logger[type](formatLog(...args))
165 |       this.onDebugLog?.(`[${type}] ${formatLog(...args)}`)
166 |       agoraLogger[type](...args)
167 |     }
168 |     this.callMessagePrint(
169 |       ELoggerType.debug,
170 |       `${ConversationalAIAPI.NAME} initialized, version: ${ConversationalAIAPI.VERSION}`
171 |     )
172 | 
173 |     this.covSubRenderController = new CovSubRenderController({
174 |       onChatHistoryUpdated: this.onChatHistoryUpdated.bind(this),
175 |       onAgentStateChanged: this.onAgentStateChanged.bind(this),
176 |       onAgentInterrupted: this.onAgentInterrupted.bind(this),
177 |       onDebugLog: this.onDebugLog.bind(this),
178 |       onAgentMetrics: this.onAgentMetrics.bind(this),
179 |       onAgentError: this.onAgentError.bind(this),
180 |       onMessageReceipt: this.onMessageReceiptUpdated.bind(this),
181 |       onMessageError: this.onMessageError.bind(this),
182 |       onMessageSalStatus: this.onMessageSalStatus.bind(this)
183 |     })
184 |   }
185 | 
186 |   /**
187 |    * Gets the singleton instance of ConversationalAIAPI.
188 |    *
189 |    * Retrieves the singleton instance of the ConversationalAIAPI class. This method
190 |    * ensures that only one instance of ConversationalAIAPI exists throughout the
191 |    * application lifecycle.
192 |    *
193 |    * @remarks
194 |    * - Must call {@link init} before using this method
195 |    * - Throws error if instance is not initialized
196 |    *
197 |    * @returns The singleton instance of ConversationalAIAPI
198 |    * @throws {@link NotFoundError} When ConversationalAIAPI is not initialized
199 |    * @since 1.6.0
200 |    */
201 |   public static getInstance() {
202 |     if (!ConversationalAIAPI._instance) {
203 |       throw new NotFoundError('ConversationalAIAPI is not initialized')
204 |     }
205 |     return ConversationalAIAPI._instance
206 |   }
207 | 
208 |   public getCfg() {
209 |     if (!this.rtcEngine || !this.rtmEngine) {
210 |       throw new NotFoundError('ConversationalAIAPI is not initialized')
211 |     }
212 |     return {
213 |       rtcEngine: this.rtcEngine,
214 |       rtmEngine: this.rtmEngine,
215 |       renderMode: this.renderMode,
216 |       channel: this.channel,
217 |       enableLog: this.enableLog
218 |     }
219 |   }
220 | 
221 |   /**
222 |    * Initializes the ConversationalAIAPI singleton instance.
223 |    *
224 |    * This method sets up the RTC and RTM engines, render mode, and logging options.
225 |    * It must be called before any other methods of ConversationalAIAPI can be used.
226 |    *
227 |    * @remarks
228 |    * - Only one instance can be initialized at a time
229 |    * - Throws error if already initialized
230 |    *
231 |    * @param cfg - Configuration object for initializing the API
232 |    * @returns The initialized instance of ConversationalAIAPI
233 |    * @throws {@link Error} If ConversationalAIAPI is already initialized
234 |    * @since 1.6.0
235 |    */
236 |   public static init(cfg: IConversationalAIAPIConfig) {
237 |     if (!ConversationalAIAPI._instance) {
238 |       ConversationalAIAPI._instance = new ConversationalAIAPI()
239 |     }
240 |     ConversationalAIAPI._instance.rtcEngine = cfg.rtcEngine
241 |     ConversationalAIAPI._instance.rtmEngine = cfg.rtmEngine
242 |     ConversationalAIAPI._instance.renderMode =
243 |       cfg.renderMode ?? ETranscriptHelperMode.UNKNOWN
244 |     ConversationalAIAPI._instance.enableLog = cfg.enableLog ?? false
245 | 
246 |     return ConversationalAIAPI._instance
247 |   }
248 | 
249 |   /**
250 |    * Subscribes to a message channel for real-time updates.
251 |    *
252 |    * This method binds the necessary RTC and RTM events, sets the channel,
253 |    * and starts the CovSubRenderController to handle incoming messages.
254 |    *
255 |    * @remarks
256 |    * - Must call {@link init} before using this method
257 |    * - Throws error if not initialized
258 |    *
259 |    * @param channel - The channel to subscribe to for messages
260 |    * @since 1.6.0
261 |    */
262 |   public subscribeMessage(channel: string) {
263 |     this.bindRtcEvents()
264 |     this.bindRtmEvents()
265 | 
266 |     this.channel = channel
267 |     this.covSubRenderController.setMode(this.renderMode)
268 |     this.covSubRenderController.run()
269 |   }
270 | 
271 |   /**
272 |    * Unsubscribes from the message channel and cleans up resources.
273 |    *
274 |    * This method unbinds the RTC and RTM events, clears the channel,
275 |    * and cleans up the CovSubRenderController.
276 |    *
277 |    * @remarks
278 |    * - Must call {@link subscribeMessage} before using this method
279 |    * - Throws error if not initialized
280 |    *
281 |    * @since 1.6.0
282 |    */
283 |   public unsubscribe() {
284 |     this.unbindRtcEvents()
285 |     this.unbindRtmEvents()
286 | 
287 |     this.channel = null
288 |     this.covSubRenderController.cleanup()
289 |   }
290 | 
291 |   /**
292 |    * Destroys the ConversationalAIAPI instance and cleans up resources.
293 |    *
294 |    * This method unbinds all RTC and RTM events, clears the channel,
295 |    * and cleans up the CovSubRenderController.
296 |    *
297 |    * @remarks
298 |    * - Must call {@link unsubscribe} before using this method
299 |    * - Throws error if not initialized
300 |    *
301 |    * @since 1.6.0
302 |    */
303 |   public destroy() {
304 |     const instance = ConversationalAIAPI.getInstance()
305 |     if (instance) {
306 |       instance?.rtcEngine?.removeAllListeners()
307 |       instance.rtcEngine = null
308 |       instance?.rtmEngine?.removeAllListeners()
309 |       instance.rtmEngine = null
310 |       instance.renderMode = ETranscriptHelperMode.UNKNOWN
311 |       instance.channel = null
312 |       instance.removeAllEventListeners()
313 |       ConversationalAIAPI._instance = null
314 |     }
315 |     this.callMessagePrint(
316 |       ELoggerType.debug,
317 |       `${ConversationalAIAPI.NAME} destroyed`
318 |     )
319 |   }
320 | 
321 |   /**
322 |    * Sends a chat message to the conversational AI agent.
323 |    *
324 |    * @param agentUserId - The unique identifier of the agent user
325 |    * @param message - The chat message to send, can be either text or image type
326 |    * @returns A promise that resolves with the result of sending the message
327 |    * @throws {Error} When an unsupported chat message type is provided
328 |    *
329 |    * @since 1.7.0
330 |    *
331 |    * @example
332 |    * ```typescript
333 |    * // Send a text message
334 |    * const textMessage: IChatMessageText = {
335 |    *   messageType: EChatMessageType.TEXT,
336 |    *   priority: EChatMessagePriority.HIGH,
337 |    *   responseInterruptable: true,
338 |    *   text: "Hello, how are you?"
339 |    * };
340 |    * await api.chat("user123", textMessage);
341 |    *
342 |    * // Send an image message
343 |    * const imageMessage: IChatMessageImage = {
344 |    *   messageType: EChatMessageType.IMAGE,
345 |    *   uuid: "msg-456",
346 |    *   url: "https://example.com/image.jpg"
347 |    * };
348 |    * await api.chat("user123", imageMessage);
349 |    * ```
350 |    */
351 |   public async chat(
352 |     agentUserId: string,
353 |     message: IChatMessageText | IChatMessageImage
354 |   ) {
355 |     switch (message.messageType) {
356 |       case EChatMessageType.TEXT:
357 |         return this.sendText(agentUserId, message as IChatMessageText)
358 |       case EChatMessageType.IMAGE:
359 |         return this.sendImage(agentUserId, message as IChatMessageImage)
360 |       default:
361 |         throw new Error('Unsupported chat message type')
362 |     }
363 |   }
364 | 
365 |   /**
366 |    * Sends a text message to the specified agent user through RTM engine.
367 |    *
368 |    * @param agentUserId - The unique identifier of the agent user to send the message to
369 |    * @param message - The chat message object containing text content and optional settings
370 |    * @param message.priority - Optional priority level for the message (defaults to INTERRUPTED)
371 |    * @param message.responseInterruptable - Optional flag indicating if the response can be interrupted (defaults to true)
372 |    * @param message.text - The actual text content of the message
373 |    *
374 |    * @returns Promise that resolves when the message is successfully sent
375 |    *
376 |    * @throws {Error} Throws an error with message "failed to send chat message" if the RTM publish operation fails
377 |    *
378 |    * @since 1.7.0
379 |    *
380 |    * @example
381 |    * ```typescript
382 |    * await api.sendText('user123', {
383 |    *   text: 'Hello, how can I help you?',
384 |    *   priority: EChatMessagePriority.HIGH,
385 |    *   responseInterruptable: false
386 |    * });
387 |    * ```
388 |    */
389 |   public async sendText(agentUserId: string, message: IChatMessageText) {
390 |     const traceId = genTranceID()
391 |     this.callMessagePrint(
392 |       ELoggerType.debug,
393 |       `>>> [trancID:${traceId}] [chat] ${agentUserId}`,
394 |       message
395 |     )
396 | 
397 |     const { rtmEngine } = this.getCfg()
398 | 
399 |     const payload = {
400 |       priority: message.priority ?? EChatMessagePriority.INTERRUPTED,
401 |       interruptable: message.responseInterruptable ?? true,
402 |       message: message.text ?? ''
403 |     }
404 | 
405 |     try {
406 |       const payloadStr = JSON.stringify(payload)
407 |       const options = {
408 |         channelType: 'USER' as ChannelType,
409 |         customType: EMessageType.USER_TRANSCRIPTION
410 |       }
411 | 
412 |       this.callMessagePrint(
413 |         ELoggerType.debug,
414 |         `msg: [traceID: ${traceId}] rtm publish`,
415 |         payloadStr
416 |       )
417 | 
418 |       const result = await rtmEngine.publish(agentUserId, payloadStr, options)
419 | 
420 |       this.callMessagePrint(
421 |         ELoggerType.debug,
422 |         `>>> [trancID:${traceId}] [chat]`,
423 |         'sucessfully sent chat message',
424 |         result
425 |       )
426 |     } catch (error: unknown) {
427 |       this.callMessagePrint(
428 |         ELoggerType.error,
429 |         `>>> [trancID:${traceId}] [chat]`,
430 |         'failed to send chat message',
431 |         error
432 |       )
433 |       throw new Error('failed to send chat message')
434 |     }
435 |   }
436 | 
437 |   /**
438 |    * Sends an image message to a specific agent user through RTM (Real-Time Messaging).
439 |    *
440 |    * @param agentUserId - The unique identifier of the agent user to send the image to
441 |    * @param message - The image message object containing UUID and either URL or base64 data
442 |    * @param message.uuid - Unique identifier for the message
443 |    * @param message.url - Optional URL of the image to send
444 |    * @param message.base64 - Optional base64 encoded image data
445 |    *
446 |    * @throws {Error} Throws an error with message "failed to send chat message" if the RTM publish operation fails
447 |    *
448 |    * @returns Promise that resolves when the image message is successfully sent
449 |    *
450 |    * @since 1.7.0
451 |    *
452 |    * @example
453 |    * ```typescript
454 |    * await sendImage('user123', {
455 |    *   uuid: 'msg-456',
456 |    *   url: 'https://example.com/image.jpg'
457 |    * });
458 |    * ```
459 |    */
460 |   public async sendImage(agentUserId: string, message: IChatMessageImage) {
461 |     const traceId = genTranceID()
462 |     this.callMessagePrint(
463 |       ELoggerType.debug,
464 |       `>>> [trancID:${traceId}] [chat] ${agentUserId}`,
465 |       message
466 |     )
467 | 
468 |     const { rtmEngine } = this.getCfg()
469 | 
470 |     const payload = {
471 |       uuid: message.uuid,
472 |       image_url: message?.url || '',
473 |       image_base64: message?.base64 || ''
474 |     }
475 | 
476 |     try {
477 |       const payloadStr = JSON.stringify(payload)
478 |       const options = {
479 |         channelType: 'USER' as ChannelType,
480 |         customType: EMessageType.IMAGE_UPLOAD
481 |       }
482 | 
483 |       this.callMessagePrint(
484 |         ELoggerType.debug,
485 |         `msg: [traceID: ${traceId}] rtm publish`,
486 |         payloadStr
487 |       )
488 | 
489 |       const result = await rtmEngine.publish(agentUserId, payloadStr, options)
490 | 
491 |       this.callMessagePrint(
492 |         ELoggerType.debug,
493 |         `>>> [trancID:${traceId}] [chat]`,
494 |         'sucessfully sent chat message',
495 |         result
496 |       )
497 |     } catch (error: unknown) {
498 |       this.callMessagePrint(
499 |         ELoggerType.error,
500 |         `>>> [trancID:${traceId}] [chat]`,
501 |         'failed to send chat message',
502 |         error
503 |       )
504 |       throw new Error('failed to send chat message')
505 |     }
506 |   }
507 | 
508 |   /**
509 |    * Sends an interrupt message to the specified agent user.
510 |    *
511 |    * This method publishes an interrupt message to the RTM channel of the specified agent user.
512 |    * It is used to signal that the current interaction should be interrupted.
513 |    *
514 |    * @remarks
515 |    * - Must call {@link init} before using this method
516 |    * - Throws error if not initialized or if sending fails
517 |    *
518 |    * @param agentUserId - The user ID of the agent to interrupt
519 |    * @since 1.6.0
520 |    */
521 |   public async interrupt(agentUserId: string) {
522 |     const traceId = genTranceID()
523 |     this.callMessagePrint(
524 |       ELoggerType.debug,
525 |       `>>> [trancID:${traceId}] [interrupt]`,
526 |       agentUserId
527 |     )
528 | 
529 |     const { rtmEngine } = this.getCfg()
530 | 
531 |     const options = {
532 |       channelType: 'USER' as ChannelType,
533 |       customType: EMessageType.MSG_INTERRUPTED
534 |     }
535 |     const messageStr = JSON.stringify({
536 |       customType: EMessageType.MSG_INTERRUPTED
537 |     })
538 | 
539 |     try {
540 |       const result = await rtmEngine.publish(agentUserId, messageStr, options)
541 |       this.callMessagePrint(
542 |         ELoggerType.debug,
543 |         `>>> [trancID:${traceId}] [interrupt]`,
544 |         'sucessfully sent interrupt message',
545 |         result
546 |       )
547 |     } catch (error: unknown) {
548 |       this.callMessagePrint(
549 |         ELoggerType.error,
550 |         `>>> [trancID:${traceId}] [interrupt]`,
551 |         'failed to send interrupt message',
552 |         error
553 |       )
554 |       throw new Error('failed to send interrupt message')
555 |     }
556 |   }
557 | 
558 |   private onChatHistoryUpdated(
559 |     chatHistory: ITranscriptHelperItem<
560 |       Partial<IUserTranscription | IAgentTranscription>
561 |     >[]
562 |   ) {
563 |     this.callMessagePrint(
564 |       ELoggerType.debug,
565 |       `>>> ${EConversationalAIAPIEvents.TRANSCRIPT_UPDATED}`,
566 |       chatHistory
567 |     )
568 |     this.emit(EConversationalAIAPIEvents.TRANSCRIPT_UPDATED, chatHistory)
569 |   }
570 |   private onAgentStateChanged(agentUserId: string, event: TStateChangeEvent) {
571 |     this.callMessagePrint(
572 |       ELoggerType.debug,
573 |       `>>> ${EConversationalAIAPIEvents.AGENT_STATE_CHANGED}`,
574 |       agentUserId,
575 |       event
576 |     )
577 |     this.emit(
578 |       EConversationalAIAPIEvents.AGENT_STATE_CHANGED,
579 |       agentUserId,
580 |       event
581 |     )
582 |   }
583 |   private onAgentInterrupted(
584 |     agentUserId: string,
585 |     event: { turnID: number; timestamp: number }
586 |   ) {
587 |     this.callMessagePrint(
588 |       ELoggerType.debug,
589 |       `>>> ${EConversationalAIAPIEvents.AGENT_INTERRUPTED}`,
590 |       agentUserId,
591 |       event
592 |     )
593 |     this.emit(EConversationalAIAPIEvents.AGENT_INTERRUPTED, agentUserId, event)
594 |   }
595 |   private onDebugLog(message: string) {
596 |     this.emit(EConversationalAIAPIEvents.DEBUG_LOG, message)
597 |   }
598 |   private onAgentMetrics(agentUserId: string, metrics: TAgentMetric) {
599 |     this.callMessagePrint(
600 |       ELoggerType.debug,
601 |       `>>> ${EConversationalAIAPIEvents.AGENT_METRICS}`,
602 |       agentUserId,
603 |       metrics
604 |     )
605 |     this.emit(EConversationalAIAPIEvents.AGENT_METRICS, agentUserId, metrics)
606 |   }
607 |   private onAgentError(agentUserId: string, error: TModuleError) {
608 |     this.callMessagePrint(
609 |       ELoggerType.error,
610 |       `>>> ${EConversationalAIAPIEvents.AGENT_ERROR}`,
611 |       agentUserId,
612 |       error
613 |     )
614 |     this.emit(EConversationalAIAPIEvents.AGENT_ERROR, agentUserId, error)
615 |   }
616 |   private onMessageReceiptUpdated(
617 |     agentUserId: string,
618 |     messageReceipt: TMessageReceipt
619 |   ) {
620 |     this.callMessagePrint(
621 |       ELoggerType.error,
622 |       `>>> ${EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED}`,
623 |       agentUserId,
624 |       messageReceipt
625 |     )
626 |     this.emit(
627 |       EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED,
628 |       agentUserId,
629 |       messageReceipt
630 |     )
631 |   }
632 |   private onMessageError(
633 |     agentUserId: string,
634 |     error: {
635 |       type: EChatMessageType
636 |       code: number
637 |       message: string
638 |       timestamp: number
639 |     }
640 |   ) {
641 |     this.callMessagePrint(
642 |       ELoggerType.error,
643 |       `>>> ${EConversationalAIAPIEvents.MESSAGE_ERROR}`,
644 |       agentUserId,
645 |       error
646 |     )
647 |     this.emit(EConversationalAIAPIEvents.MESSAGE_ERROR, agentUserId, error)
648 |   }
649 | 
650 |   private onMessageSalStatus(agentUserId: string, message: IMessageSalStatus) {
651 |     this.callMessagePrint(
652 |       ELoggerType.debug,
653 |       `>>> ${EConversationalAIAPIEvents.MESSAGE_SAL_STATUS}`,
654 |       agentUserId,
655 |       message
656 |     )
657 |     this.emit(
658 |       EConversationalAIAPIEvents.MESSAGE_SAL_STATUS,
659 |       agentUserId,
660 |       message
661 |     )
662 |   }
663 | 
664 |   private bindRtcEvents() {
665 |     // this.getCfg().rtcEngine.on(
666 |     //   ERTCEvents.AUDIO_METADATA,
667 |     //   this._handleRtcAudioMetadata.bind(this)
668 |     // )
669 |     this.getCfg().rtcEngine.on(
670 |       ERTCEvents.AUDIO_PTS,
671 |       this._handleRtcAudioPTS.bind(this)
672 |     )
673 |   }
674 |   private unbindRtcEvents() {
675 |     // this.getCfg().rtcEngine.off(
676 |     //   ERTCEvents.AUDIO_METADATA,
677 |     //   this._handleRtcAudioMetadata.bind(this)
678 |     // )
679 |     this.getCfg().rtcEngine.off(
680 |       ERTCEvents.AUDIO_PTS,
681 |       this._handleRtcAudioPTS.bind(this)
682 |     )
683 |   }
684 |   private bindRtmEvents() {
685 |     // - message
686 |     this.getCfg().rtmEngine.addEventListener(
687 |       ERTMEvents.MESSAGE,
688 |       this._handleRtmMessage.bind(this)
689 |     )
690 |     // - presence
691 |     this.getCfg().rtmEngine.addEventListener(
692 |       ERTMEvents.PRESENCE,
693 |       this._handleRtmPresence.bind(this)
694 |     )
695 |     // - status
696 |     this.getCfg().rtmEngine.addEventListener(
697 |       ERTMEvents.STATUS,
698 |       this._handleRtmStatus.bind(this)
699 |     )
700 |   }
701 |   private unbindRtmEvents() {
702 |     // - message
703 |     this.getCfg().rtmEngine.removeEventListener(
704 |       ERTMEvents.MESSAGE,
705 |       this._handleRtmMessage.bind(this)
706 |     )
707 |     // - presence
708 |     this.getCfg().rtmEngine.removeEventListener(
709 |       ERTMEvents.PRESENCE,
710 |       this._handleRtmPresence.bind(this)
711 |     )
712 |     // - status
713 |     this.getCfg().rtmEngine.removeEventListener(
714 |       ERTMEvents.STATUS,
715 |       this._handleRtmStatus.bind(this)
716 |     )
717 |   }
718 | 
719 |   // private _handleRtcAudioMetadata(metadata: Uint8Array) {
720 |   //   try {
721 |   //     const pts64 = Number(new DataView(metadata.buffer).getBigUint64(0, true))
722 |   //     this.callMessagePrint(
723 |   //       ELoggerType.debug,
724 |   //       `<<< ${ERTCEvents.AUDIO_METADATA}`,
725 |   //       pts64
726 |   //     )
727 |   //     this.covSubRenderController.setPts(pts64)
728 |   //   } catch (error) {
729 |   //     this.callMessagePrint(
730 |   //       ELoggerType.error,
731 |   //       `<<< ${ERTCEvents.AUDIO_METADATA}`,
732 |   //       metadata,
733 |   //       error
734 |   //     )
735 |   //   }
736 |   // }
737 | 
738 |   private _handleRtcAudioPTS(pts: number) {
739 |     try {
740 |       this.callMessagePrint(
741 |         ELoggerType.debug,
742 |         `<<< ${ERTCEvents.AUDIO_PTS}`,
743 |         pts
744 |       )
745 |       this.covSubRenderController.setPts(pts)
746 |     } catch (error) {
747 |       this.callMessagePrint(
748 |         ELoggerType.error,
749 |         `<<< ${ERTCEvents.AUDIO_PTS}`,
750 |         pts,
751 |         error
752 |       )
753 |     }
754 |   }
755 | 
756 |   private _handleRtmMessage(message: RTMEvents.MessageEvent) {
757 |     const traceId = genTranceID()
758 |     this.callMessagePrint(
759 |       ELoggerType.debug,
760 |       `>>> [trancID:${traceId}] ${ERTMEvents.MESSAGE}`,
761 |       `Publisher: ${message.publisher}, type: ${message.messageType}`
762 |     )
763 |     // Handle the message
764 |     try {
765 |       const messageData = message.message
766 |       // if string, parse it
767 |       if (typeof messageData === 'string') {
768 |         const parsedMessage = JSON.parse(messageData)
769 |         this.callMessagePrint(
770 |           ELoggerType.debug,
771 |           `>>> [trancID:${traceId}] ${ERTMEvents.MESSAGE}`,
772 |           parsedMessage
773 |         )
774 |         this.covSubRenderController.handleMessage(parsedMessage, {
775 |           publisher: message.publisher
776 |         })
777 |         return
778 |       }
779 |       // if Uint8Array, convert to string
780 |       if (messageData instanceof Uint8Array) {
781 |         const decoder = new TextDecoder('utf-8')
782 |         const messageString = decoder.decode(messageData)
783 |         const parsedMessage = JSON.parse(messageString)
784 |         this.callMessagePrint(
785 |           ELoggerType.debug,
786 |           `>>> [trancID:${traceId}] ${ERTMEvents.MESSAGE}`,
787 |           parsedMessage
788 |         )
789 |         this.covSubRenderController.handleMessage(parsedMessage, {
790 |           publisher: message.publisher
791 |         })
792 |         return
793 |       }
794 |       this.callMessagePrint(
795 |         ELoggerType.warn,
796 |         `>>> [trancID:${traceId}] ${ERTMEvents.MESSAGE}`,
797 |         'Unsupported message type received'
798 |       )
799 |     } catch (error) {
800 |       this.callMessagePrint(
801 |         ELoggerType.error,
802 |         `>>> [trancID:${traceId}] ${ERTMEvents.MESSAGE}`,
803 |         'Failed to parse message',
804 |         error
805 |       )
806 |     }
807 |   }
808 |   private _handleRtmPresence(presence: RTMEvents.PresenceEvent) {
809 |     const traceId = genTranceID()
810 |     this.callMessagePrint(
811 |       ELoggerType.debug,
812 |       `>>> [trancID:${traceId}] ${ERTMEvents.PRESENCE}`,
813 |       `Publisher: ${presence.publisher}`
814 |     )
815 |     // Handle the presence event
816 |     const stateChanged = presence.stateChanged
817 |     if (stateChanged?.state && stateChanged?.turn_id) {
818 |       this.callMessagePrint(
819 |         ELoggerType.debug,
820 |         `>>> [trancID:${traceId}] ${ERTMEvents.PRESENCE}`,
821 |         `State changed: ${stateChanged.state}, Turn ID: ${stateChanged.turn_id}, timestamp: ${presence.timestamp}`
822 |       )
823 |       this.covSubRenderController.handleAgentStatus(
824 |         presence as Omit<RTMEvents.PresenceEvent, 'stateChanged'> & {
825 |           stateChanged: {
826 |             state: EAgentState
827 |             turn_id: string
828 |           }
829 |         }
830 |       )
831 |     }
832 |     this.callMessagePrint(
833 |       ELoggerType.debug,
834 |       `>>> [trancID:${traceId}] ${ERTMEvents.PRESENCE}`,
835 |       'No state change detected, skipping handling presence event'
836 |     )
837 |   }
838 |   private _handleRtmStatus(
839 |     status:
840 |       | RTMEvents.RTMConnectionStatusChangeEvent
841 |       | RTMEvents.StreamChannelConnectionStatusChangeEvent
842 |   ) {
843 |     const traceId = genTranceID()
844 |     this.callMessagePrint(
845 |       ELoggerType.debug,
846 |       `>>> [trancID:${traceId}] ${ERTMEvents.STATUS}`,
847 |       status
848 |     )
849 |   }
850 | }


--------------------------------------------------------------------------------
/utils/sub-renders.ts:
--------------------------------------------------------------------------------
   1 | import { logger as agoraLogger } from '@agora-js/report'
   2 | import type { RTMEvents } from 'agora-rtm'
   3 | import _ from 'lodash'
   4 | 
   5 | import {
   6 |   type EAgentState,
   7 |   EChatMessageType,
   8 |   type EConversationalAIAPIEvents,
   9 |   EMessageType,
  10 |   EModuleType,
  11 |   ETranscriptHelperMode,
  12 |   ETurnStatus,
  13 |   type IAgentTranscription,
  14 |   type IConversationalAIAPIEventHandlers,
  15 |   type IMessageError,
  16 |   type IMessageInterrupt,
  17 |   type IMessageMetrics,
  18 |   type IMessageSalStatus,
  19 |   type IPresenceState,
  20 |   type ITranscriptHelperItem,
  21 |   type ITranscriptionBase,
  22 |   type IUserTranscription,
  23 |   type TDataChunkMessageWord,
  24 |   type TQueueItem,
  25 |   type TTranscriptHelperObjectWord
  26 | } from '@/conversational-ai-api/type'
  27 | import { factoryFormatLog } from '@/conversational-ai-api/utils'
  28 | import { ELoggerType, logger } from '@/lib/logger'
  29 | 
  30 | const TAG = 'CovSubRenderController'
  31 | const CONSOLE_LOG_PREFIX = `[${TAG}]`
  32 | const SELF_USER_ID = 0
  33 | const VERSION = '1.8.0'
  34 | 
  35 | const DEFAULT_INTERVAL = 200 // milliseconds
  36 | const DEFAULT_CHUNK_INTERVAL = 100 // milliseconds, 10 char/s
  37 | 
  38 | const formatLog = factoryFormatLog({ tag: TAG })
  39 | 
  40 | /**
  41 |  * CovSubRenderController is a service that manages the transcript messages from RTM messages.
  42 |  *
  43 |  * Best practices:
  44 |  *
  45 |  * 1. Bind `onChatHistoryUpdated` and `onAgentStateChanged` callbacks to handle chat history updates and agent state changes when initializing the service.
  46 |  *
  47 |  * 2. Call `run` method to start the service. One common use case is to call it after the user joins a channel.
  48 |  *
  49 |  * 3. Call `setPts` method to update the current PTS (Presentation Time Stamp) when receiving new media data. This is crucial for synchronizing the transcripts with the media playback.
  50 |  *
  51 |  * 4. [Cleanup] Call `cleanup` method to reset the service state when leaving a channel or when the service is no longer needed. This will clear the chat history, queue, and other internal states.
  52 |  */
  53 | export class CovSubRenderController {
  54 |   private static NAME = TAG
  55 |   private static VERSION = VERSION
  56 |   private callMessagePrint: (type: ELoggerType, ...args: unknown[]) => void
  57 |   public static self_uid = SELF_USER_ID
  58 | 
  59 |   private _mode: ETranscriptHelperMode = ETranscriptHelperMode.UNKNOWN
  60 |   private _queue: TQueueItem[] = []
  61 |   private _interval: number
  62 |   private _intervalRef: NodeJS.Timeout | null = null
  63 |   private _pts: number = 0 // current pts
  64 |   private _lastPoppedQueueItem: TQueueItem | null | undefined = null
  65 |   private _isRunning: boolean = false
  66 |   private _agentMessageState: {
  67 |     state: EAgentState
  68 |     turn_id: string | number
  69 |     timestamp: number
  70 |   } | null = null
  71 |   private _transcriptChunk: {
  72 |     index: number
  73 |     data: IAgentTranscription
  74 |     uid: string
  75 |   } | null = null
  76 | 
  77 |   public chatHistory: ITranscriptHelperItem<
  78 |     Partial<IUserTranscription | IAgentTranscription>
  79 |   >[] = []
  80 |   public onChatHistoryUpdated:
  81 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.TRANSCRIPT_UPDATED]
  82 |     | null = null
  83 |   public onAgentStateChanged:
  84 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_STATE_CHANGED]
  85 |     | null
  86 |   public onAgentInterrupted:
  87 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_INTERRUPTED]
  88 |     | null = null
  89 |   public onDebugLog:
  90 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.DEBUG_LOG]
  91 |     | null = null
  92 |   public onAgentMetrics:
  93 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_METRICS]
  94 |     | null = null
  95 |   public onAgentError:
  96 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_ERROR]
  97 |     | null = null
  98 |   public onMessageReceipt:
  99 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED]
 100 |     | null = null
 101 |   public onMessageError:
 102 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.MESSAGE_ERROR]
 103 |     | null = null
 104 |   public onMessageSalStatus:
 105 |     | IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.MESSAGE_SAL_STATUS]
 106 |     | null = null
 107 | 
 108 |   constructor(
 109 |     options: {
 110 |       messageCacheTimeout?: number
 111 |       interval?: number
 112 |       onChatHistoryUpdated?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.TRANSCRIPT_UPDATED]
 113 |       onAgentStateChanged?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_STATE_CHANGED]
 114 |       onAgentInterrupted?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_INTERRUPTED]
 115 |       onDebugLog?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.DEBUG_LOG]
 116 |       onAgentMetrics?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_METRICS]
 117 |       onAgentError?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.AGENT_ERROR]
 118 |       onMessageReceipt?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.MESSAGE_RECEIPT_UPDATED]
 119 |       onMessageError?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.MESSAGE_ERROR]
 120 |       onMessageSalStatus?: IConversationalAIAPIEventHandlers[EConversationalAIAPIEvents.MESSAGE_SAL_STATUS]
 121 |     } = {}
 122 |   ) {
 123 |     this.callMessagePrint = (
 124 |       type: ELoggerType = ELoggerType.debug,
 125 |       ...args: unknown[]
 126 |     ) => {
 127 |       logger[type](formatLog(...args))
 128 |       this.onDebugLog?.(`[${type}] ${formatLog(...args)}`)
 129 |       agoraLogger[type](...args)
 130 |     }
 131 |     this.callMessagePrint(
 132 |       ELoggerType.debug,
 133 |       `${CovSubRenderController.NAME} initialized, version: ${CovSubRenderController.VERSION}`
 134 |     )
 135 |     this._interval = options.interval ?? DEFAULT_INTERVAL
 136 |     this.onChatHistoryUpdated = options.onChatHistoryUpdated ?? null
 137 |     this.onAgentStateChanged = options.onAgentStateChanged ?? null
 138 |     this.onAgentInterrupted = options.onAgentInterrupted ?? null
 139 |     this.onDebugLog = options.onDebugLog ?? null
 140 |     this.onAgentMetrics = options.onAgentMetrics ?? null
 141 |     this.onAgentError = options.onAgentError ?? null
 142 |     this.onMessageReceipt = options.onMessageReceipt ?? null
 143 |     this.onMessageError = options.onMessageError ?? null
 144 |     this.onMessageSalStatus = options.onMessageSalStatus ?? null
 145 |   }
 146 | 
 147 |   private _preSetupInterval() {
 148 |     if (!this._isRunning) {
 149 |       console.error(CONSOLE_LOG_PREFIX, 'Message service is not running')
 150 |       this.callMessagePrint(
 151 |         ELoggerType.error,
 152 |         '_preSetupInterval',
 153 |         'Message service is not running'
 154 |       )
 155 |       return
 156 |     }
 157 |   }
 158 | 
 159 |   private _setupIntervalForWords(options?: { isForce?: boolean }) {
 160 |     this._preSetupInterval()
 161 |     // if force: clean older and reset interval
 162 |     if (options?.isForce) {
 163 |       if (this._intervalRef) {
 164 |         clearInterval(this._intervalRef)
 165 |         this._intervalRef = null
 166 |       }
 167 |       this._intervalRef = setInterval(
 168 |         this._handleQueue.bind(this),
 169 |         this._interval
 170 |       )
 171 |       return
 172 |     }
 173 |     // else(if not forced): skip if interval is already set, otherwise set an interval
 174 |     if (this._intervalRef) {
 175 |       return
 176 |     }
 177 |     this._intervalRef = setInterval(
 178 |       this._handleQueue.bind(this),
 179 |       this._interval
 180 |     )
 181 |   }
 182 | 
 183 |   private _handleQueue() {
 184 |     const queueLength = this._queue.length
 185 |     // empty queue, skip
 186 |     if (queueLength === 0) {
 187 |       // console.debug(CONSOLE_LOG_PREFIX, 'Queue is empty, skip')
 188 |       return
 189 |     }
 190 |     const curPTS = this._pts
 191 |     // only one item, update chatHistory with queueItem
 192 |     if (queueLength === 1) {
 193 |       // console.debug(
 194 |       //   CONSOLE_LOG_PREFIX,
 195 |       //   'Queue has only one item, update chatHistory',
 196 |       //   JSON.stringify(this._queue[0])
 197 |       // )
 198 |       const queueItem = this._queue[0]
 199 |       this._handleTurnObj(queueItem, curPTS)
 200 |       this._mutateChatHistory()
 201 |       return
 202 |     }
 203 |     if (queueLength > 2) {
 204 |       // console.error(
 205 |       //   CONSOLE_LOG_PREFIX,
 206 |       //   'Queue length is greater than 2, but it should not happen'
 207 |       // )
 208 |       this.callMessagePrint(
 209 |         ELoggerType.error,
 210 |         'Queue length is greater than 2, but it should not happen'
 211 |       )
 212 |     }
 213 |     // assume the queueLength is 2
 214 |     if (queueLength > 1) {
 215 |       this._queue = this._queue.sort((a, b) => a.turn_id - b.turn_id)
 216 |       const nextItem = this._queue[this._queue.length - 1]
 217 |       const lastItem = this._queue[this._queue.length - 2]
 218 |       // check if nextItem is started
 219 |       const firstWordOfNextItem = nextItem.words[0]
 220 |       // if firstWordOfNextItem.start_ms > curPTS, work on lastItem
 221 |       if (firstWordOfNextItem.start_ms > curPTS) {
 222 |         this._handleTurnObj(lastItem, curPTS)
 223 |         this._mutateChatHistory()
 224 |         return
 225 |       }
 226 |       // if firstWordOfNextItem.start_ms <= curPTS, work on nextItem, assume lastItem is interrupted(and drop it)
 227 |       const lastItemCorrespondingChatHistoryItem = this.chatHistory.find(
 228 |         (item) =>
 229 |           item.turn_id === lastItem.turn_id &&
 230 |           item.stream_id === lastItem.stream_id
 231 |       )
 232 |       if (!lastItemCorrespondingChatHistoryItem) {
 233 |         this.callMessagePrint(
 234 |           ELoggerType.warn,
 235 |           'No corresponding chatHistory item found',
 236 |           lastItem
 237 |         )
 238 |         return
 239 |       }
 240 |       lastItemCorrespondingChatHistoryItem.status = ETurnStatus.INTERRUPTED
 241 |       this._lastPoppedQueueItem = this._queue.shift()
 242 |       // handle nextItem
 243 |       this._handleTurnObj(nextItem, curPTS)
 244 |       this._mutateChatHistory()
 245 |       return
 246 |     }
 247 |   }
 248 | 
 249 |   private _handleTurnObj(queueItem: TQueueItem, curPTS: number) {
 250 |     let correspondingChatHistoryItem = this.chatHistory.find(
 251 |       (item) =>
 252 |         item.turn_id === queueItem.turn_id &&
 253 |         item.stream_id === queueItem.stream_id
 254 |     )
 255 |     this.callMessagePrint(
 256 |       ELoggerType.debug,
 257 |       'handleTurnObj',
 258 |       queueItem,
 259 |       'correspondingChatHistoryItem',
 260 |       correspondingChatHistoryItem
 261 |     )
 262 |     if (!correspondingChatHistoryItem) {
 263 |       this.callMessagePrint(
 264 |         ELoggerType.debug,
 265 |         'handleTurnObj',
 266 |         'No corresponding chatHistory item found',
 267 |         'push to chatHistory'
 268 |       )
 269 |       correspondingChatHistoryItem = {
 270 |         turn_id: queueItem.turn_id,
 271 |         uid: queueItem.uid,
 272 |         stream_id: queueItem.stream_id,
 273 |         _time: new Date().getTime(),
 274 |         text: '',
 275 |         status: queueItem.status,
 276 |         metadata: queueItem
 277 |       }
 278 |       this._appendChatHistory(correspondingChatHistoryItem)
 279 |     }
 280 |     // update correspondingChatHistoryItem._time for chatHistory auto-scroll
 281 |     correspondingChatHistoryItem._time = new Date().getTime()
 282 |     // update correspondingChatHistoryItem.metadata
 283 |     correspondingChatHistoryItem.metadata = queueItem
 284 |     // update correspondingChatHistoryItem.status if queueItem.status is interrupted(from message.interrupt event)
 285 |     if (queueItem.status === ETurnStatus.INTERRUPTED) {
 286 |       correspondingChatHistoryItem.status = ETurnStatus.INTERRUPTED
 287 |     }
 288 |     // pop all valid word items(those word.start_ms <= curPTS) in queueItem
 289 |     const validWords: TTranscriptHelperObjectWord[] = []
 290 |     const restWords: TTranscriptHelperObjectWord[] = []
 291 |     for (const word of queueItem.words) {
 292 |       if (word.start_ms <= curPTS) {
 293 |         validWords.push(word)
 294 |       } else {
 295 |         restWords.push(word)
 296 |       }
 297 |     }
 298 |     // check if restWords is empty
 299 |     const isRestWordsEmpty = restWords.length === 0
 300 |     // check if validWords last word is final
 301 |     const isLastWordFinal =
 302 |       validWords[validWords.length - 1]?.word_status !== ETurnStatus.IN_PROGRESS
 303 |     // if restWords is empty and validWords last word is final, this turn is ended
 304 |     if (isRestWordsEmpty && isLastWordFinal) {
 305 |       // update chatHistory with queueItem
 306 |       correspondingChatHistoryItem.text = queueItem.text
 307 |       correspondingChatHistoryItem.status = queueItem.status
 308 |       // pop queueItem
 309 |       this._lastPoppedQueueItem = this._queue.shift()
 310 |       return
 311 |     }
 312 |     // if restWords is not empty, update correspondingChatHistoryItem.text
 313 |     const validWordsText = validWords
 314 |       .filter((word) => word.start_ms <= this._pts)
 315 |       .map((word) => word.word)
 316 |       .join('')
 317 |     correspondingChatHistoryItem.text = validWordsText
 318 |     // if validWords last word is interrupted, this turn is ended
 319 |     const isLastWordInterrupted =
 320 |       validWords[validWords.length - 1]?.word_status === ETurnStatus.INTERRUPTED
 321 |     if (isLastWordInterrupted) {
 322 |       // pop queueItem
 323 |       this._lastPoppedQueueItem = this._queue.shift()
 324 |       return
 325 |     }
 326 |     return
 327 |   }
 328 | 
 329 |   private _mutateChatHistory() {
 330 |     // console.debug(CONSOLE_LOG_PREFIX, 'Mutate chatHistory', this.chatHistory)
 331 |     this.callMessagePrint(
 332 |       ELoggerType.debug,
 333 |       '>>> onChatHistoryUpdated',
 334 |       `pts: ${this._pts}, chatHistory length: ${this.chatHistory.length}`,
 335 |       this.chatHistory
 336 |         .map((item) => `${item.uid}:${item.text}[status: ${item.status}]`)
 337 |         .join('\n')
 338 |     )
 339 |     this.onChatHistoryUpdated?.(this.chatHistory)
 340 |   }
 341 | 
 342 |   private _appendChatHistory(
 343 |     item: ITranscriptHelperItem<
 344 |       Partial<IUserTranscription | IAgentTranscription>
 345 |     >
 346 |   ) {
 347 |     // if item.turn_id is 0, append to the front of chatHistory(greeting message)
 348 |     if (item.turn_id === 0) {
 349 |       this.chatHistory = [item, ...this.chatHistory]
 350 |     } else {
 351 |       this.chatHistory.push(item)
 352 |     }
 353 |   }
 354 | 
 355 |   private _interruptQueue(options: { turn_id: number; start_ms: number }) {
 356 |     const turn_id = options.turn_id
 357 |     const start_ms = options.start_ms
 358 |     const correspondingQueueItem = this._queue.find(
 359 |       (item) => item.turn_id === turn_id
 360 |     )
 361 |     this.callMessagePrint(
 362 |       ELoggerType.debug,
 363 |       'interruptQueue',
 364 |       `turn_id: ${turn_id}, start_ms: ${start_ms}, correspondingQueueItem: ${correspondingQueueItem}`
 365 |     )
 366 |     if (!correspondingQueueItem) {
 367 |       // console.debug(
 368 |       //   CONSOLE_LOG_PREFIX,
 369 |       //   'No corresponding queue item found',
 370 |       //   options
 371 |       // )
 372 |       return
 373 |     }
 374 |     // if correspondingQueueItem exists, update its status to interrupted
 375 |     correspondingQueueItem.status = ETurnStatus.INTERRUPTED
 376 |     // split words into two parts, set left one word and all right words to interrupted
 377 |     const leftWords = correspondingQueueItem.words.filter(
 378 |       (word) => word.start_ms <= start_ms
 379 |     )
 380 |     const rightWords = correspondingQueueItem.words.filter(
 381 |       (word) => word.start_ms > start_ms
 382 |     )
 383 |     // check if leftWords is empty
 384 |     const isLeftWordsEmpty = leftWords.length === 0
 385 |     if (isLeftWordsEmpty) {
 386 |       // if leftWords is empty, set all words to interrupted
 387 |       correspondingQueueItem.words.forEach((word) => {
 388 |         word.word_status = ETurnStatus.INTERRUPTED
 389 |       })
 390 |     } else {
 391 |       // if leftWords is not empty, set leftWords[leftWords.length - 1].word_status to interrupted
 392 |       leftWords[leftWords.length - 1].word_status = ETurnStatus.INTERRUPTED
 393 |       // workaround: pts < interrupt.start_ms, and interrupt will be ignored
 394 |       if (leftWords?.[leftWords.length - 2]) {
 395 |         leftWords[leftWords.length - 2].word_status = ETurnStatus.INTERRUPTED
 396 |       }
 397 |       // and all right words to interrupted
 398 |       rightWords.forEach((word) => {
 399 |         word.word_status = ETurnStatus.INTERRUPTED
 400 |       })
 401 |       // update words
 402 |       correspondingQueueItem.words = [...leftWords, ...rightWords]
 403 |     }
 404 |   }
 405 | 
 406 |   private _pushToQueue(data: {
 407 |     turn_id: number
 408 |     words: TTranscriptHelperObjectWord[]
 409 |     text: string
 410 |     status: ETurnStatus
 411 |     stream_id: number
 412 |     uid: string
 413 |   }) {
 414 |     const targetQueueItem = this._queue.find(
 415 |       (item) => item.turn_id === data.turn_id
 416 |     )
 417 |     const latestTurnId = this._queue.reduce((max, item) => {
 418 |       return Math.max(max, item.turn_id)
 419 |     }, 0)
 420 |     // if not found, push to queue or drop if turn_id is less than latestTurnId
 421 |     if (!targetQueueItem) {
 422 |       // drop if turn_id is less than latestTurnId
 423 |       if (data.turn_id < latestTurnId) {
 424 |         this.callMessagePrint(
 425 |           ELoggerType.debug,
 426 |           `[Word Mode]`,
 427 |           `[${data.uid}]`,
 428 |           'Drop message with turn_id less than latestTurnId',
 429 |           `turn_id: ${data.turn_id}, latest turn_id: ${latestTurnId}`,
 430 |           data
 431 |         )
 432 |         return
 433 |       }
 434 |       const newQueueItem = {
 435 |         turn_id: data.turn_id,
 436 |         text: data.text,
 437 |         words: this.sortWordsWithStatus(data.words, data.status),
 438 |         status: data.status,
 439 |         stream_id: data.stream_id,
 440 |         uid: data.uid
 441 |       }
 442 |       this.callMessagePrint(
 443 |         ELoggerType.debug,
 444 |         `[Word Mode]`,
 445 |         `[${data.uid}]`,
 446 |         'push to queue',
 447 |         newQueueItem
 448 |       )
 449 |       // push to queue
 450 |       this._queue.push(newQueueItem)
 451 |       return
 452 |     }
 453 |     // if found, update text, words(sorted with status) and turn_status
 454 |     this.callMessagePrint(
 455 |       ELoggerType.debug,
 456 |       `[Word Mode]`,
 457 |       `[${data.uid}]`,
 458 |       'update queue item',
 459 |       targetQueueItem,
 460 |       data
 461 |     )
 462 |     targetQueueItem.text = data.text
 463 |     targetQueueItem.words = this.sortWordsWithStatus(
 464 |       [...targetQueueItem.words, ...data.words],
 465 |       data.status
 466 |     )
 467 |     // if targetQueueItem.status is end, and data.status is in_progress, skip status update (unexpected case)
 468 |     if (
 469 |       targetQueueItem.status !== ETurnStatus.IN_PROGRESS &&
 470 |       data.status === ETurnStatus.IN_PROGRESS
 471 |     ) {
 472 |       return
 473 |     }
 474 |     targetQueueItem.status = data.status
 475 |   }
 476 | 
 477 |   private _teardownInterval() {
 478 |     if (this._intervalRef) {
 479 |       clearInterval(this._intervalRef)
 480 |       this._intervalRef = null
 481 |     }
 482 |   }
 483 | 
 484 |   protected sortWordsWithStatus(
 485 |     words: TDataChunkMessageWord[],
 486 |     turn_status: ETurnStatus
 487 |   ) {
 488 |     if (words.length === 0) {
 489 |       return words
 490 |     }
 491 |     const sortedWords: TTranscriptHelperObjectWord[] = words
 492 |       .map((word) => ({
 493 |         ...word,
 494 |         word_status: ETurnStatus.IN_PROGRESS
 495 |       }))
 496 |       .sort((a, b) => a.start_ms - b.start_ms)
 497 |       .reduce((acc, curr) => {
 498 |         // Only add if start_ms is unique
 499 |         if (!acc.find((word) => word.start_ms === curr.start_ms)) {
 500 |           acc.push(curr)
 501 |         }
 502 |         return acc
 503 |       }, [] as TTranscriptHelperObjectWord[])
 504 |     const isMessageFinal = turn_status !== ETurnStatus.IN_PROGRESS
 505 |     if (isMessageFinal) {
 506 |       sortedWords[sortedWords.length - 1].word_status = turn_status
 507 |     }
 508 |     return sortedWords
 509 |   }
 510 | 
 511 |   protected handleTextMessage(uid: string, message: IUserTranscription) {
 512 |     const turn_id = message.turn_id
 513 |     const text = message.text || ''
 514 |     const stream_id = message.stream_id
 515 |     const turn_status = ETurnStatus.END
 516 | 
 517 |     const targetChatHistoryItem = this.chatHistory.find(
 518 |       (item) => item.turn_id === turn_id && item.stream_id === stream_id
 519 |     )
 520 |     // if not found, push to chatHistory
 521 |     if (!targetChatHistoryItem) {
 522 |       this.callMessagePrint(
 523 |         ELoggerType.debug,
 524 |         `[Text Mode]`,
 525 |         `[${uid}]`,
 526 |         'new item',
 527 |         message
 528 |       )
 529 |       this._appendChatHistory({
 530 |         turn_id,
 531 |         uid: message.stream_id
 532 |           ? `${CovSubRenderController.self_uid}`
 533 |           : `${uid}`,
 534 |         stream_id,
 535 |         _time: new Date().getTime(),
 536 |         text,
 537 |         status: turn_status,
 538 |         metadata: message
 539 |       })
 540 |     } else {
 541 |       // if found, update text and status
 542 |       targetChatHistoryItem.text = text
 543 |       targetChatHistoryItem.status = turn_status
 544 |       targetChatHistoryItem.metadata = message
 545 |       targetChatHistoryItem._time = new Date().getTime()
 546 |       this.callMessagePrint(
 547 |         ELoggerType.debug,
 548 |         `[Text Mode]`,
 549 |         `[${uid}]`,
 550 |         targetChatHistoryItem
 551 |       )
 552 |     }
 553 |     this._mutateChatHistory()
 554 |   }
 555 | 
 556 |   private _handleTranscriptChunk() {
 557 |     if (!this._transcriptChunk) {
 558 |       this.callMessagePrint(
 559 |         ELoggerType.warn,
 560 |         `[${ETranscriptHelperMode.CHUNK} Mode]`,
 561 |         '_handleTranscriptChunk',
 562 |         'missing _transcriptChunk'
 563 |       )
 564 |       return
 565 |     }
 566 |     const currentIdx = this._transcriptChunk.index
 567 |     const currentTranscript = this._transcriptChunk.data
 568 |     const currentMaxLength = currentTranscript.text.length
 569 |     const uid = this._transcriptChunk.uid
 570 | 
 571 |     const nextIdx =
 572 |       currentIdx + 1 >= currentMaxLength ? currentMaxLength : currentIdx + 1
 573 |     this._transcriptChunk.index = nextIdx
 574 |     const validTranscriptString = currentTranscript.text.substring(0, nextIdx)
 575 |     const isValidTranscriptStringEnded =
 576 |       validTranscriptString.length > 0 &&
 577 |       currentTranscript.turn_status !== ETurnStatus.IN_PROGRESS &&
 578 |       validTranscriptString.length === currentTranscript.text.length
 579 | 
 580 |     const targetChatHistoryItem = this.chatHistory.find(
 581 |       (item) =>
 582 |         item.turn_id === currentTranscript.turn_id &&
 583 |         item.stream_id === currentTranscript.stream_id
 584 |     )
 585 |     // if not found, push to chatHistory
 586 |     if (!targetChatHistoryItem) {
 587 |       this.callMessagePrint(
 588 |         ELoggerType.debug,
 589 |         `[${ETranscriptHelperMode.CHUNK} Mode]`,
 590 |         `[${uid}]`,
 591 |         'new transcriptChunk',
 592 |         this._transcriptChunk
 593 |       )
 594 |       this._appendChatHistory({
 595 |         turn_id: currentTranscript.turn_id,
 596 |         uid: currentTranscript.stream_id
 597 |           ? `${CovSubRenderController.self_uid}`
 598 |           : `${uid}`,
 599 |         stream_id: currentTranscript.stream_id,
 600 |         _time: Date.now(),
 601 |         text: validTranscriptString,
 602 |         status: currentTranscript.turn_status,
 603 |         metadata: currentTranscript
 604 |       })
 605 |     } else {
 606 |       // if found, update text and status
 607 |       targetChatHistoryItem.text = validTranscriptString
 608 |       targetChatHistoryItem.status = isValidTranscriptStringEnded
 609 |         ? currentTranscript.turn_status
 610 |         : targetChatHistoryItem.status
 611 |       targetChatHistoryItem.metadata = currentTranscript
 612 |       targetChatHistoryItem._time = Date.now()
 613 |       this.callMessagePrint(
 614 |         ELoggerType.debug,
 615 |         `[${ETranscriptHelperMode.CHUNK} Mode]`,
 616 |         `[${uid}]`,
 617 |         'update transcriptChunk',
 618 |         targetChatHistoryItem
 619 |       )
 620 |     }
 621 |     this._mutateChatHistory()
 622 |   }
 623 | 
 624 |   protected handleChunkTextMessage(uid: string, message: IAgentTranscription) {
 625 |     this.callMessagePrint(
 626 |       ELoggerType.debug,
 627 |       `[${ETranscriptHelperMode.CHUNK} Mode]`,
 628 |       `[${uid}]`,
 629 |       'new item',
 630 |       message
 631 |     )
 632 |     // 0. check turn_id, teardown interval if new turn
 633 |     if (
 634 |       this._transcriptChunk &&
 635 |       this._transcriptChunk.data.turn_id < message.turn_id
 636 |     ) {
 637 |       this._teardownInterval()
 638 |       // set chathistory items turn_status to ended
 639 |       const lastChatHistory = this.chatHistory.find(
 640 |         (item) =>
 641 |           item.turn_id === this._transcriptChunk?.data.turn_id &&
 642 |           item.uid === uid
 643 |       )
 644 |       if (lastChatHistory) {
 645 |         lastChatHistory.status = ETurnStatus.END
 646 |       }
 647 |       // set _transcriptChunk to null
 648 |       this._transcriptChunk = null
 649 |     }
 650 |     // 1. update _transcriptChunk
 651 |     this._transcriptChunk = {
 652 |       index: this._transcriptChunk?.index ?? 0,
 653 |       data: message,
 654 |       uid
 655 |     }
 656 |     // 2. check if interval is set, if not, set it
 657 |     if (!this._intervalRef) {
 658 |       this._intervalRef = setInterval(
 659 |         this._handleTranscriptChunk.bind(this),
 660 |         this._interval
 661 |       )
 662 |     }
 663 |   }
 664 | 
 665 |   protected handleMessageInterrupt(uid: string, message: IMessageInterrupt) {
 666 |     this.callMessagePrint(
 667 |       ELoggerType.debug,
 668 |       '<<< [onInterrupted]',
 669 |       `pts: ${this._pts}, uid: ${uid}`,
 670 |       message
 671 |     )
 672 |     const turn_id = message.turn_id
 673 |     // workaround: pts < interrupt.start_ms, and interrupt will be ignored
 674 |     const start_ms = _.min([message.start_ms, this._pts]) || message.start_ms
 675 |     this._interruptQueue({
 676 |       turn_id,
 677 |       start_ms
 678 |     })
 679 |     // interrupt chunk
 680 |     if (this._transcriptChunk) {
 681 |       this._teardownInterval()
 682 |       // set chathistory items turn_status to ended
 683 |       const lastChatHistory = this.chatHistory.find(
 684 |         (item) =>
 685 |           item.turn_id === this._transcriptChunk?.data.turn_id &&
 686 |           item.uid === uid
 687 |       )
 688 |       if (lastChatHistory) {
 689 |         lastChatHistory.status = ETurnStatus.INTERRUPTED
 690 |       }
 691 |       // set _transcriptChunk to null
 692 |       this._transcriptChunk = null
 693 |     }
 694 |     this._mutateChatHistory()
 695 |     this.onAgentInterrupted?.(`${uid}`, {
 696 |       turnID: turn_id,
 697 |       timestamp: start_ms
 698 |     })
 699 |   }
 700 | 
 701 |   protected handleMessageMetrics(uid: string, message: IMessageMetrics) {
 702 |     // this.callMessagePrint(
 703 |     //   ELoggerType.debug,
 704 |     //   '<<< [onMetrics]',
 705 |     //   `pts: ${this._pts}, uid: ${uid}`,
 706 |     //   message
 707 |     // )
 708 |     const latency_ms = message.latency_ms
 709 |     const messageModule = message.module
 710 |     const metric_name = message.metric_name
 711 | 
 712 |     if (!Object.values(EModuleType).includes(messageModule)) {
 713 |       this.callMessagePrint(ELoggerType.warn, 'Unknown metric module:', message)
 714 |       return
 715 |     }
 716 | 
 717 |     this.onAgentMetrics?.(`${uid}`, {
 718 |       type: messageModule,
 719 |       name: metric_name,
 720 |       value: latency_ms,
 721 |       timestamp: message.send_ts
 722 |     })
 723 |   }
 724 | 
 725 |   protected handleMessageSalStatus(uid: string, message: IMessageSalStatus) {
 726 |     this.callMessagePrint(ELoggerType.debug, 'handleMessageSalStatus', message)
 727 |     this.onMessageSalStatus?.(`${uid}`, message)
 728 |   }
 729 | 
 730 |   protected handleMessageError(uid: string, message: IMessageError) {
 731 |     // this.callMessagePrint(
 732 |     //   ELoggerType.debug,
 733 |     //   '<<< [onError]',
 734 |     //   `pts: ${this._pts}, uid: ${uid}`,
 735 |     //   message
 736 |     // )
 737 |     const errorCode = message.code || -1
 738 |     const errorMessage = message.message
 739 |     const messageModule = message.module
 740 | 
 741 |     if (!Object.values(EModuleType).includes(messageModule)) {
 742 |       this.callMessagePrint(ELoggerType.warn, 'Unknown error module:', message)
 743 |       return
 744 |     }
 745 | 
 746 |     if (messageModule === EModuleType.CONTEXT) {
 747 |       try {
 748 |         const messageData = JSON.parse(errorMessage)
 749 |         const errorPayload = {
 750 |           type:
 751 |             messageData?.module === 'picture'
 752 |               ? EChatMessageType.IMAGE
 753 |               : EChatMessageType.UNKNOWN,
 754 |           code: errorCode,
 755 |           message: errorMessage,
 756 |           timestamp: (message?.send_ts as number) || Date.now()
 757 |         }
 758 |         this.onMessageError?.(`${uid}`, errorPayload)
 759 |       } catch (error: unknown) {
 760 |         this.callMessagePrint(
 761 |           ELoggerType.error,
 762 |           'Failed to parse context error message',
 763 |           error,
 764 |           message
 765 |         )
 766 |       }
 767 |     }
 768 | 
 769 |     this.onAgentError?.(`${uid}`, {
 770 |       type: messageModule,
 771 |       code: errorCode,
 772 |       message: errorMessage,
 773 |       timestamp: (message?.send_ts as number) || Date.now()
 774 |     })
 775 |   }
 776 | 
 777 |   // current only used for image messages
 778 |   protected handleMessageInfo(uid: string, message: Record<string, unknown>) {
 779 |     try {
 780 |       const messageStr = (message?.message as string) || ''
 781 |       const messageObj = JSON.parse(messageStr)
 782 |       const moduleType = message?.module as EModuleType
 783 |       const turnId = message?.turn_id as number
 784 |       if (!messageStr || !messageObj || !moduleType || !turnId) {
 785 |         this.callMessagePrint(
 786 |           ELoggerType.error,
 787 |           'handleMessageInfo',
 788 |           'Invalid message object',
 789 |           message
 790 |         )
 791 |         return
 792 |       }
 793 |       const messageType =
 794 |         message?.resource_type === 'picture'
 795 |           ? EChatMessageType.IMAGE
 796 |           : EChatMessageType.UNKNOWN
 797 |       this.onMessageReceipt?.(uid, {
 798 |         moduleType,
 799 |         messageType,
 800 |         message: messageStr,
 801 |         turnId
 802 |       })
 803 |     } catch (error: unknown) {
 804 |       this.callMessagePrint(
 805 |         ELoggerType.debug,
 806 |         'handleMessageInfo',
 807 |         'Failed to parse message string from image info message',
 808 |         error,
 809 |         message
 810 |       )
 811 |     }
 812 |   }
 813 | 
 814 |   public handleAgentStatus(metadata: IPresenceState) {
 815 |     // this.callMessagePrint(
 816 |     //   ELoggerType.debug,
 817 |     //   'handleAgentStatus',
 818 |     //   `pts: ${this._pts}, uid: ${metadata.publisher}`,
 819 |     //   `prev-state: ${this._agentMessageState}, state: ${metadata.stateChanged.state}, turn_id: ${metadata.stateChanged.turn_id}, timestamp: ${metadata.stateChanged.timestamp}`
 820 |     // )
 821 |     const message = metadata.stateChanged
 822 |     const currentTurnId = _.toNumber(message.turn_id) || 0
 823 |     if (_.toNumber(this._agentMessageState?.turn_id || 0) > currentTurnId) {
 824 |       this.callMessagePrint(
 825 |         ELoggerType.debug,
 826 |         'handleAgentStatus',
 827 |         'ignore older message(turn_id)'
 828 |       )
 829 |       return
 830 |     }
 831 |     // check if message is older(by timestamp) than previous one, if so, skip
 832 |     const currentMsgTs = metadata.timestamp
 833 |     if (_.toNumber(this._agentMessageState?.timestamp || 0) >= currentMsgTs) {
 834 |       // console.debug(
 835 |       //   CONSOLE_LOG_PREFIX,
 836 |       //   'handleAgentStatus',
 837 |       //   'ignore older message(timestamp)',
 838 |       //   message?.timestamp,
 839 |       //   currentMsgTs
 840 |       // )
 841 |       this.callMessagePrint(
 842 |         ELoggerType.debug,
 843 |         'handleAgentStatus',
 844 |         'ignore older message(timestamp)'
 845 |       )
 846 |       return
 847 |     }
 848 |     this.callMessagePrint(
 849 |       ELoggerType.debug,
 850 |       '>>> handleAgentStatus',
 851 |       `pts: ${this._pts}, uid: ${metadata.publisher}`,
 852 |       `prev-state: ${this._agentMessageState?.state}, prev-turn_id: ${this._agentMessageState?.turn_id}, prev-timestamp: ${this._agentMessageState?.timestamp}`,
 853 |       `current-state: ${metadata.stateChanged.state}, turn_id: ${metadata.stateChanged.turn_id}, timestamp: ${metadata.timestamp}`
 854 |     )
 855 |     // set current message state
 856 |     this._agentMessageState = {
 857 |       state: message.state,
 858 |       turn_id: message.turn_id,
 859 |       timestamp: currentMsgTs
 860 |     }
 861 |     this.onAgentStateChanged?.(metadata.publisher, {
 862 |       state: message.state,
 863 |       turnID: _.toNumber(message.turn_id),
 864 |       timestamp: currentMsgTs,
 865 |       reason: ''
 866 |     })
 867 |   }
 868 | 
 869 |   protected handleWordAgentMessage(uid: string, message: IAgentTranscription) {
 870 |     // drop message if turn_status is undefined
 871 |     if (typeof message.turn_status === 'undefined') {
 872 |       this.callMessagePrint(
 873 |         ELoggerType.debug,
 874 |         `[Word Mode]`,
 875 |         `[${uid}]`,
 876 |         'Drop message with undefined turn_status',
 877 |         message.turn_id
 878 |       )
 879 |       return
 880 |     }
 881 | 
 882 |     const turn_id = message.turn_id
 883 |     const text = message.text || ''
 884 |     const words = message.words || []
 885 |     const stream_id = message.stream_id
 886 |     const lastPoppedQueueItemTurnId = this._lastPoppedQueueItem?.turn_id
 887 |     // drop message if turn_id is less than last popped queue item
 888 |     // except for the first turn(greeting message, turn_id is 0)
 889 |     if (
 890 |       lastPoppedQueueItemTurnId &&
 891 |       turn_id !== 0 &&
 892 |       turn_id <= lastPoppedQueueItemTurnId
 893 |     ) {
 894 |       this.callMessagePrint(
 895 |         ELoggerType.debug,
 896 |         `[Word Mode]`,
 897 |         `[${uid}]`,
 898 |         'Drop message with turn_id less than last popped queue item',
 899 |         `turn_id: ${turn_id}, last popped queue item turn_id: ${lastPoppedQueueItemTurnId}`
 900 |       )
 901 |       return
 902 |     }
 903 |     this._pushToQueue({
 904 |       uid: message.stream_id ? `${CovSubRenderController.self_uid}` : `${uid}`,
 905 |       turn_id,
 906 |       words,
 907 |       text,
 908 |       status: message.turn_status,
 909 |       stream_id
 910 |     })
 911 |   }
 912 | 
 913 |   public setMode(mode: ETranscriptHelperMode) {
 914 |     if (this._mode !== ETranscriptHelperMode.UNKNOWN) {
 915 |       this.callMessagePrint(
 916 |         ELoggerType.warn,
 917 |         `Mode should only be set once, but it is set[${mode}] again`,
 918 |         'current mode:',
 919 |         this._mode
 920 |       )
 921 |       return
 922 |     }
 923 |     if (mode === ETranscriptHelperMode.UNKNOWN) {
 924 |       this.callMessagePrint(ELoggerType.warn, 'Unknown mode should not be set')
 925 |       return
 926 |     }
 927 |     if (mode === ETranscriptHelperMode.CHUNK) {
 928 |       // set interval to chunk interval
 929 |       this._interval = DEFAULT_CHUNK_INTERVAL
 930 |     } else {
 931 |       // set interval to default interval
 932 |       this._interval = DEFAULT_INTERVAL
 933 |     }
 934 |     this.callMessagePrint(
 935 |       ELoggerType.debug,
 936 |       `setMode`,
 937 |       ETranscriptHelperMode.TEXT
 938 |     )
 939 |     this._mode = mode
 940 |   }
 941 | 
 942 |   public handleMessage<T extends ITranscriptionBase>(
 943 |     message: T,
 944 |     options: {
 945 |       publisher: RTMEvents.MessageEvent['publisher']
 946 |     }
 947 |   ) {
 948 |     const messageObject = message?.object
 949 |     if (!Object.values(EMessageType).includes(messageObject)) {
 950 |       this.callMessagePrint(
 951 |         ELoggerType.info,
 952 |         `<<< [unknown message]`,
 953 |         options,
 954 |         message
 955 |       )
 956 |       return
 957 |     }
 958 | 
 959 |     const isAgentMessage = message.object === EMessageType.AGENT_TRANSCRIPTION
 960 |     const isUserMessage = message.object === EMessageType.USER_TRANSCRIPTION
 961 |     const isMessageInterrupt = message.object === EMessageType.MSG_INTERRUPTED
 962 |     const isMessageMetrics = message.object === EMessageType.MSG_METRICS
 963 |     const isMessageError = message.object === EMessageType.MSG_ERROR
 964 |     // const isMessageState = message.object === EMessageType.MSG_STATE
 965 |     const isMessageInfo = message.object === EMessageType.MESSAGE_INFO
 966 |     const isMessageSalStatus =
 967 |       message.object === EMessageType.MESSAGE_SAL_STATUS
 968 | 
 969 |     // set mode (only once)
 970 |     if (isAgentMessage && this._mode === ETranscriptHelperMode.UNKNOWN) {
 971 |       // 2025-09-28 check if words array is empty or not undefined, and set mode
 972 |       if (
 973 |         !message.words ||
 974 |         (Array.isArray(message.words) && message.words.length === 0)
 975 |       ) {
 976 |         this.setMode(ETranscriptHelperMode.TEXT)
 977 |       } else {
 978 |         this._setupIntervalForWords({ isForce: true })
 979 |         this.setMode(ETranscriptHelperMode.WORD)
 980 |       }
 981 |     }
 982 | 
 983 |     // handle Agent Message
 984 |     if (isAgentMessage && this._mode === ETranscriptHelperMode.WORD) {
 985 |       this._setupIntervalForWords({ isForce: false })
 986 |       this.handleWordAgentMessage(
 987 |         options.publisher,
 988 |         message as unknown as IAgentTranscription
 989 |       )
 990 |       return
 991 |     }
 992 |     if (isAgentMessage && this._mode === ETranscriptHelperMode.TEXT) {
 993 |       this.handleTextMessage(
 994 |         options.publisher,
 995 |         message as unknown as IUserTranscription
 996 |       )
 997 |       return
 998 |     }
 999 |     if (isAgentMessage && this._mode === ETranscriptHelperMode.CHUNK) {
1000 |       this.handleChunkTextMessage(
1001 |         options.publisher,
1002 |         message as unknown as IAgentTranscription
1003 |       )
1004 |       return
1005 |     }
1006 |     // handle User Message
1007 |     if (isUserMessage) {
1008 |       this.handleTextMessage(
1009 |         options.publisher,
1010 |         message as unknown as IUserTranscription
1011 |       )
1012 |       return
1013 |     }
1014 |     // handle Message Interrupt
1015 |     if (isMessageInterrupt) {
1016 |       this.handleMessageInterrupt(
1017 |         options.publisher,
1018 |         message as unknown as IMessageInterrupt
1019 |       )
1020 |       return
1021 |     }
1022 |     // if (isMessageState) {
1023 |     //   this.handleAgentStatus(message as unknown as IMessageState)
1024 |     //   return
1025 |     // }
1026 |     if (isMessageInfo) {
1027 |       this.handleMessageInfo(
1028 |         options.publisher,
1029 |         message as unknown as Record<string, unknown>
1030 |       )
1031 |       return
1032 |     }
1033 |     if (isMessageMetrics) {
1034 |       this.handleMessageMetrics(
1035 |         options.publisher,
1036 |         message as unknown as IMessageMetrics
1037 |       )
1038 |       return
1039 |     }
1040 |     if (isMessageError) {
1041 |       this.handleMessageError(
1042 |         options.publisher,
1043 |         message as unknown as IMessageError
1044 |       )
1045 |       return
1046 |     }
1047 | 
1048 |     if (isMessageSalStatus) {
1049 |       this.handleMessageSalStatus(options.publisher, message as unknown as any)
1050 |       return
1051 |     }
1052 |   }
1053 | 
1054 |   public run() {
1055 |     this._isRunning = true
1056 |   }
1057 | 
1058 |   public setPts(pts: number) {
1059 |     if (this._pts < pts && pts !== 0) {
1060 |       this._pts = pts
1061 |     }
1062 |   }
1063 | 
1064 |   public cleanup() {
1065 |     // console.debug(CONSOLE_LOG_PREFIX, 'Cleanup message service')
1066 |     this.callMessagePrint(ELoggerType.debug, 'cleanup')
1067 |     this._isRunning = false
1068 |     this._teardownInterval()
1069 |     // cleanup queue
1070 |     this._queue = []
1071 |     this._lastPoppedQueueItem = null
1072 |     this._pts = 0
1073 |     // cleanup chatHistory
1074 |     this.chatHistory = []
1075 |     // cleanup mode
1076 |     this._mode = ETranscriptHelperMode.UNKNOWN
1077 |     this._agentMessageState = null
1078 |     this._transcriptChunk = null
1079 |   }
1080 | }


--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
   1 | <!DOCTYPE html>
   2 | <html lang="en">
   3 | <head>
   4 |     <meta charset="UTF-8">
   5 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
   6 |     <title>Central City Mall - Voice AI Assistant with Emotions</title>
   7 |     <style>
   8 |         * {
   9 |             margin: 0;
  10 |             padding: 0;
  11 |             box-sizing: border-box;
  12 |         }
  13 | 
  14 |         body {
  15 |             font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
  16 |             overflow: hidden;
  17 |             height: 100vh;
  18 |             background: linear-gradient(135deg, #0f2027 0%, #203a43 50%, #2c5364 100%);
  19 |             position: relative;
  20 |         }
  21 | 
  22 |         /* Animated Background */
  23 |         .background {
  24 |             position: fixed;
  25 |             top: 0;
  26 |             left: 0;
  27 |             width: 100%;
  28 |             height: 100%;
  29 |             overflow: hidden;
  30 |             z-index: 0;
  31 |         }
  32 | 
  33 |         .city-silhouette {
  34 |             position: absolute;
  35 |             bottom: 0;
  36 |             left: 0;
  37 |             width: 100%;
  38 |             height: 30%;
  39 |             background: linear-gradient(to top, rgba(15, 32, 39, 0.8), transparent);
  40 |         }
  41 | 
  42 |         .building {
  43 |             position: absolute;
  44 |             bottom: 0;
  45 |             background: linear-gradient(to top, rgba(0, 150, 255, 0.1), rgba(0, 200, 255, 0.05));
  46 |             border-top: 2px solid rgba(0, 255, 255, 0.3);
  47 |             box-shadow: 0 0 20px rgba(0, 200, 255, 0.2);
  48 |         }
  49 | 
  50 |         .building-1 { left: 5%; width: 80px; height: 150px; animation: buildingGlow 4s infinite; }
  51 |         .building-2 { left: 15%; width: 100px; height: 200px; animation: buildingGlow 5s infinite 1s; }
  52 |         .building-3 { left: 30%; width: 70px; height: 180px; animation: buildingGlow 4.5s infinite 0.5s; }
  53 |         .building-4 { left: 45%; width: 120px; height: 220px; animation: buildingGlow 5.5s infinite 1.5s; }
  54 |         .building-5 { left: 60%; width: 90px; height: 170px; animation: buildingGlow 4s infinite 2s; }
  55 |         .building-6 { left: 75%; width: 110px; height: 190px; animation: buildingGlow 5s infinite 0.8s; }
  56 |         .building-7 { right: 5%; width: 85px; height: 160px; animation: buildingGlow 4.2s infinite 1.2s; }
  57 | 
  58 |         @keyframes buildingGlow {
  59 |             0%, 100% { opacity: 0.5; box-shadow: 0 0 20px rgba(0, 200, 255, 0.2); }
  60 |             50% { opacity: 0.8; box-shadow: 0 0 40px rgba(0, 255, 255, 0.4); }
  61 |         }
  62 | 
  63 |         .particles {
  64 |             position: absolute;
  65 |             width: 100%;
  66 |             height: 100%;
  67 |         }
  68 | 
  69 |         .particle {
  70 |             position: absolute;
  71 |             width: 3px;
  72 |             height: 3px;
  73 |             background: rgba(0, 255, 255, 0.5);
  74 |             border-radius: 50%;
  75 |             animation: float 15s infinite;
  76 |         }
  77 | 
  78 |         @keyframes float {
  79 |             0% { transform: translateY(100vh) translateX(0); opacity: 0; }
  80 |             10% { opacity: 1; }
  81 |             90% { opacity: 1; }
  82 |             100% { transform: translateY(-100vh) translateX(100px); opacity: 0; }
  83 |         }
  84 | 
  85 |         /* Emotion Display at Top */
  86 |         .emotion-display {
  87 |             position: fixed;
  88 |             top: 100px;
  89 |             left: 50%;
  90 |             transform: translateX(-50%);
  91 |             z-index: 1500;
  92 |             opacity: 0;
  93 |             transition: opacity 0.3s ease;
  94 |         }
  95 | 
  96 |         .emotion-display.active {
  97 |             opacity: 1;
  98 |             animation: emotionPulse 0.6s ease-out;
  99 |         }
 100 | 
 101 |         .emotion-emoji {
 102 |             font-size: 120px;
 103 |             filter: drop-shadow(0 10px 30px rgba(0, 0, 0, 0.5));
 104 |             animation: emotionFloat 3s ease-in-out infinite;
 105 |         }
 106 | 
 107 |         @keyframes emotionPulse {
 108 |             0% { transform: translateX(-50%) scale(0); }
 109 |             50% { transform: translateX(-50%) scale(1.2); }
 110 |             100% { transform: translateX(-50%) scale(1); }
 111 |         }
 112 | 
 113 |         @keyframes emotionFloat {
 114 |             0%, 100% { transform: translateY(0) rotate(-5deg); }
 115 |             50% { transform: translateY(-15px) rotate(5deg); }
 116 |         }
 117 | 
 118 |         .emotion-label {
 119 |             text-align: center;
 120 |             color: white;
 121 |             font-size: 18px;
 122 |             font-weight: 600;
 123 |             margin-top: 10px;
 124 |             text-transform: uppercase;
 125 |             letter-spacing: 2px;
 126 |             text-shadow: 0 2px 10px rgba(0, 0, 0, 0.5);
 127 |             background: rgba(0, 0, 0, 0.3);
 128 |             padding: 8px 20px;
 129 |             border-radius: 20px;
 130 |             backdrop-filter: blur(10px);
 131 |         }
 132 | 
 133 |         /* Top Bar */
 134 |         .top-bar {
 135 |             position: fixed;
 136 |             top: 0;
 137 |             left: 0;
 138 |             right: 0;
 139 |             height: 80px;
 140 |             background: rgba(255, 255, 255, 0.05);
 141 |             backdrop-filter: blur(20px);
 142 |             border-bottom: 1px solid rgba(255, 255, 255, 0.1);
 143 |             display: flex;
 144 |             align-items: center;
 145 |             justify-content: space-between;
 146 |             padding: 0 40px;
 147 |             z-index: 1000;
 148 |             box-shadow: 0 4px 30px rgba(0, 0, 0, 0.1);
 149 |         }
 150 | 
 151 |         .logo {
 152 |             display: flex;
 153 |             align-items: center;
 154 |             gap: 12px;
 155 |             color: white;
 156 |             font-size: 20px;
 157 |             font-weight: 600;
 158 |             letter-spacing: 1px;
 159 |         }
 160 | 
 161 |         .logo-icon {
 162 |             width: 40px;
 163 |             height: 40px;
 164 |             background: linear-gradient(135deg, #00d4ff, #0096ff);
 165 |             border-radius: 12px;
 166 |             display: flex;
 167 |             align-items: center;
 168 |             justify-content: center;
 169 |             font-size: 24px;
 170 |             box-shadow: 0 4px 15px rgba(0, 200, 255, 0.4);
 171 |         }
 172 | 
 173 |         .nav-buttons {
 174 |             display: flex;
 175 |             gap: 15px;
 176 |         }
 177 | 
 178 |         .nav-btn {
 179 |             padding: 12px 28px;
 180 |             border: none;
 181 |             border-radius: 25px;
 182 |             font-size: 14px;
 183 |             font-weight: 600;
 184 |             cursor: pointer;
 185 |             transition: all 0.3s ease;
 186 |             backdrop-filter: blur(10px);
 187 |             letter-spacing: 0.5px;
 188 |         }
 189 | 
 190 |         .btn-start {
 191 |             background: linear-gradient(135deg, #00d4ff, #0096ff);
 192 |             color: white;
 193 |             box-shadow: 0 4px 15px rgba(0, 200, 255, 0.3);
 194 |         }
 195 | 
 196 |         .btn-start:hover:not(:disabled) {
 197 |             transform: translateY(-2px);
 198 |             box-shadow: 0 6px 25px rgba(0, 200, 255, 0.5);
 199 |         }
 200 | 
 201 |         .btn-stop {
 202 |             background: rgba(255, 80, 80, 0.2);
 203 |             color: #ff5050;
 204 |             border: 1px solid rgba(255, 80, 80, 0.3);
 205 |         }
 206 | 
 207 |         .btn-stop:hover:not(:disabled) {
 208 |             background: rgba(255, 80, 80, 0.3);
 209 |             transform: translateY(-2px);
 210 |         }
 211 | 
 212 |         .btn-config, .btn-transcripts {
 213 |             background: rgba(255, 255, 255, 0.1);
 214 |             color: white;
 215 |             border: 1px solid rgba(255, 255, 255, 0.2);
 216 |         }
 217 | 
 218 |         .btn-config:hover, .btn-transcripts:hover {
 219 |             background: rgba(255, 255, 255, 0.15);
 220 |             transform: translateY(-2px);
 221 |         }
 222 | 
 223 |         .nav-btn:disabled {
 224 |             opacity: 0.4;
 225 |             cursor: not-allowed;
 226 |         }
 227 | 
 228 |         /* Main Content */
 229 |         .main-content {
 230 |             position: relative;
 231 |             height: 100vh;
 232 |             display: flex;
 233 |             flex-direction: column;
 234 |             align-items: center;
 235 |             justify-content: center;
 236 |             z-index: 1;
 237 |             padding: 100px 20px 120px;
 238 |         }
 239 | 
 240 |         .ai-presence {
 241 |             position: relative;
 242 |             width: 300px;
 243 |             height: 300px;
 244 |             margin-bottom: 40px;
 245 |         }
 246 | 
 247 |         .ai-orb {
 248 |             position: absolute;
 249 |             top: 50%;
 250 |             left: 50%;
 251 |             transform: translate(-50%, -50%);
 252 |             width: 200px;
 253 |             height: 200px;
 254 |             background: radial-gradient(circle, rgba(0, 212, 255, 0.3), transparent);
 255 |             border-radius: 50%;
 256 |             display: flex;
 257 |             align-items: center;
 258 |             justify-content: center;
 259 |             transition: all 0.5s ease;
 260 |         }
 261 | 
 262 |         .ai-orb.listening {
 263 |             animation: pulse 2s infinite;
 264 |             background: radial-gradient(circle, rgba(0, 255, 200, 0.4), transparent);
 265 |         }
 266 | 
 267 |         @keyframes pulse {
 268 |             0%, 100% { transform: translate(-50%, -50%) scale(1); opacity: 0.8; }
 269 |             50% { transform: translate(-50%, -50%) scale(1.1); opacity: 1; }
 270 |         }
 271 | 
 272 |         .microphone-icon {
 273 |             font-size: 80px;
 274 |             color: rgba(0, 212, 255, 0.8);
 275 |             transition: all 0.3s ease;
 276 |         }
 277 | 
 278 |         .ai-orb.listening .microphone-icon {
 279 |             color: rgba(0, 255, 200, 1);
 280 |             animation: float-icon 3s infinite;
 281 |         }
 282 | 
 283 |         @keyframes float-icon {
 284 |             0%, 100% { transform: translateY(0); }
 285 |             50% { transform: translateY(-10px); }
 286 |         }
 287 | 
 288 |         .ripple {
 289 |             position: absolute;
 290 |             top: 50%;
 291 |             left: 50%;
 292 |             transform: translate(-50%, -50%);
 293 |             width: 200px;
 294 |             height: 200px;
 295 |             border: 2px solid rgba(0, 212, 255, 0.4);
 296 |             border-radius: 50%;
 297 |             animation: ripple 2s infinite;
 298 |         }
 299 | 
 300 |         .ripple:nth-child(2) { animation-delay: 0.5s; }
 301 |         .ripple:nth-child(3) { animation-delay: 1s; }
 302 | 
 303 |         @keyframes ripple {
 304 |             0% { width: 200px; height: 200px; opacity: 1; }
 305 |             100% { width: 400px; height: 400px; opacity: 0; }
 306 |         }
 307 | 
 308 |         .status-text {
 309 |             font-size: 28px;
 310 |             color: white;
 311 |             text-align: center;
 312 |             font-weight: 300;
 313 |             letter-spacing: 2px;
 314 |             text-shadow: 0 2px 10px rgba(0, 0, 0, 0.5);
 315 |             margin-bottom: 20px;
 316 |         }
 317 | 
 318 |         .subtitle {
 319 |             font-size: 16px;
 320 |             color: rgba(255, 255, 255, 0.6);
 321 |             text-align: center;
 322 |             letter-spacing: 1px;
 323 |         }
 324 | 
 325 |         /* Configuration Panel */
 326 |         .config-panel {
 327 |             position: fixed;
 328 |             top: 80px;
 329 |             right: -450px;
 330 |             width: 400px;
 331 |             height: calc(100vh - 80px);
 332 |             background: rgba(20, 30, 40, 0.95);
 333 |             backdrop-filter: blur(20px);
 334 |             border-left: 1px solid rgba(255, 255, 255, 0.1);
 335 |             transition: right 0.4s cubic-bezier(0.4, 0, 0.2, 1);
 336 |             z-index: 999;
 337 |             overflow-y: auto;
 338 |             padding: 30px;
 339 |         }
 340 | 
 341 |         .config-panel.active {
 342 |             right: 0;
 343 |         }
 344 | 
 345 |         /* Transcripts Panel */
 346 |         .transcripts-panel {
 347 |             position: fixed;
 348 |             top: 80px;
 349 |             left: -500px;
 350 |             width: 450px;
 351 |             height: calc(100vh - 80px);
 352 |             background: rgba(20, 30, 40, 0.95);
 353 |             backdrop-filter: blur(20px);
 354 |             border-right: 1px solid rgba(255, 255, 255, 0.1);
 355 |             transition: left 0.4s cubic-bezier(0.4, 0, 0.2, 1);
 356 |             z-index: 999;
 357 |             display: flex;
 358 |             flex-direction: column;
 359 |         }
 360 | 
 361 |         .transcripts-panel.active {
 362 |             left: 0;
 363 |         }
 364 | 
 365 |         .panel-header {
 366 |             padding: 30px 30px 20px;
 367 |             border-bottom: 1px solid rgba(255, 255, 255, 0.1);
 368 |         }
 369 | 
 370 |         .panel-title {
 371 |             color: white;
 372 |             font-size: 24px;
 373 |             margin-bottom: 10px;
 374 |             font-weight: 600;
 375 |             display: flex;
 376 |             align-items: center;
 377 |             gap: 10px;
 378 |         }
 379 | 
 380 |         .panel-subtitle {
 381 |             color: rgba(255, 255, 255, 0.5);
 382 |             font-size: 13px;
 383 |             letter-spacing: 0.5px;
 384 |         }
 385 | 
 386 |         .transcripts-container {
 387 |             flex: 1;
 388 |             overflow-y: auto;
 389 |             padding: 20px 30px;
 390 |         }
 391 | 
 392 |         .transcript-item {
 393 |             margin-bottom: 20px;
 394 |             animation: slideIn 0.3s ease-out;
 395 |         }
 396 | 
 397 |         @keyframes slideIn {
 398 |             from {
 399 |                 opacity: 0;
 400 |                 transform: translateX(-20px);
 401 |             }
 402 |             to {
 403 |                 opacity: 1;
 404 |                 transform: translateX(0);
 405 |             }
 406 |         }
 407 | 
 408 |         .transcript-header {
 409 |             display: flex;
 410 |             align-items: center;
 411 |             gap: 10px;
 412 |             margin-bottom: 8px;
 413 |         }
 414 | 
 415 |         .transcript-role {
 416 |             font-size: 12px;
 417 |             font-weight: 600;
 418 |             text-transform: uppercase;
 419 |             letter-spacing: 1px;
 420 |             padding: 4px 12px;
 421 |             border-radius: 12px;
 422 |         }
 423 | 
 424 |         .transcript-role.user {
 425 |             background: rgba(0, 212, 255, 0.2);
 426 |             color: #00d4ff;
 427 |         }
 428 | 
 429 |         .transcript-role.assistant {
 430 |             background: rgba(0, 255, 136, 0.2);
 431 |             color: #00ff88;
 432 |         }
 433 | 
 434 |         .transcript-time {
 435 |             font-size: 11px;
 436 |             color: rgba(255, 255, 255, 0.4);
 437 |             font-family: monospace;
 438 |         }
 439 | 
 440 |         .transcript-emotion {
 441 |             font-size: 11px;
 442 |             background: rgba(255, 200, 0, 0.2);
 443 |             color: #ffc800;
 444 |             padding: 2px 8px;
 445 |             border-radius: 8px;
 446 |             margin-left: auto;
 447 |         }
 448 | 
 449 |         .transcript-text {
 450 |             color: rgba(255, 255, 255, 0.9);
 451 |             font-size: 14px;
 452 |             line-height: 1.6;
 453 |             padding: 12px 15px;
 454 |             background: rgba(255, 255, 255, 0.03);
 455 |             border-radius: 8px;
 456 |             border-left: 3px solid rgba(0, 212, 255, 0.3);
 457 |         }
 458 | 
 459 |         .transcript-item.assistant .transcript-text {
 460 |             border-left-color: rgba(0, 255, 136, 0.3);
 461 |         }
 462 | 
 463 |         .transcript-empty {
 464 |             text-align: center;
 465 |             color: rgba(255, 255, 255, 0.3);
 466 |             padding: 60px 20px;
 467 |             font-size: 14px;
 468 |         }
 469 | 
 470 |         .config-title {
 471 |             color: white;
 472 |             font-size: 24px;
 473 |             margin-bottom: 30px;
 474 |             font-weight: 600;
 475 |         }
 476 | 
 477 |         .config-group {
 478 |             margin-bottom: 25px;
 479 |         }
 480 | 
 481 |         .config-label {
 482 |             color: rgba(255, 255, 255, 0.7);
 483 |             font-size: 12px;
 484 |             text-transform: uppercase;
 485 |             letter-spacing: 1px;
 486 |             margin-bottom: 8px;
 487 |             display: block;
 488 |         }
 489 | 
 490 |         .config-input {
 491 |             width: 100%;
 492 |             padding: 12px 15px;
 493 |             background: rgba(255, 255, 255, 0.05);
 494 |             border: 1px solid rgba(255, 255, 255, 0.1);
 495 |             border-radius: 8px;
 496 |             color: white;
 497 |             font-family: monospace;
 498 |             font-size: 13px;
 499 |             transition: all 0.3s ease;
 500 |         }
 501 | 
 502 |         .config-input:focus {
 503 |             outline: none;
 504 |             background: rgba(255, 255, 255, 0.08);
 505 |             border-color: rgba(0, 212, 255, 0.5);
 506 |             box-shadow: 0 0 15px rgba(0, 212, 255, 0.2);
 507 |         }
 508 | 
 509 |         /* Toast Notifications */
 510 |         .toast-container {
 511 |             position: fixed;
 512 |             bottom: 30px;
 513 |             left: 50%;
 514 |             transform: translateX(-50%);
 515 |             display: flex;
 516 |             flex-direction: column-reverse;
 517 |             gap: 10px;
 518 |             z-index: 2000;
 519 |             pointer-events: none;
 520 |         }
 521 | 
 522 |         .toast {
 523 |             background: rgba(0, 0, 0, 0.8);
 524 |             backdrop-filter: blur(10px);
 525 |             color: white;
 526 |             padding: 15px 25px;
 527 |             border-radius: 12px;
 528 |             border: 1px solid rgba(255, 255, 255, 0.1);
 529 |             box-shadow: 0 4px 20px rgba(0, 0, 0, 0.5);
 530 |             display: flex;
 531 |             align-items: center;
 532 |             gap: 12px;
 533 |             animation: slideUp 0.4s ease-out;
 534 |             min-width: 300px;
 535 |         }
 536 | 
 537 |         @keyframes slideUp {
 538 |             from {
 539 |                 transform: translateY(50px);
 540 |                 opacity: 0;
 541 |             }
 542 |             to {
 543 |                 transform: translateY(0);
 544 |                 opacity: 1;
 545 |             }
 546 |         }
 547 | 
 548 |         .toast-icon {
 549 |             font-size: 20px;
 550 |         }
 551 | 
 552 |         .toast-message {
 553 |             font-size: 14px;
 554 |             letter-spacing: 0.5px;
 555 |         }
 556 | 
 557 |         .toast.info { border-left: 3px solid #00d4ff; }
 558 |         .toast.success { border-left: 3px solid #00ff88; }
 559 |         .toast.error { border-left: 3px solid #ff5050; }
 560 | 
 561 |         /* Clear Button */
 562 |         .clear-transcripts-btn {
 563 |             margin: 0 30px 20px;
 564 |             padding: 10px 20px;
 565 |             background: rgba(255, 80, 80, 0.2);
 566 |             color: #ff5050;
 567 |             border: 1px solid rgba(255, 80, 80, 0.3);
 568 |             border-radius: 8px;
 569 |             cursor: pointer;
 570 |             font-size: 13px;
 571 |             font-weight: 600;
 572 |             transition: all 0.3s ease;
 573 |         }
 574 | 
 575 |         .clear-transcripts-btn:hover {
 576 |             background: rgba(255, 80, 80, 0.3);
 577 |             transform: translateY(-1px);
 578 |         }
 579 |     </style>
 580 | </head>
 581 | <body>
 582 |     <!-- Animated Background -->
 583 |     <div class="background">
 584 |         <div class="city-silhouette">
 585 |             <div class="building building-1"></div>
 586 |             <div class="building building-2"></div>
 587 |             <div class="building building-3"></div>
 588 |             <div class="building building-4"></div>
 589 |             <div class="building building-5"></div>
 590 |             <div class="building building-6"></div>
 591 |             <div class="building building-7"></div>
 592 |         </div>
 593 |         <div class="particles" id="particles"></div>
 594 |     </div>
 595 | 
 596 |     <!-- Emotion Display -->
 597 |     <div class="emotion-display" id="emotionDisplay">
 598 |         <div class="emotion-emoji" id="emotionEmoji">😊</div>
 599 |         <div class="emotion-label" id="emotionLabel">Happy</div>
 600 |     </div>
 601 | 
 602 |     <!-- Top Navigation Bar -->
 603 |     <div class="top-bar">
 604 |         <div class="logo">
 605 |             <div class="logo-icon">🏢</div>
 606 |             <span>Central City Mall AI</span>
 607 |         </div>
 608 |         <div class="nav-buttons">
 609 |             <button class="nav-btn btn-start" id="startBtn">▶ Start Conversation</button>
 610 |             <button class="nav-btn btn-stop" id="stopBtn" disabled>⏹ Stop Conversation</button>
 611 |             <button class="nav-btn btn-transcripts" id="transcriptsBtn">📝 Transcripts</button>
 612 |             <button class="nav-btn btn-config" id="configBtn">⚙ Configuration</button>
 613 |         </div>
 614 |     </div>
 615 | 
 616 |     <!-- Main Content -->
 617 |     <div class="main-content">
 618 |         <div class="ai-presence">
 619 |             <div class="ai-orb" id="aiOrb">
 620 |                 <div class="ripple"></div>
 621 |                 <div class="ripple"></div>
 622 |                 <div class="ripple"></div>
 623 |                 <div class="microphone-icon">🎙️</div>
 624 |             </div>
 625 |         </div>
 626 |         <div class="status-text" id="statusText">Ready to Assist</div>
 627 |         <div class="subtitle">Your intelligent mall guide is ready</div>
 628 |     </div>
 629 | 
 630 |     <!-- Transcripts Panel -->
 631 |     <div class="transcripts-panel" id="transcriptsPanel">
 632 |         <div class="panel-header">
 633 |             <div class="panel-title">
 634 |                 <span>📝</span>
 635 |                 <span>Live Transcripts</span>
 636 |             </div>
 637 |             <div class="panel-subtitle">Real-time conversation with emotions</div>
 638 |         </div>
 639 |         <div class="transcripts-container" id="transcriptsContainer">
 640 |             <div class="transcript-empty">No transcripts yet. Start a conversation to see them here.</div>
 641 |         </div>
 642 |         <button class="clear-transcripts-btn" id="clearTranscriptsBtn">🗑️ Clear Transcripts</button>
 643 |     </div>
 644 | 
 645 |     <!-- Configuration Panel -->
 646 |     <div class="config-panel" id="configPanel">
 647 |         <div class="config-title">⚙ Configuration</div>
 648 | 
 649 |         <div class="config-group">
 650 |             <label class="config-label">App ID</label>
 651 |             <input type="text" class="config-input" id="appId" value="550749b706214846a1a2eef3612a8cd3">
 652 |         </div>
 653 | 
 654 |         <div class="config-group">
 655 |             <label class="config-label">Channel Name</label>
 656 |             <input type="text" class="config-input" id="channel" value="test">
 657 |         </div>
 658 | 
 659 |         <div class="config-group">
 660 |             <label class="config-label">User UID</label>
 661 |             <input type="text" class="config-input" id="userUid" value="1002">
 662 |         </div>
 663 | 
 664 |         <div class="config-group">
 665 |             <label class="config-label">Token</label>
 666 |             <input type="text" class="config-input" id="token" value="007eJxTYJj6sdD/Qn3N4rXlwj/XzzxxVPZDzYnEr2bpf9MMbJoqKrgUGExNDcxNLJPMDcyMDE0sTMwSDRONUlPTjM0MjRItklOM/W97ZArwMTAIewsxMzEwMrAAMYjPBCaZwSQLlCxJLS5hYTA0MDACqWSEioL4AAZDIBw=">
 667 |         </div>
 668 |     </div>
 669 | 
 670 |     <!-- Toast Container -->
 671 |     <div class="toast-container" id="toastContainer"></div>
 672 | 
 673 |     <!-- Agora SDK -->
 674 |     <script src="https://download.agora.io/sdk/release/AgoraRTC_N-4.20.0.js"></script>
 675 |     <script src="./agora-rtm-2.2.3.min.js"></script>
 676 | 
 677 |     <script>
 678 |         // Emotion mapping configuration
 679 |         const EMOTION_CONFIG = {
 680 |             happy: { emoji: '😊', label: 'Happy', keywords: ['special offer', 'discount', 'sale', 'great', 'wonderful', 'excellent', 'amazing', 'fantastic', 'perfect', 'love', 'enjoy'] },
 681 |             excited: { emoji: '🎉', label: 'Excited', keywords: ['exciting', 'wow', 'incredible', 'awesome', 'celebration', 'congratulations'] },
 682 |             surprised: { emoji: '😲', label: 'Surprised', keywords: ['surprise', 'unexpected', 'amazing', 'wow', 'believe', 'incredible'] },
 683 |             thinking: { emoji: '🤔', label: 'Thinking', keywords: ['let me', 'checking', 'look up', 'finding', 'searching', 'moment'] },
 684 |             helpful: { emoji: '🙋', label: 'Helpful', keywords: ['help', 'assist', 'guide', 'show', 'direct', 'located'] },
 685 |             neutral: { emoji: '😐', label: 'Neutral', keywords: ['located', 'floor', 'available', 'open', 'hours'] },
 686 |             sad: { emoji: '😔', label: 'Sad', keywords: ['sorry', 'unfortunately', 'closed', 'unavailable', 'apologize'] },
 687 |             welcoming: { emoji: '👋', label: 'Welcome', keywords: ['hello', 'welcome', 'hi', 'greetings', 'good morning', 'good afternoon'] }
 688 |         };
 689 | 
 690 |         // Sentiment analysis function
 691 |         function analyzeSentiment(text) {
 692 |             if (!text) return null;
 693 | 
 694 |             const lowerText = text.toLowerCase();
 695 |             let bestMatch = { emotion: 'neutral', score: 0 };
 696 | 
 697 |             // Check each emotion for keyword matches
 698 |             for (const [emotion, config] of Object.entries(EMOTION_CONFIG)) {
 699 |                 let score = 0;
 700 |                 for (const keyword of config.keywords) {
 701 |                     if (lowerText.includes(keyword)) {
 702 |                         score += 1;
 703 |                     }
 704 |                 }
 705 | 
 706 |                 if (score > bestMatch.score) {
 707 |                     bestMatch = { emotion, score };
 708 |                 }
 709 |             }
 710 | 
 711 |             // If no keywords matched, analyze text patterns
 712 |             if (bestMatch.score === 0) {
 713 |                 if (lowerText.includes('!')) bestMatch.emotion = 'excited';
 714 |                 else if (lowerText.includes('?')) bestMatch.emotion = 'thinking';
 715 |             }
 716 | 
 717 |             return bestMatch.emotion;
 718 |         }
 719 | 
 720 |         // Display emotion on screen
 721 |         function displayEmotion(emotion) {
 722 |             const emotionDisplay = document.getElementById('emotionDisplay');
 723 |             const emotionEmoji = document.getElementById('emotionEmoji');
 724 |             const emotionLabel = document.getElementById('emotionLabel');
 725 | 
 726 |             const config = EMOTION_CONFIG[emotion] || EMOTION_CONFIG.neutral;
 727 | 
 728 |             emotionEmoji.textContent = config.emoji;
 729 |             emotionLabel.textContent = config.label;
 730 | 
 731 |             // Show emotion
 732 |             emotionDisplay.classList.remove('active');
 733 |             setTimeout(() => {
 734 |                 emotionDisplay.classList.add('active');
 735 |             }, 50);
 736 | 
 737 |             // Hide after 4 seconds
 738 |             setTimeout(() => {
 739 |                 emotionDisplay.classList.remove('active');
 740 |             }, 4000);
 741 | 
 742 |             console.log(`🎭 Emotion detected: ${emotion} (${config.label})`);
 743 | 
 744 |             // Send emotion to LED control service
 745 |             sendEmotionToLED(emotion);
 746 |         }
 747 | 
 748 |         // Send emotion to LED control service
 749 |         async function sendEmotionToLED(emotion) {
 750 |             try {
 751 |                 const response = await fetch('http://localhost:5000/emotion', {
 752 |                     method: 'POST',
 753 |                     headers: {
 754 |                         'Content-Type': 'application/json',
 755 |                     },
 756 |                     body: JSON.stringify({
 757 |                         emotion: emotion,
 758 |                         duration: 1.0
 759 |                     })
 760 |                 });
 761 | 
 762 |                 if (response.ok) {
 763 |                     const data = await response.json();
 764 |                     console.log(`💡 LED animation triggered: ${data.message}`);
 765 |                 } else {
 766 |                     console.log('⚠️ LED service not available');
 767 |                 }
 768 |             } catch (error) {
 769 |                 // Silently fail if LED service is not running
 770 |                 console.log('💡 LED service offline (optional)');
 771 |             }
 772 |         }
 773 | 
 774 |         // Extract clean text (remove emotion labels in brackets)
 775 |         function extractCleanText(text) {
 776 |             // Remove content in brackets: (emotion), [emotion], {emotion}, （emotion）, 【emotion】
 777 |             return text
 778 |                 .replace(/\([^)]*\)/g, '')
 779 |                 .replace(/\[[^\]]*\]/g, '')
 780 |                 .replace(/\{[^}]*\}/g, '')
 781 |                 .replace(/（[^）]*）/g, '')
 782 |                 .replace(/【[^】]*】/g, '')
 783 |                 .trim();
 784 |         }
 785 | 
 786 |         // Extract emotion from bracketed labels
 787 |         function extractEmotionLabel(text) {
 788 |             const patterns = [
 789 |                 /\(([^)]+)\)/,    // (emotion)
 790 |                 /\[([^\]]+)\]/,   // [emotion]
 791 |                 /\{([^}]+)\}/,    // {emotion}
 792 |                 /（([^）]+)）/,    // （emotion）
 793 |                 /【([^】]+)】/     // 【emotion】
 794 |             ];
 795 | 
 796 |             for (const pattern of patterns) {
 797 |                 const match = text.match(pattern);
 798 |                 if (match) {
 799 |                     const label = match[1].toLowerCase().trim();
 800 |                     // Check if it matches our emotion keywords
 801 |                     for (const [emotion, config] of Object.entries(EMOTION_CONFIG)) {
 802 |                         if (label.includes(emotion) || config.keywords.some(k => label.includes(k))) {
 803 |                             return emotion;
 804 |                         }
 805 |                     }
 806 |                 }
 807 |             }
 808 |             return null;
 809 |         }
 810 | 
 811 |         // Generate floating particles
 812 |         const particlesContainer = document.getElementById('particles');
 813 |         for (let i = 0; i < 30; i++) {
 814 |             const particle = document.createElement('div');
 815 |             particle.className = 'particle';
 816 |             particle.style.left = Math.random() * 100 + '%';
 817 |             particle.style.animationDelay = Math.random() * 15 + 's';
 818 |             particle.style.animationDuration = (15 + Math.random() * 10) + 's';
 819 |             particlesContainer.appendChild(particle);
 820 |         }
 821 | 
 822 |         // Toast notification system
 823 |         function showToast(message, type = 'info') {
 824 |             const container = document.getElementById('toastContainer');
 825 |             const toast = document.createElement('div');
 826 |             toast.className = `toast ${type}`;
 827 | 
 828 |             const icons = {
 829 |                 info: 'ℹ️',
 830 |                 success: '✅',
 831 |                 error: '❌'
 832 |             };
 833 | 
 834 |             toast.innerHTML = `
 835 |                 <span class="toast-icon">${icons[type]}</span>
 836 |                 <span class="toast-message">${message}</span>
 837 |             `;
 838 | 
 839 |             container.appendChild(toast);
 840 | 
 841 |             setTimeout(() => {
 842 |                 toast.style.animation = 'slideUp 0.4s ease-out reverse';
 843 |                 setTimeout(() => toast.remove(), 400);
 844 |             }, 3000);
 845 |         }
 846 | 
 847 |         // Panel toggles
 848 |         const configBtn = document.getElementById('configBtn');
 849 |         const configPanel = document.getElementById('configPanel');
 850 |         const transcriptsBtn = document.getElementById('transcriptsBtn');
 851 |         const transcriptsPanel = document.getElementById('transcriptsPanel');
 852 | 
 853 |         configBtn.addEventListener('click', () => {
 854 |             configPanel.classList.toggle('active');
 855 |             if (transcriptsPanel.classList.contains('active')) {
 856 |                 transcriptsPanel.classList.remove('active');
 857 |             }
 858 |         });
 859 | 
 860 |         transcriptsBtn.addEventListener('click', () => {
 861 |             transcriptsPanel.classList.toggle('active');
 862 |             if (configPanel.classList.contains('active')) {
 863 |                 configPanel.classList.remove('active');
 864 |             }
 865 |         });
 866 | 
 867 |         // Transcript Management
 868 |         const transcriptsContainer = document.getElementById('transcriptsContainer');
 869 |         const clearTranscriptsBtn = document.getElementById('clearTranscriptsBtn');
 870 | 
 871 |         function addTranscript(role, text, emotion = null, timestamp = null) {
 872 |             // Remove empty state
 873 |             const emptyState = transcriptsContainer.querySelector('.transcript-empty');
 874 |             if (emptyState) {
 875 |                 emptyState.remove();
 876 |             }
 877 | 
 878 |             const time = timestamp || new Date().toLocaleTimeString();
 879 |             const cleanText = extractCleanText(text);
 880 | 
 881 |             const transcriptItem = document.createElement('div');
 882 |             transcriptItem.className = `transcript-item ${role}`;
 883 | 
 884 |             const emotionBadge = emotion ? `<span class="transcript-emotion">${EMOTION_CONFIG[emotion]?.emoji || ''} ${emotion}</span>` : '';
 885 | 
 886 |             transcriptItem.innerHTML = `
 887 |                 <div class="transcript-header">
 888 |                     <span class="transcript-role ${role}">${role === 'user' ? '👤 You' : '🤖 Assistant'}</span>
 889 |                     <span class="transcript-time">${time}</span>
 890 |                     ${emotionBadge}
 891 |                 </div>
 892 |                 <div class="transcript-text">${cleanText}</div>
 893 |             `;
 894 | 
 895 |             transcriptsContainer.appendChild(transcriptItem);
 896 |             transcriptsContainer.scrollTop = transcriptsContainer.scrollHeight;
 897 | 
 898 |             // Also log to console
 899 |             console.log(`[${time}] ${role.toUpperCase()}${emotion ? ` [${emotion}]` : ''}: ${cleanText}`);
 900 |         }
 901 | 
 902 |         clearTranscriptsBtn.addEventListener('click', () => {
 903 |             transcriptsContainer.innerHTML = '<div class="transcript-empty">No transcripts yet. Start a conversation to see them here.</div>';
 904 |             showToast('Transcripts cleared', 'info');
 905 |         });
 906 | 
 907 |         // Agora integration
 908 |         (function() {
 909 |             let checkCount = 0;
 910 |             const maxChecks = 50;
 911 | 
 912 |             function checkAgoraLoaded() {
 913 |                 checkCount++;
 914 |                 if (typeof AgoraRTC !== 'undefined' && typeof AgoraRTM !== 'undefined') {
 915 |                     showToast('Agora SDKs loaded successfully', 'success');
 916 |                     initApp();
 917 |                 } else if (checkCount < maxChecks) {
 918 |                     setTimeout(checkAgoraLoaded, 100);
 919 |                 } else {
 920 |                     showToast('Failed to load Agora SDKs', 'error');
 921 |                 }
 922 |             }
 923 | 
 924 |             if (document.readyState === 'loading') {
 925 |                 document.addEventListener('DOMContentLoaded', checkAgoraLoaded);
 926 |             } else {
 927 |                 checkAgoraLoaded();
 928 |             }
 929 |         })();
 930 | 
 931 |         function initApp() {
 932 |             let client = null;
 933 |             let localAudioTrack = null;
 934 |             let rtmClient = null;
 935 |             let rtmChannel = null;
 936 |             let isConnected = false;
 937 | 
 938 |             const startBtn = document.getElementById('startBtn');
 939 |             const stopBtn = document.getElementById('stopBtn');
 940 |             const statusText = document.getElementById('statusText');
 941 |             const aiOrb = document.getElementById('aiOrb');
 942 | 
 943 |             const appIdInput = document.getElementById('appId');
 944 |             const channelInput = document.getElementById('channel');
 945 |             const tokenInput = document.getElementById('token');
 946 |             const userUidInput = document.getElementById('userUid');
 947 | 
 948 |             function getConfig() {
 949 |                 return {
 950 |                     appId: appIdInput.value.trim(),
 951 |                     channel: channelInput.value.trim(),
 952 |                     token: tokenInput.value.trim(),
 953 |                     userUid: userUidInput.value.trim()
 954 |                 };
 955 |             }
 956 | 
 957 |             // Track last message to prevent duplicates
 958 |             let lastTranscriptText = '';
 959 |             let transcriptDebounceTimer = null;
 960 | 
 961 |             async function setupRTM(config) {
 962 |                 try {
 963 |                     console.log('🔌 Setting up RTM for transcripts...');
 964 | 
 965 |                     // Create RTM client
 966 |                     rtmClient = new AgoraRTM.RTM(config.appId, config.userUid, {
 967 |                         token: config.token
 968 |                     });
 969 | 
 970 |                     // Login to RTM
 971 |                     await rtmClient.login({token:"xxxxxxxxxxxxxxxxxxxxxxxxxxxxx"});
 972 |                     console.log('✅ RTM login successful');
 973 |                     showToast('Connected to transcript service', 'success');
 974 | 
 975 |                     // Subscribe to channel
 976 |                     rtmChannel = await rtmClient.subscribe(config.channel);
 977 |                     console.log(`✅ Subscribed to RTM channel: ${config.channel}`);
 978 | 
 979 |                     // Listen for transcript messages
 980 |                     rtmClient.addEventListener('message', (event) => {
 981 |                         console.log('📨 RTM Message received:', event);
 982 | 
 983 |                         if (event.channelName === config.channel) {
 984 |                             try {
 985 |                                 // Handle both string and object message formats
 986 |                                 let message;
 987 |                                 if (typeof event.message === 'string') {
 988 |                                     message = JSON.parse(event.message);
 989 |                                 } else {
 990 |                                     message = event.message;
 991 |                                 }
 992 | 
 993 |                                 console.log('📝 Parsed transcript:', message);
 994 | 
 995 |                                 // Check if it's an assistant transcription message
 996 |                                 if (message.object === 'assistant.transcription' && message.text) {
 997 |                                     const text = message.text;
 998 |                                     const dataType = message.data_type;
 999 | 
1000 |                                     console.log(`📝 Transcript received - Type: ${dataType}, Text: "${text}"`);
1001 | 
1002 |                                     // Skip if this is just an incremental update (ends with ., .., or ...)
1003 |                                     // and is very similar to the last message
1004 |                                     if (lastTranscriptText && text.startsWith(lastTranscriptText.replace(/\.+$/, ''))) {
1005 |                                         console.log('⏭️ Skipping incremental update');
1006 |                                         lastTranscriptText = text;
1007 | 
1008 |                                         // Clear previous timer
1009 |                                         if (transcriptDebounceTimer) {
1010 |                                             clearTimeout(transcriptDebounceTimer);
1011 |                                         }
1012 | 
1013 |                                         // Set new timer to show final message after 1 second of no updates
1014 |                                         transcriptDebounceTimer = setTimeout(() => {
1015 |                                             processTranscript(text);
1016 |                                         }, 1000);
1017 | 
1018 |                                         return;
1019 |                                     }
1020 | 
1021 |                                     // Clear any pending timer
1022 |                                     if (transcriptDebounceTimer) {
1023 |                                         clearTimeout(transcriptDebounceTimer);
1024 |                                         transcriptDebounceTimer = null;
1025 |                                     }
1026 | 
1027 |                                     // Process new/complete transcript immediately
1028 |                                     lastTranscriptText = text;
1029 |                                     processTranscript(text);
1030 |                                 }
1031 |                                 // Handle other message types if needed
1032 |                                 else if (message.type === 'transcripts') {
1033 |                                     console.log('📝 Alternative transcript format:', message);
1034 |                                 }
1035 |                             } catch (error) {
1036 |                                 console.error('❌ Error parsing transcript:', error);
1037 |                                 console.error('Raw message:', event.message);
1038 |                             }
1039 |                         }
1040 |                     });
1041 | 
1042 |                     // Helper function to process transcript
1043 |                     function processTranscript(text) {
1044 |                         // Extract emotion from label if present
1045 |                         let detectedEmotion = extractEmotionLabel(text);
1046 | 
1047 |                         // If no label, analyze sentiment
1048 |                         if (!detectedEmotion) {
1049 |                             detectedEmotion = analyzeSentiment(text);
1050 |                         }
1051 | 
1052 |                         console.log(`🎭 Detected emotion: ${detectedEmotion}`);
1053 | 
1054 |                         // Display emotion at top of screen
1055 |                         if (detectedEmotion) {
1056 |                             displayEmotion(detectedEmotion);
1057 |                         }
1058 | 
1059 |                         // Add to transcript panel
1060 |                         const role = 'assistant';
1061 |                         addTranscript(role, text, detectedEmotion);
1062 |                     }
1063 | 
1064 |                     // Listen for connection status
1065 |                     rtmClient.addEventListener('status', (event) => {
1066 |                         console.log('🔄 RTM Status:', event);
1067 |                     });
1068 | 
1069 |                 } catch (error) {
1070 |                     console.error('❌ RTM setup error:', error);
1071 |                     showToast('Transcript service error: ' + error.message, 'error');
1072 |                 }
1073 |             }
1074 | 
1075 |             async function joinChannel() {
1076 |                 try {
1077 |                     const config = getConfig();
1078 | 
1079 |                     statusText.textContent = 'Connecting...';
1080 |                     showToast('Initializing voice connection', 'info');
1081 | 
1082 |                     // Setup RTC
1083 |                     client = AgoraRTC.createClient({ mode: "rtc", codec: "vp8" });
1084 | 
1085 |                     client.on("user-published", async (user, mediaType) => {
1086 |                         showToast(`AI Agent connected (UID: ${user.uid})`, 'info');
1087 |                         await client.subscribe(user, mediaType);
1088 | 
1089 |                         if (mediaType === "audio") {
1090 |                             user.audioTrack.play();
1091 |                             showToast('Receiving AI response', 'success');
1092 |                         }
1093 |                     });
1094 | 
1095 |                     client.on("user-joined", (user) => {
1096 |                         showToast(`User ${user.uid} joined`, 'info');
1097 |                     });
1098 | 
1099 |                     await client.join(config.appId, config.channel, config.token, parseInt(config.userUid));
1100 |                     showToast('Channel joined successfully', 'success');
1101 | 
1102 |                     localAudioTrack = await AgoraRTC.createMicrophoneAudioTrack();
1103 |                     await client.publish([localAudioTrack]);
1104 | 
1105 |                     // Setup RTM for transcripts
1106 |                     await setupRTM(config);
1107 | 
1108 |                     showToast('Microphone active - Start speaking', 'success');
1109 | 
1110 |                     isConnected = true;
1111 |                     statusText.textContent = 'Listening...';
1112 |                     aiOrb.classList.add('listening');
1113 |                     startBtn.disabled = true;
1114 |                     stopBtn.disabled = false;
1115 | 
1116 |                 } catch (error) {
1117 |                     showToast(`Connection failed: ${error.message}`, 'error');
1118 |                     console.error('Connection error:', error);
1119 |                     statusText.textContent = 'Connection Failed';
1120 |                     startBtn.disabled = false;
1121 |                 }
1122 |             }
1123 | 
1124 |             async function leaveChannel() {
1125 |                 try {
1126 |                     showToast('Disconnecting...', 'info');
1127 | 
1128 |                     // Cleanup RTM
1129 |                     if (rtmChannel) {
1130 |                         await rtmClient.unsubscribe(channelInput.value.trim());
1131 |                         rtmChannel = null;
1132 |                     }
1133 | 
1134 |                     if (rtmClient) {
1135 |                         await rtmClient.logout();
1136 |                         rtmClient = null;
1137 |                     }
1138 | 
1139 |                     // Cleanup RTC
1140 |                     if (localAudioTrack) {
1141 |                         localAudioTrack.close();
1142 |                         localAudioTrack = null;
1143 |                     }
1144 | 
1145 |                     if (client) {
1146 |                         await client.leave();
1147 |                         client = null;
1148 |                     }
1149 | 
1150 |                     isConnected = false;
1151 |                     statusText.textContent = 'Ready to Assist';
1152 |                     aiOrb.classList.remove('listening');
1153 |                     startBtn.disabled = false;
1154 |                     stopBtn.disabled = true;
1155 | 
1156 |                     showToast('Disconnected successfully', 'success');
1157 | 
1158 |                 } catch (error) {
1159 |                     showToast(`Error: ${error.message}`, 'error');
1160 |                     console.error('Disconnect error:', error);
1161 |                 }
1162 |             }
1163 | 
1164 |             startBtn.addEventListener('click', () => {
1165 |                 startBtn.disabled = true;
1166 |                 showToast('Make sure join_api.py is running', 'info');
1167 |                 joinChannel();
1168 |             });
1169 | 
1170 |             stopBtn.addEventListener('click', () => {
1171 |                 stopBtn.disabled = true;
1172 |                 leaveChannel();
1173 |             });
1174 | 
1175 |             window.addEventListener('beforeunload', () => {
1176 |                 if (isConnected) leaveChannel();
1177 |             });
1178 |         }
1179 |     </script>
1180 | </body>
1181 | </html>
1182 | 


--------------------------------------------------------------------------------