├── .gitignore ├── .pre-commit-config.yaml ├── .python-version ├── LICENSE ├── Makefile ├── README.md ├── app ├── __init__.py ├── constants.py ├── functions.py ├── llm.py ├── main.py └── schema.py ├── assets └── logo.webp ├── env.sh ├── poetry.lock ├── pyproject.toml └── scripts ├── __init__.py └── demo.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | *.pkl 163 | *secret* 164 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.6.0 4 | hooks: 5 | - id: check-added-large-files 6 | args: ['--maxkb=600'] 7 | - id: check-yaml 8 | - id: check-json 9 | - id: check-toml 10 | - id: end-of-file-fixer 11 | - id: trailing-whitespace 12 | - id: check-docstring-first 13 | - repo: https://github.com/python-poetry/poetry 14 | rev: 1.8.0 15 | hooks: 16 | - id: poetry-check 17 | - repo: https://github.com/astral-sh/ruff-pre-commit 18 | rev: v0.4.2 19 | hooks: 20 | - id: ruff 21 | args: [ --fix ] 22 | - id: ruff-format 23 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.9.18 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Vocal AI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: checks 2 | checks: 3 | poetry run pre-commit run --all-files 4 | 5 | .PHONY: app-start 6 | app-start: 7 | poetry run uvicorn app.main:app --port=8080 --host=0.0.0.0 --workers=2 8 | 9 | .PHONY: public-url 10 | public-url: 11 | cloudflared tunnel --url http://0.0.0.0:8080 --protocol http2 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![logo](assets/logo.webp) 2 | 3 | # :robot: Proactive Voice Agent 4 | 5 | Demo of Mistral plugged into Retell, see the [demo](https://x.com/eliotthoff/status/1783980026649625032). 6 | 7 | ## Steps to run locally 8 | 9 | 1. First install dependencies 10 | 11 | ```bash 12 | poetry install 13 | ``` 14 | 15 | 2. Fill out the API keys in `env.sh` 16 | 17 | 3. In another bash, use `ngrok` or `cloudflared` to expose the port `8080` to public network. 18 | 19 | ```bash 20 | make host-url 21 | ``` 22 | 23 | 2. Update the host name and export the environment variables. 24 | 25 | ``` 26 | source env.sh 27 | ``` 28 | 29 | 4. Start the websocket server 30 | 31 | ```bash 32 | make app-start 33 | ``` 34 | 35 | The custom LLM URL (to enter in Retell) would look like 36 | `wss://henry-corrected-julia-drive.trycloudflare.com/llm-websocket` 37 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Get-Vocal/proactive-voice-agent/5309984fcb581cf5e508498666296f87632aa1a3/app/__init__.py -------------------------------------------------------------------------------- /app/constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # Global config 4 | DEBUG = False 5 | USE_ZAPIER = False 6 | 7 | # API keys 8 | MISTRAL_API_KEY = os.environ.get("MISTRAL_API_KEY") 9 | RETELL_API_KEY = os.environ.get("RETELL_API_KEY") 10 | 11 | # Mistral parameters 12 | MODEL = "mistral-large-latest" 13 | LLM_KWARGS = { 14 | "temperature": 0.2, 15 | "max_tokens": 500, 16 | } 17 | 18 | # RAG parameters 19 | TOP_K = 2 20 | 21 | # Zapier parameters 22 | HOST_NAME = os.environ.get("HOST_NAME") 23 | GET_AVAILABILITY_WEBHOOK = os.environ.get("GET_AVAILABILITY_WEBHOOK") 24 | BOOK_SLOT_WEBHOOK = os.environ.get("BOOK_SLOT_WEBHOOK") 25 | SEND_MAIL_WEBHOOK = os.environ.get("SEND_MAIL_WEBHOOK") 26 | MAX_WAIT = 15 27 | CHECK_EVERY = 0.1 28 | 29 | # Prompting 30 | SYSTEM_PROMPT = ( 31 | "You are engaging voice conversation with a patient.\n" 32 | "You have the following capabilities:\n" 33 | "1. Ask about the reason for the consultation.\n" 34 | "2. Ask about the patient's name.\n" 35 | "3. Ask about the date of the appointment.\n" 36 | "4. Use a function to get the availability of the doctor.\n" 37 | "5. Use a function to book a slot with the doctor.\n" 38 | "6. Use a function to get additional information.\n" 39 | ) 40 | REMINDER_PROMPT = "(Now the user has not responded in a while, you would say:)" 41 | ERROR_PROMPT = "An error occured" 42 | DOCUMENT_PROMPT = """## Documents 43 | {document_stack}\n 44 | """ 45 | 46 | # Hardcoded answers 47 | GREETINGS = "Hey there, I'm Ema and I work at the Dental Office, how can I help you?" 48 | -------------------------------------------------------------------------------- /app/functions.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import uuid 3 | 4 | import requests 5 | from langchain_community.vectorstores import FAISS 6 | 7 | from app.constants import ( 8 | BOOK_SLOT_WEBHOOK, 9 | DEBUG, 10 | DOCUMENT_PROMPT, 11 | ERROR_PROMPT, 12 | GET_AVAILABILITY_WEBHOOK, 13 | HOST_NAME, 14 | SEND_MAIL_WEBHOOK, 15 | TOP_K, 16 | USE_ZAPIER, 17 | ) 18 | 19 | with open("rag.pkl", "rb") as f: 20 | rag: FAISS = pickle.load(f) 21 | 22 | 23 | def get_informartion(query): 24 | print(f"[CALL] call_clinic_rag: {query}") 25 | docs = rag.similarity_search(query, k=TOP_K) 26 | document_stack = "\n###\n".join([doc.page_content for doc in docs]) 27 | return DOCUMENT_PROMPT.format(document_stack=document_stack) 28 | 29 | 30 | def get_availability(**kwargs): 31 | print("[CALL] get_availability", kwargs) 32 | if not USE_ZAPIER: 33 | return "available [you should ask for confirmation and book the slot]" 34 | 35 | hour = kwargs["hour"] - 2 36 | start_at = f"2024-04-27T{hour:02d}:00:00.000Z" 37 | end_at = f"2024-04-27T{hour:02d}:30:00.000Z" 38 | 39 | callback_id = str(uuid.uuid4()) 40 | callback_url = f"{HOST_NAME}/zapier-callback/{callback_id}" 41 | response = requests.post( 42 | url=GET_AVAILABILITY_WEBHOOK, 43 | data={ 44 | "start_at": start_at, 45 | "end_at": end_at, 46 | "callback_url": callback_url, 47 | }, 48 | ) 49 | 50 | if response.status_code != 200: 51 | return ERROR_PROMPT 52 | 53 | response = requests.get(f"{HOST_NAME}/zapier-callback-result/{callback_id}") 54 | if response.status_code != 200: 55 | return ERROR_PROMPT 56 | callback_value = response.json()["callback_value"] 57 | 58 | if callback_value.get("event") is None: 59 | return "available [you should ask for confirmation and book the slot]" 60 | else: 61 | return "busy" 62 | 63 | 64 | def send_email(**kwargs): 65 | response = requests.post( 66 | url=SEND_MAIL_WEBHOOK, 67 | data=kwargs, 68 | ) 69 | 70 | if response.status_code != 200 and DEBUG: 71 | print("Error") 72 | 73 | 74 | def book_slot(**kwargs): 75 | print("[CALL] get_availability", kwargs) 76 | if not USE_ZAPIER: 77 | return "available [you should ask for confirmation and book the slot]" 78 | 79 | hour = kwargs["hour"] - 2 80 | start_at = f"2024-04-27T{hour:02d}:00:00.000Z" 81 | end_at = f"2024-04-27T{hour:02d}:30:00.000Z" 82 | 83 | content = kwargs["conversation_summary"] 84 | subject = "Appointment booked for " + kwargs["patient_name"] 85 | response = requests.post( 86 | url=BOOK_SLOT_WEBHOOK, 87 | data={ 88 | "start_at": start_at, 89 | "end_at": end_at, 90 | "title": subject, 91 | "description": content, 92 | }, 93 | ) 94 | 95 | if response.status_code != 200: 96 | return ERROR_PROMPT 97 | 98 | send_email(subject=subject, content=content) 99 | 100 | return "booked" 101 | 102 | 103 | TOOLS = [ 104 | { 105 | "type": "function", 106 | "function": { 107 | "name": "get_informartion", 108 | "description": "Get information about the clinic.", 109 | "parameters": { 110 | "type": "object", 111 | "properties": { 112 | "query": { 113 | "type": "string", 114 | "description": "The query necessitating additionnal information.", 115 | } 116 | }, 117 | "required": ["query"], 118 | }, 119 | }, 120 | }, 121 | { 122 | "type": "function", 123 | "function": { 124 | "name": "get_availability", 125 | "description": "Get the availability of the doctor.", 126 | "parameters": { 127 | "type": "object", 128 | "properties": { 129 | "hour": { 130 | "type": "integer", 131 | "description": "The hour of the appointment.", 132 | }, 133 | "patient_name": { 134 | "type": "string", 135 | "description": "The name of the patient.", 136 | }, 137 | "reason_for_consultation": { 138 | "type": "string", 139 | "description": "The reason for the consultation.", 140 | }, 141 | }, 142 | "required": ["hour", "patient_name", "reason_for_consultation"], 143 | }, 144 | }, 145 | }, 146 | { 147 | "type": "function", 148 | "function": { 149 | "name": "book_slot", 150 | "description": "Book a slot with the doctor when a slot has been found.", 151 | "parameters": { 152 | "type": "object", 153 | "properties": { 154 | "hour": { 155 | "type": "integer", 156 | "description": "The hour of the appointment.", 157 | }, 158 | "conversation_summary": { 159 | "type": "string", 160 | "description": "The summary of the conversation.", 161 | }, 162 | "patient_name": { 163 | "type": "string", 164 | "description": "The name of the patient.", 165 | }, 166 | "reason_for_consultation": { 167 | "type": "string", 168 | "description": "The reason for the consultation.", 169 | }, 170 | }, 171 | "required": [ 172 | "hour", 173 | "conversation_summary", 174 | "patient_name", 175 | "reason_for_consultation", 176 | ], 177 | }, 178 | }, 179 | }, 180 | ] 181 | 182 | NAME_TO_FUNCTIONS = { 183 | "get_informartion": get_informartion, 184 | "get_availability": get_availability, 185 | "book_slot": book_slot, 186 | } 187 | 188 | NAME_TO_FILLER = { 189 | "get_informartion": "I will get the information for you.\n", 190 | "get_availability": "Well... Let me check if the doctor is available at that time.\n", 191 | "book_slot": "I will book the slot for you.\n", 192 | } 193 | -------------------------------------------------------------------------------- /app/llm.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from typing import List 4 | 5 | from mistralai.client import MistralClient 6 | from mistralai.models.chat_completion import ChatMessage 7 | 8 | from app.constants import DEBUG, GREETINGS, LLM_KWARGS, MISTRAL_API_KEY, MODEL, REMINDER_PROMPT, SYSTEM_PROMPT 9 | from app.functions import NAME_TO_FILLER, NAME_TO_FUNCTIONS, TOOLS 10 | from app.schema import CustomLlmRequest, CustomLlmResponse, Utterance 11 | 12 | 13 | class LlmClient: 14 | def __init__(self): 15 | self.client = MistralClient(api_key=MISTRAL_API_KEY) 16 | 17 | def greetings(self): 18 | response = CustomLlmResponse( 19 | response_id=0, 20 | content=GREETINGS, 21 | content_complete=True, 22 | end_call=False, 23 | ) 24 | return response 25 | 26 | def convert_transcript_to_messages(self, transcript: List[Utterance]): 27 | messages = [] 28 | for utterance in transcript: 29 | if utterance["role"] == "agent": 30 | messages.append(ChatMessage(role="assistant", content=utterance["content"])) 31 | else: 32 | messages.append(ChatMessage(role="user", content=utterance["content"])) 33 | return messages 34 | 35 | def prepare_prompt(self, request: CustomLlmRequest): 36 | transcript_messages = self.convert_transcript_to_messages(request.transcript) 37 | prompt = [ChatMessage(role="system", content=SYSTEM_PROMPT)] + transcript_messages 38 | 39 | if request.interaction_type == "reminder_required": 40 | prompt.append(ChatMessage(role="user", content=REMINDER_PROMPT)) 41 | return prompt 42 | 43 | def stream_response(self, request, stream=None): 44 | logger = logging.getLogger("uvicorn") 45 | if stream is None: 46 | prompt = self.prepare_prompt(request) 47 | stream = self.client.chat_stream( 48 | model=MODEL, messages=prompt, tools=TOOLS, tool_choice="auto", **LLM_KWARGS 49 | ) 50 | logged_begin = False 51 | all_tool_calls = [] 52 | full_content = "" 53 | for chunk in stream: 54 | if DEBUG: 55 | if not logged_begin: 56 | logger.info("LLM Response received") 57 | logged_begin = True 58 | # Step 3: Extract the functions 59 | if len(chunk.choices) == 0: 60 | continue 61 | if chunk.choices[0].delta.tool_calls: 62 | all_tool_calls += chunk.choices[0].delta.tool_calls 63 | 64 | # Parse transcripts 65 | if chunk.choices[0].delta.content: 66 | full_content += chunk.choices[0].delta.content 67 | response = CustomLlmResponse( 68 | response_id=request.response_id, 69 | content=chunk.choices[0].delta.content, 70 | content_complete=False, 71 | end_call=False, 72 | ) 73 | yield response 74 | 75 | # Step 4: Call the functions 76 | if all_tool_calls: 77 | tool_results = [ChatMessage(role="assistant", content="", name=None, tool_calls=all_tool_calls)] 78 | for tool_call in all_tool_calls: 79 | function_name = tool_call.function.name 80 | func_resp = NAME_TO_FILLER.get("function_name", "") 81 | response = CustomLlmResponse( 82 | response_id=request.response_id, 83 | content=func_resp, 84 | content_complete=False, 85 | end_call=False, 86 | ) 87 | yield response 88 | 89 | function_params = json.loads(tool_call.function.arguments) 90 | function_result = NAME_TO_FUNCTIONS[function_name](**function_params) 91 | tool_results.append( 92 | { 93 | "role": "tool", 94 | "content": function_result, 95 | "name": function_name, 96 | } 97 | ) 98 | 99 | stream = self.client.chat_stream( 100 | model=MODEL, messages=prompt + tool_results, tools=TOOLS, tool_choice="auto", **LLM_KWARGS 101 | ) 102 | yield from self.stream_response(request, stream) 103 | 104 | else: 105 | # No functions, complete response 106 | response = CustomLlmResponse( 107 | response_id=request.response_id, 108 | content="", 109 | content_complete=True, 110 | end_call=False, 111 | ) 112 | yield response 113 | -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | import logging 4 | from concurrent.futures import TimeoutError as ConnectionTimeoutError 5 | 6 | from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect 7 | from fastapi.responses import JSONResponse 8 | from retell import Retell 9 | 10 | from app.constants import CHECK_EVERY, DEBUG, MAX_WAIT, RETELL_API_KEY 11 | from app.llm import LlmClient 12 | from app.schema import CustomLlmRequest, CustomLlmResponse 13 | 14 | app = FastAPI() 15 | retell = Retell(api_key=RETELL_API_KEY) 16 | callbacks = {} 17 | 18 | 19 | @app.post("/zapier-callback/{callback_id}") 20 | async def zapier_callback(body: dict, callback_id: str): 21 | callbacks[callback_id] = body 22 | return {"status": "success"} 23 | 24 | 25 | @app.get("/zapier-callback-result/{callback_id}") 26 | async def zapier_callback_result(callback_id: str): 27 | wait = 0.0 28 | print("zapier_callback_result", callbacks) 29 | while callbacks.get(callback_id) is None: 30 | await asyncio.sleep(CHECK_EVERY) 31 | wait += CHECK_EVERY 32 | if wait > MAX_WAIT: 33 | print("max wait") 34 | break 35 | else: 36 | callback_value = callbacks[callback_id] 37 | del callbacks[callback_id] 38 | return {"callback_value": callback_value} 39 | return JSONResponse(status_code=500, content={"message": "Internal Server Error"}) 40 | 41 | 42 | # Only used for web frontend to register call so that frontend don't need api key 43 | @app.post("/register-call-on-your-server") 44 | async def handle_register_call(request: Request): 45 | try: 46 | post_data = await request.json() 47 | call_response = retell.call.register( 48 | agent_id=post_data["agent_id"], 49 | audio_websocket_protocol="web", 50 | audio_encoding="s16le", 51 | sample_rate=post_data["sample_rate"], # Sample rate has to be 8000 for Twilio 52 | ) 53 | if DEBUG: 54 | print(f"Call response: {call_response}") 55 | except Exception as err: 56 | if DEBUG: 57 | print(f"Error in register call: {err}") 58 | return JSONResponse(status_code=500, content={"message": "Internal Server Error"}) 59 | 60 | 61 | # Custom LLM Websocket handler, receive audio transcription and send back text response 62 | @app.websocket("/llm-websocket/{call_id}") 63 | async def websocket_handler(websocket: WebSocket, call_id: str): 64 | if DEBUG: 65 | logger = logging.getLogger("uvicorn") 66 | await websocket.accept() 67 | llm_client = LlmClient() 68 | 69 | # Send optional config to Retell server 70 | config = CustomLlmResponse( 71 | response_type="config", 72 | config={ 73 | "auto_reconnect": True, 74 | "call_details": True, 75 | }, 76 | response_id=1, 77 | ) 78 | await websocket.send_text(json.dumps(config.__dict__)) 79 | 80 | # Send first message to signal ready of server 81 | response_id = 0 82 | first_event = llm_client.greetings() 83 | await websocket.send_text(json.dumps(first_event.__dict__)) 84 | 85 | async def stream_response(request: CustomLlmRequest): 86 | nonlocal response_id 87 | for event in llm_client.stream_response(request): 88 | await websocket.send_text(json.dumps(event.__dict__)) 89 | if request.response_id < response_id: 90 | return # new response needed, abandon this one 91 | 92 | try: 93 | while True: 94 | message = await asyncio.wait_for(websocket.receive_text(), timeout=100 * 6000) # 100 minutes 95 | request_json = json.loads(message) 96 | request: CustomLlmRequest = CustomLlmRequest(**request_json) 97 | if DEBUG: 98 | logger.info("LLM Request received") 99 | 100 | # There are 5 types of interaction_type: call_details, pingpong, update_only, response_required, 101 | # and reminder_required. 102 | # Not all of them need to be handled, only response_required and reminder_required. 103 | if request.interaction_type == "call_details": 104 | continue 105 | if request.interaction_type == "ping_pong": 106 | await websocket.send_text(json.dumps({"response_type": "ping_pong", "timestamp": request.timestamp})) 107 | continue 108 | if request.interaction_type == "update_only": 109 | continue 110 | if request.interaction_type == "response_required" or request.interaction_type == "reminder_required": 111 | asyncio.create_task(stream_response(request)) 112 | except WebSocketDisconnect: 113 | print(f"LLM WebSocket disconnected for {call_id}") 114 | except ConnectionTimeoutError as e: 115 | print(f"Connection timeout error: {e}") 116 | except Exception as e: 117 | print(f"Error in LLM WebSocket: {e}") 118 | finally: 119 | print(f"LLM WebSocket connection closed for {call_id}") 120 | -------------------------------------------------------------------------------- /app/schema.py: -------------------------------------------------------------------------------- 1 | """Retell API schema""" 2 | 3 | from typing import Any, List, Literal, Optional 4 | 5 | from pydantic import BaseModel 6 | 7 | 8 | class Utterance(BaseModel): 9 | role: Literal["agent", "user", "system"] 10 | content: str 11 | 12 | 13 | class CustomLlmRequest(BaseModel): 14 | interaction_type: Literal[ 15 | "update_only", 16 | "response_required", 17 | "reminder_required", 18 | "ping_pong", 19 | "call_details", 20 | ] 21 | response_id: Optional[int] = 0 # Used by response_required and reminder_required 22 | transcript: Optional[List[Any]] = [] # Used by response_required and reminder_required 23 | call: Optional[dict] = None # Used by call_details 24 | timestamp: Optional[int] = None # Used by ping_pong 25 | 26 | 27 | class CustomLlmResponse(BaseModel): 28 | response_type: Literal["response", "config", "ping_pong"] = "response" 29 | response_id: Optional[int] = None # Used by response 30 | content: Any = None # Used by response 31 | content_complete: Optional[bool] = False # Used by response 32 | end_call: Optional[bool] = False # Used by response 33 | config: Optional[dict] = None # Used by config 34 | timestamp: Optional[int] = None # Used by ping_pong 35 | -------------------------------------------------------------------------------- /assets/logo.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Get-Vocal/proactive-voice-agent/5309984fcb581cf5e508498666296f87632aa1a3/assets/logo.webp -------------------------------------------------------------------------------- /env.sh: -------------------------------------------------------------------------------- 1 | export RETELL_API_KEY="" 2 | export MISTRAL_API_KEY="" 3 | export HOST_NAME="" 4 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | line-length = 119 3 | target-version = "py39" 4 | 5 | [tool.poetry] 6 | name = "proactive-voice-agent" 7 | version = "0.1.0" 8 | description = "" 9 | authors = [] 10 | readme = "README.md" 11 | 12 | [tool.poetry.dependencies] 13 | python = "^3.9" 14 | fastapi = {extras = ["uvicorn"], version = "^0.110.2"} 15 | mistralai = "^0.1.8" 16 | langchain-community = "^0.0.34" 17 | uvicorn = "^0.29.0" 18 | retell-sdk = "^3.11.0" 19 | sentence-transformers = "^2.7.0" 20 | faiss-cpu = "^1.8.0" 21 | 22 | [tool.poetry.group.dev] 23 | optional = true 24 | 25 | [tool.poetry.group.dev.dependencies] 26 | pre-commit = "^3.7.0" 27 | 28 | [build-system] 29 | requires = ["poetry-core"] 30 | build-backend = "poetry.core.masonry.api" 31 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Get-Vocal/proactive-voice-agent/5309984fcb581cf5e508498666296f87632aa1a3/scripts/__init__.py -------------------------------------------------------------------------------- /scripts/demo.py: -------------------------------------------------------------------------------- 1 | """Demo file to test different scenarios. 2 | 3 | Run with: 4 | ``` 5 | poetry run python -m scripts.demo 6 | ``` 7 | """ 8 | 9 | from app.llm import LlmClient 10 | from app.schema import CustomLlmRequest 11 | 12 | llm_client = LlmClient() 13 | first_event = llm_client.greetings() 14 | 15 | 16 | def complete_scenario(name, scenario): 17 | it_scenario = iter(scenario) 18 | transcript = [] 19 | print(f"\n[BEGIN] scenario: {name}") 20 | try: 21 | while True: 22 | message = next(it_scenario) 23 | print(f"Message: {message}") 24 | transcript.append({"role": "user", "content": message}) 25 | request = CustomLlmRequest(interaction_type="response_required", transcript=transcript) 26 | 27 | response = "" 28 | for event in llm_client.stream_response(request): 29 | response += event.content 30 | print(f"Response: {response}") 31 | transcript.append({"role": "agent", "content": response}) 32 | except StopIteration: 33 | print(f"[END] scenario: {name}\n") 34 | except Exception as e: 35 | print(f"[Error]: {e}") 36 | 37 | 38 | scenarios = { 39 | "final_demo": [ 40 | "Hello, I need to book an appointment", 41 | "I am having teeth pain", 42 | "I'm mister Smith", 43 | "Monday at 10am", 44 | "Okay, then on Monday at 3pm", 45 | "Yes please", 46 | "Perfect, what is the nearest metro sataion?", 47 | "By the way, could you ask my dentist if I can take a painkiller ?", 48 | ], 49 | "ask_doctor_demo": [ 50 | "Hello, I have a high teeth pain. Please ask a dentist what I should do.", 51 | ], 52 | "basic_scenario": [ 53 | "Hello, I need to book an appointment", 54 | "On Monday at 3pm", 55 | "Okay, then on Tuesday at 4pm", 56 | "Great, thanks", 57 | "Okay, bye", 58 | ], 59 | "information_scenario": [ 60 | "Hello, what is the nearest metro sataion to the clinic?", 61 | ], 62 | } 63 | 64 | for name, scenario in scenarios.items(): 65 | complete_scenario(name, scenario) 66 | --------------------------------------------------------------------------------