├── api ├── __init__.py └── router.py ├── config ├── __init__.py ├── prompts.py └── main.py ├── tools ├── __init__.py ├── definitions.py └── get_weather.py ├── utils ├── __init__.py ├── singleton.py └── stream.py ├── constants └── __init__.py ├── services ├── __init__.py ├── assistant_setup.py └── chat.py ├── .env.development ├── demo.png ├── .gitignore ├── main.py ├── requirements.txt ├── README.md └── templates └── index.html /api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /constants/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.env.development: -------------------------------------------------------------------------------- 1 | ASSISTANT_ID= 2 | OPENWEATHER_API_KEY= 3 | OPENAI_API_KEY== 4 | -------------------------------------------------------------------------------- /demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meeran03/streaming_with_function_calling_fastapi/HEAD/demo.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # pycache 2 | **/__pycache__/ 3 | .DS_Store 4 | 5 | *.pyc 6 | 7 | env 8 | .env 9 | 10 | .vscode 11 | !.vscode/launch.json 12 | -------------------------------------------------------------------------------- /config/prompts.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file will house all the prompts used in the application. 3 | """ 4 | 5 | SYS_PROMPT = """ 6 | You are an AI Assistant that tells people what activities they can do based on the weather. 7 | When responding, you don't need to provide the weather information in the response. 8 | Just depending on the overall weather, suggest the activities. 9 | """ 10 | -------------------------------------------------------------------------------- /utils/singleton.py: -------------------------------------------------------------------------------- 1 | """ 2 | Contains a Singleton Metaclass. 3 | """ 4 | 5 | class Singleton(type): 6 | """ 7 | metaclass 8 | """ 9 | _instances = {} 10 | def __call__(cls, *args, **kwargs): 11 | if cls not in cls._instances: 12 | cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) 13 | return cls._instances[cls] 14 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | """ 2 | The entry file for the FastAPI application. 3 | """ 4 | 5 | from fastapi import FastAPI 6 | from fastapi.middleware.cors import CORSMiddleware 7 | 8 | from api.router import api_router 9 | 10 | app = FastAPI(title="Activities Suggester App", version="1.0", debug=True) 11 | 12 | app.add_middleware( 13 | CORSMiddleware, 14 | allow_origins=["*"], 15 | allow_credentials=True, 16 | allow_methods=["*"], 17 | allow_headers=["*"], 18 | ) 19 | 20 | app.include_router(api_router) 21 | -------------------------------------------------------------------------------- /utils/stream.py: -------------------------------------------------------------------------------- 1 | """ 2 | Stream related utilities. 3 | """ 4 | 5 | async def stream_generator(data): 6 | """ 7 | Generator function to simulate streaming data. 8 | """ 9 | async for message in data: 10 | json_data = message 11 | if hasattr(message, 'model_dump_json'): 12 | json_data = message.model_dump_json() 13 | if isinstance(json_data, str) and json_data.startswith('data:'): 14 | yield json_data 15 | else: 16 | yield f"data: {json_data}\n\n" 17 | -------------------------------------------------------------------------------- /config/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains all project configs read from env file. 3 | """ 4 | 5 | import os 6 | 7 | class Base(object): 8 | """ 9 | Base configuration class. Contains all the default configurations. 10 | """ 11 | 12 | DEBUG: bool = True 13 | 14 | class Config(Base): 15 | """ 16 | Main configuration class. Contains all the configurations for the project. 17 | """ 18 | 19 | DEBUG: bool = True 20 | OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY") 21 | ASSISTANT_ID: str = os.getenv("ASSISTANT_ID") 22 | OPENAI_MODEL: str = "gpt-4o" 23 | OPENWEATHER_API_KEY: str = os.getenv("OPENWEATHER_API_KEY") 24 | 25 | 26 | config = Config() 27 | -------------------------------------------------------------------------------- /tools/definitions.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains json-schemas for the tools 3 | 4 | """ 5 | 6 | GET_WEATHER_INFORMATION = { 7 | "type": "function", 8 | "function": { 9 | "name": "get_weather_information", 10 | "description": "Gets the weather information for a given latitude and longitude", 11 | "parameters": { 12 | "type": "object", 13 | "properties": { 14 | "latitude": { 15 | "type": "number", 16 | "description": "The latitude of the location", 17 | }, 18 | "longitude": { 19 | "type": "number", 20 | "description": "The longitude of the location", 21 | }, 22 | }, 23 | "required": ["latitude", "longitude"], 24 | }, 25 | }, 26 | } 27 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.9.5 2 | aiosignal==1.3.1 3 | annotated-types==0.7.0 4 | anyio==4.4.0 5 | async-timeout==4.0.3 6 | attrs==23.2.0 7 | certifi==2024.6.2 8 | charset-normalizer==3.3.2 9 | click==8.1.7 10 | distro==1.9.0 11 | dnspython==2.6.1 12 | email_validator==2.2.0 13 | exceptiongroup==1.2.1 14 | fastapi==0.111.0 15 | fastapi-cli==0.0.4 16 | frozenlist==1.4.1 17 | h11==0.14.0 18 | httpcore==1.0.5 19 | httptools==0.6.1 20 | httpx==0.27.0 21 | idna==3.7 22 | Jinja2==3.1.4 23 | markdown-it-py==3.0.0 24 | MarkupSafe==2.1.5 25 | mdurl==0.1.2 26 | multidict==6.0.5 27 | openai==1.35.7 28 | orjson==3.10.5 29 | pydantic==2.7.4 30 | pydantic_core==2.18.4 31 | Pygments==2.18.0 32 | python-dotenv==1.0.1 33 | python-multipart==0.0.9 34 | PyYAML==6.0.1 35 | requests==2.32.3 36 | rich==13.7.1 37 | shellingham==1.5.4 38 | sniffio==1.3.1 39 | starlette==0.37.2 40 | tqdm==4.66.4 41 | typer==0.12.3 42 | typing_extensions==4.12.2 43 | ujson==5.10.0 44 | urllib3==2.2.2 45 | uvicorn==0.30.1 46 | uvloop==0.19.0 47 | watchfiles==0.22.0 48 | websockets==12.0 49 | yarl==1.9.4 50 | -------------------------------------------------------------------------------- /api/router.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file is responsible for routing the incoming requests to the respective endpoints. 3 | """ 4 | 5 | from fastapi.responses import JSONResponse, StreamingResponse 6 | from fastapi.templating import Jinja2Templates 7 | from fastapi import APIRouter 8 | from fastapi.requests import Request 9 | from pydantic import BaseModel 10 | 11 | from services.chat import ChatService 12 | from utils.stream import stream_generator 13 | 14 | api_router = APIRouter() 15 | chat_service = ChatService() 16 | 17 | templates = Jinja2Templates(directory="templates") 18 | 19 | 20 | class GetChatResponseRequest(BaseModel): 21 | """ 22 | This class is used to validate the request for getting a chat response 23 | """ 24 | 25 | user_query: str 26 | 27 | 28 | @api_router.get("/ping", response_class=JSONResponse) 29 | async def ping(): 30 | """ 31 | This function is used for health check of the application. 32 | """ 33 | return {"message": "Application is Running!", "status": "success"} 34 | 35 | 36 | @api_router.post("/chat/{chat_id}") 37 | async def get_chat_response(chat_id: str, data: GetChatResponseRequest): 38 | """ 39 | This function generates response for a user query 40 | """ 41 | query = data.user_query 42 | response = chat_service.generate(chat_id, query) 43 | return StreamingResponse(stream_generator(response)) 44 | 45 | @api_router.get("/", response_class=JSONResponse) 46 | async def chat_frontend(request: Request): 47 | """ 48 | This function renders the chat frontend 49 | """ 50 | return templates.TemplateResponse("index.html", {"request": request}) 51 | -------------------------------------------------------------------------------- /tools/get_weather.py: -------------------------------------------------------------------------------- 1 | """ 2 | main file for accessing services. 3 | """ 4 | 5 | import os 6 | from datetime import datetime 7 | import aiohttp 8 | 9 | from config.main import config 10 | 11 | os.environ["OPENWEATHER_API_KEY"] = config.OPENWEATHER_API_KEY 12 | 13 | 14 | async def get_weather_information(latitude: int, longitude: int) -> str: 15 | """Gets the weather information for a given latitude and longitude.""" 16 | try: 17 | url = "https://history.openweathermap.org/data/2.5/aggregated/day" 18 | current_day, current_month = datetime.now().day, datetime.now().month 19 | params = { 20 | "lat": latitude, 21 | "lon": longitude, 22 | "appid": os.environ.get("OPENWEATHER_API_KEY"), 23 | "month": current_month, 24 | "day": current_day, 25 | } 26 | result = None 27 | async with aiohttp.ClientSession() as session: 28 | async with session.get(url, params=params) as response: 29 | if response.status != 200: 30 | return "Sorry, I couldn't find the weather information for the given location." 31 | result = await response.json() 32 | # we format the response to be more user friendly 33 | result = result.get("result") 34 | if not result: 35 | return ( 36 | "Sorry, I couldn't find the weather information for the given location." 37 | ) 38 | return f""" 39 | For given Location: 40 | Mean temperature: {result['temp']['mean']} Kelvin 41 | Mean humidity: {result['humidity']['mean']} % 42 | Mean wind_speed: {result['wind']['mean']} m/s 43 | Mean pressure: {result['pressure']['mean']} hPa 44 | Mean precipitation: {result['precipitation']['mean']} mm 45 | """ 46 | except Exception: # pylint: disable=broad-except 47 | return "Sorry, I couldn't find the weather information for the given location." 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenAI Assistant Streaming with Function Calling in FastAPI 2 | 3 | This project showcases how you can use asynchronous streaming with OpenAI assistant and at the same time utilize function calling 4 | in FastAPI. 5 | 6 | You can read about it in detail in the following blog post: [OpenAI Assistant Streaming with Function Calling in FastAPI](https://medium.com/@meeran2003/async-streaming-openai-assistant-api-with-function-calling-in-fastapi-0dfe5935f238) 7 | 8 | ![OpenAI Assistant Streaming with Function Calling in FastAPI](./demo.png?raw=true "Demo") 9 | 10 | ## Description 11 | 12 | This project demonstrates how you can use FastAPI to create a real-time chat interface that communicates with OpenAI's GPT models for automated responses. The application also supports function calling, allowing you to execute commands and retrieve information in real-time. 13 | 14 | ## Features 15 | 16 | - Asynchronous streaming for real-time chat communication. 17 | - Function calling for executing commands and retrieving information. 18 | - Integration with OpenAI's GPT models for automated responses. 19 | - Weather information retrieval using the OpenWeather API. 20 | - Text to Speech and Speech to Text using web APIs. 21 | - Chat interface for real-time communication. 22 | 23 | ## Getting Started 24 | 25 | ### Dependencies 26 | 27 | - Python 3.8 or higher 28 | - FastAPI 29 | - OpenAI API 30 | - aiohttp, httpx for asynchronous HTTP requests 31 | 32 | Refer to `requirements.txt` for a complete list of dependencies. 33 | 34 | ### Installing 35 | 36 | 1. Clone the repository to your local machine. 37 | 2. Create a virtual environment: 38 | 39 | ```sh 40 | python -m venv env 41 | ``` 42 | 43 | 3. Activate the virtual environment: 44 | 45 | - On Windows: 46 | 47 | ```sh 48 | env\Scripts\activate 49 | ``` 50 | 51 | - On Unix or MacOS: 52 | 53 | ```sh 54 | source env/bin/activate 55 | ``` 56 | 57 | 4. Install the required packages: 58 | 59 | ```sh 60 | pip install -r requirements.txt 61 | ``` 62 | 63 | ### Configuration 64 | 65 | - Copy `.env.development` to `.env` and adjust the configuration variables as needed. 66 | - Ensure you have valid API keys for OpenAI and OpenWeather APIs set in your `.env` file. 67 | 68 | ### Running the Application 69 | 70 | 1. Start the application: 71 | 72 | ```sh 73 | uvicorn main:app --reload 74 | ``` 75 | 76 | 2. Visit `http://127.0.0.1:8000` in your web browser to access the chat interface. 77 | 78 | ## Usage 79 | 80 | - Use the chat interface to communicate in real-time. 81 | 82 | ## Contributing 83 | 84 | Contributions are welcome! Please feel free to submit pull requests or open issues to suggest improvements or add new features. 85 | -------------------------------------------------------------------------------- /services/assistant_setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | This class handles updates/creation of the 3 | OpenAI Assistant for the chatbot 4 | """ 5 | from openai import AsyncOpenAI as OpenAI 6 | from config.main import config 7 | 8 | class AssistantSetup: 9 | """ 10 | This class handles updates/creation of the 11 | OpenAI Assistant for the chatbot 12 | """ 13 | def __init__(self, client: OpenAI, assistant_id, sys_prompt, name, tools): 14 | self.client = client 15 | self.assistant_id = assistant_id 16 | self.tools = tools 17 | self.sys_prompt = sys_prompt 18 | self.name = name 19 | self.model = config.OPENAI_MODEL 20 | 21 | async def create_or_update_assistant(self): 22 | """ 23 | This function creates or updates the assistant 24 | """ 25 | assistant_id = self.assistant_id 26 | if assistant_id: 27 | assistant = await self.update_existing_assistant(assistant_id) 28 | else: 29 | assistant = await self.create_new_assistant() 30 | return assistant 31 | 32 | async def update_existing_assistant(self, assistant_id): 33 | """ 34 | This function updates the existing assistant 35 | with the new properties 36 | """ 37 | try: 38 | assistant = await self.client.beta.assistants.retrieve(assistant_id) 39 | await self.update_assistant_properties(assistant) 40 | except Exception as e: # pylint: disable=broad-except 41 | print(f"Error updating assistant: {e}") 42 | assistant = await self.create_new_assistant() 43 | return assistant 44 | 45 | async def create_new_assistant(self): 46 | """ 47 | This function creates a new assistant 48 | """ 49 | try: 50 | model = self.model 51 | assistant = await self.client.beta.assistants.create( 52 | name=self.name, 53 | instructions=self.sys_prompt, 54 | model=model, 55 | tools=self.tools, 56 | temperature=self.get_temperature(), 57 | ) 58 | print("Assistant created successfully!", assistant.id) 59 | except Exception as e: # pylint: disable=broad-except 60 | print(f"Error creating assistant: {e}") 61 | assistant = None 62 | return assistant 63 | 64 | def get_temperature(self): 65 | """ 66 | This function returns the temperature depending on the assistant 67 | """ 68 | return 0.5 69 | 70 | async def update_assistant_properties(self, assistant): 71 | """ 72 | This function updates the assistant properties 73 | """ 74 | try: 75 | assistant = await self.client.beta.assistants.update( 76 | assistant.id, 77 | instructions=self.sys_prompt, 78 | tools=self.tools, 79 | temperature=self.get_temperature(), 80 | ) 81 | except Exception as e: # pylint: disable=broad-except 82 | print(f"Error updating assistant: {e}") 83 | return assistant 84 | -------------------------------------------------------------------------------- /services/chat.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains the core functionality of the chat service. 3 | """ 4 | 5 | import os 6 | import asyncio 7 | import json 8 | 9 | from openai import AsyncOpenAI as OpenAI 10 | from openai.types.beta import Assistant, Thread 11 | from openai.types.beta.assistant_stream_event import ( 12 | ThreadRunRequiresAction, 13 | ThreadMessageDelta, 14 | ThreadRunFailed, 15 | ThreadRunCancelling, 16 | ThreadRunCancelled, 17 | ThreadRunExpired, 18 | ThreadRunStepFailed, 19 | ThreadRunStepCancelled, 20 | ) 21 | 22 | from config.main import config 23 | from config.prompts import SYS_PROMPT 24 | from utils.singleton import Singleton 25 | from services.assistant_setup import AssistantSetup 26 | from tools.definitions import GET_WEATHER_INFORMATION 27 | from tools.get_weather import get_weather_information 28 | 29 | os.environ["OPENAI_API_KEY"] = config.OPENAI_API_KEY 30 | 31 | class ChatService(metaclass=Singleton): 32 | """ 33 | This class is used to handle the OpenAI GPT based assistant. 34 | """ 35 | 36 | assistant: Assistant = None 37 | assistant_setup: AssistantSetup = None 38 | sys_prompt: str = SYS_PROMPT 39 | chat_to_thread_map = {} 40 | tools = [] 41 | tool_instances = {} 42 | 43 | def __init__(self) -> None: 44 | self.client = OpenAI() 45 | self.name = 'Activity Suggester' 46 | self.assistant_id = config.ASSISTANT_ID 47 | self.init_tools() 48 | self.initialize() 49 | 50 | def initialize(self): 51 | """ 52 | This function initializes the required services and objects. 53 | """ 54 | self.assistant_setup = AssistantSetup( 55 | self.client, 56 | self.assistant_id, 57 | self.sys_prompt, 58 | self.name, 59 | self.tools, 60 | ) 61 | 62 | async def create_assistant(self): 63 | """ 64 | This function creates assistant if not exists 65 | """ 66 | if not self.assistant: 67 | self.assistant = ( # pylint: disable=attribute-defined-outside-init 68 | await self.assistant_setup.create_or_update_assistant() 69 | ) 70 | 71 | async def generate(self, chat_id, content): 72 | """ 73 | It generates the response for the user query. 74 | """ 75 | await self.create_assistant() 76 | thread = await self.create_or_get_thread(chat_id) 77 | await self.client.beta.threads.messages.create( 78 | thread.id, 79 | role="user", 80 | content=content, 81 | ) 82 | stream = await self.client.beta.threads.runs.create( 83 | thread_id=thread.id, assistant_id=self.assistant.id, stream=True 84 | ) 85 | async for event in stream: 86 | async for token in self.process_event(event, thread): 87 | yield token 88 | 89 | print("Tool run completed") 90 | 91 | async def create_or_get_thread(self, chat_id) -> Thread: 92 | """ 93 | This function either creates a new thread for the chat_id or gets the existing thread. 94 | """ 95 | thread = None 96 | if self.chat_to_thread_map.get(chat_id): 97 | try: 98 | thread = await self.client.beta.threads.retrieve(self.chat_to_thread_map[chat_id]) 99 | except Exception as e: # pylint: disable=bare-except, broad-except 100 | print("Error in getting thread", e) 101 | thread = None 102 | if not thread: 103 | thread = await self.client.beta.threads.create( 104 | metadata={ 105 | "chat_id": str(chat_id), 106 | }, 107 | ) 108 | self.chat_to_thread_map[chat_id] = thread.id 109 | return thread 110 | 111 | def create_tool_output(self, tool_call, tool_result): 112 | """ 113 | This function creates the tool output. 114 | """ 115 | output = { 116 | "tool_call_id": tool_call.id, 117 | "output": tool_result, 118 | } 119 | return output 120 | 121 | async def process_event(self, event, thread: Thread, **kwargs): 122 | """ 123 | Process an event in the thread. 124 | 125 | Args: 126 | event: The event to be processed. 127 | thread: The thread object. 128 | **kwargs: Additional keyword arguments. 129 | 130 | Yields: 131 | The processed tokens. 132 | 133 | Raises: 134 | Exception: If the run fails. 135 | """ 136 | if isinstance(event, ThreadMessageDelta): 137 | data = event.data.delta.content 138 | for d in data: 139 | yield d 140 | 141 | elif isinstance(event, ThreadRunRequiresAction): 142 | run_obj = event.data 143 | tool_outputs = await self.process_tool_calls( 144 | run_obj.required_action.submit_tool_outputs.tool_calls 145 | ) 146 | tool_output_events = ( 147 | await self.client.beta.threads.runs.submit_tool_outputs( 148 | thread_id=thread.id, 149 | run_id=run_obj.id, 150 | tool_outputs=tool_outputs, 151 | stream=True, 152 | ) 153 | ) 154 | async for tool_event in tool_output_events: 155 | async for token in self.process_event( 156 | tool_event, thread=thread, **kwargs 157 | ): 158 | yield token 159 | 160 | elif any( 161 | isinstance(event, cls) 162 | for cls in [ 163 | ThreadRunFailed, 164 | ThreadRunCancelling, 165 | ThreadRunCancelled, 166 | ThreadRunExpired, 167 | ThreadRunStepFailed, 168 | ThreadRunStepCancelled, 169 | ] 170 | ): 171 | raise Exception("Run failed") # pylint: disable=broad-exception-raised 172 | 173 | def init_tools(self): 174 | """ 175 | This function initializes the tools. 176 | """ 177 | self.tools = [GET_WEATHER_INFORMATION] 178 | self.tool_instances = { 179 | "get_weather_information": get_weather_information, 180 | } 181 | 182 | async def process_tool_call(self, tool_call, tool_outputs: list, extra_args=None): 183 | """ 184 | This function processes a single tool call. 185 | And also handles the exceptions. 186 | """ 187 | result = None 188 | try: 189 | arguments = json.loads(tool_call.function.arguments) 190 | function_name = tool_call.function.name 191 | if extra_args: 192 | for key, value in extra_args.items(): 193 | arguments[key] = value 194 | if function_name not in self.tool_instances: 195 | result = "Tool not found" 196 | else: 197 | result = await self.tool_instances[function_name](**arguments) 198 | except Exception as e: # pylint: disable=broad-except 199 | result = str(e) 200 | print(e) 201 | created_tool_output = self.create_tool_output(tool_call, result) 202 | tool_outputs.append(created_tool_output) 203 | 204 | async def process_tool_calls(self, tool_calls, extra_args = None): 205 | """ 206 | This function processes all the tool calls. 207 | """ 208 | tool_outputs = [] 209 | coroutines = [] 210 | total_calls = len(tool_calls) 211 | for i in range(total_calls): 212 | tool_call = tool_calls[i] 213 | coroutines.append(self.process_tool_call(tool_call, tool_outputs, extra_args)) 214 | if coroutines: 215 | await asyncio.gather(*coroutines) 216 | return tool_outputs 217 | -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Weather Advisor Chat 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 |

Weather Advisor Chat

16 | 17 |
18 | 19 | 23 |
24 | 25 |
26 |
27 |
28 | 31 | 32 | 34 |
35 |
36 |
37 | 38 | 199 | 200 | 201 | --------------------------------------------------------------------------------