├── experiments ├── pinata.ipynb ├── agents │ ├── __init__.py │ ├── triage_agent.py │ ├── accounts_agent.py │ ├── payments_agent.py │ └── applications_agent.py ├── utils │ ├── __init__.py │ └── helpers.py ├── db.json ├── applications │ └── texput.log ├── db.py ├── agent_swarm.py ├── applications.ipynb └── bot.ipynb ├── server ├── agents │ ├── __init__.py │ ├── triage_agent.py │ ├── accounts_agent.py │ ├── payments_agent.py │ └── applications_agent.py ├── utils │ ├── __init__.py │ └── helpers.py ├── applications │ ├── loan_application_ACC123_20241117091458.pdf │ └── loan_application_ACC123_20241117091509.pdf ├── requirements.txt ├── db.json ├── socket_manager.py ├── form.tex ├── custom_types.py ├── pinata.py ├── agents.py ├── agent_swarm.py ├── llm.py ├── main.py └── db.py ├── client ├── .eslintrc.json ├── public │ ├── favicon.ico │ ├── TALKTUAHBANk.png │ ├── vercel.svg │ ├── file.svg │ ├── window.svg │ ├── globe.svg │ └── next.svg ├── src │ ├── img │ │ └── logo_dark.png │ └── app │ │ ├── fonts │ │ ├── GeistVF.woff │ │ └── GeistMonoVF.woff │ │ ├── globals.css │ │ ├── layout.tsx │ │ └── page.tsx ├── next.config.ts ├── postcss.config.mjs ├── tailwind.config.ts ├── tsconfig.json ├── package.json └── README.md ├── .gitignore └── README.md /experiments/pinata.ipynb: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/agents/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/agents/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /client/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["next/core-web-vitals", "next/typescript"] 3 | } 4 | -------------------------------------------------------------------------------- /client/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aurelisajuan/TalkTuahBank/HEAD/client/public/favicon.ico -------------------------------------------------------------------------------- /client/src/img/logo_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aurelisajuan/TalkTuahBank/HEAD/client/src/img/logo_dark.png -------------------------------------------------------------------------------- /client/public/TALKTUAHBANk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aurelisajuan/TalkTuahBank/HEAD/client/public/TALKTUAHBANk.png -------------------------------------------------------------------------------- /client/src/app/fonts/GeistVF.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aurelisajuan/TalkTuahBank/HEAD/client/src/app/fonts/GeistVF.woff -------------------------------------------------------------------------------- /client/src/app/fonts/GeistMonoVF.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aurelisajuan/TalkTuahBank/HEAD/client/src/app/fonts/GeistMonoVF.woff -------------------------------------------------------------------------------- /client/public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /client/next.config.ts: -------------------------------------------------------------------------------- 1 | import type { NextConfig } from "next"; 2 | 3 | const nextConfig: NextConfig = { 4 | /* config options here */ 5 | }; 6 | 7 | export default nextConfig; 8 | -------------------------------------------------------------------------------- /server/applications/loan_application_ACC123_20241117091458.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aurelisajuan/TalkTuahBank/HEAD/server/applications/loan_application_ACC123_20241117091458.pdf -------------------------------------------------------------------------------- /server/applications/loan_application_ACC123_20241117091509.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aurelisajuan/TalkTuahBank/HEAD/server/applications/loan_application_ACC123_20241117091509.pdf -------------------------------------------------------------------------------- /client/postcss.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('postcss-load-config').Config} */ 2 | const config = { 3 | plugins: { 4 | tailwindcss: {}, 5 | }, 6 | }; 7 | 8 | export default config; 9 | -------------------------------------------------------------------------------- /client/public/file.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /client/public/window.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /client/src/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --background: #ffffff; 7 | --foreground: #171717; 8 | } 9 | 10 | @media (prefers-color-scheme: dark) { 11 | :root { 12 | --background: #0a0a0a; 13 | --foreground: #ededed; 14 | } 15 | } 16 | 17 | body { 18 | color: var(--foreground); 19 | background: var(--background); 20 | font-family: Arial, Helvetica, sans-serif; 21 | } 22 | -------------------------------------------------------------------------------- /client/tailwind.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "tailwindcss"; 2 | 3 | export default { 4 | content: [ 5 | "./src/pages/**/*.{js,ts,jsx,tsx,mdx}", 6 | "./src/components/**/*.{js,ts,jsx,tsx,mdx}", 7 | "./src/app/**/*.{js,ts,jsx,tsx,mdx}", 8 | ], 9 | theme: { 10 | extend: { 11 | colors: { 12 | background: "var(--background)", 13 | foreground: "var(--foreground)", 14 | }, 15 | }, 16 | }, 17 | plugins: [], 18 | } satisfies Config; 19 | -------------------------------------------------------------------------------- /experiments/db.json: -------------------------------------------------------------------------------- 1 | { 2 | "users": { 3 | "user1": { 4 | "name": "Bill Zhang", 5 | "accounts": ["ACC123"] 6 | } 7 | }, 8 | "accounts": { 9 | "ACC123": { 10 | "balance": 2500.0, 11 | "statements": { 12 | "January 12th": "Transaction 1: -$500.00\nTransaction 2: +$1500.00", 13 | "February 13th": "Transaction 1: -$300.00\nTransaction 2: +$800.00" 14 | } 15 | } 16 | }, 17 | "payments": { 18 | "PAY002": { 19 | "from_account": "ACC123", 20 | "to_account": "ACC456", 21 | "amount": 150.00, 22 | "date": "2024-05-15", 23 | "status": "Completed" 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /server/requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.7.0 2 | anthropic==0.39.0 3 | anyio==4.6.2.post1 4 | certifi==2024.8.30 5 | charset-normalizer==3.4.0 6 | dataclasses-json==0.6.7 7 | distro==1.9.0 8 | fastapi==0.115.5 9 | h11==0.14.0 10 | httpcore==1.0.7 11 | httpx==0.27.2 12 | idna==3.10 13 | jiter==0.7.1 14 | jsonpath-python==1.0.6 15 | marshmallow==3.23.1 16 | mypy-extensions==1.0.0 17 | packaging==24.2 18 | pydantic==2.9.2 19 | pydantic_core==2.23.4 20 | python-dateutil==2.9.0.post0 21 | python-dotenv==1.0.1 22 | requests==2.32.3 23 | six==1.16.0 24 | sniffio==1.3.1 25 | starlette==0.41.2 26 | typing-inspect==0.9.0 27 | typing_extensions==4.12.2 28 | urllib3==2.2.3 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | node_modules 5 | .next 6 | /.pnp 7 | .pnp.* 8 | .yarn/* 9 | !.yarn/patches 10 | !.yarn/plugins 11 | !.yarn/releases 12 | !.yarn/versions 13 | 14 | # testing 15 | /coverage 16 | 17 | # next.js 18 | /.next/ 19 | /out/ 20 | 21 | # production 22 | /build 23 | 24 | # misc 25 | .DS_Store 26 | *.pem 27 | 28 | # debug 29 | npm-debug.log* 30 | yarn-debug.log* 31 | yarn-error.log* 32 | 33 | # env files (can opt-in for committing if needed) 34 | .env* 35 | 36 | # vercel 37 | .vercel 38 | 39 | # typescript 40 | *.tsbuildinfo 41 | next-env.d.ts 42 | 43 | # python 44 | venv/ 45 | .env 46 | __pycache__ 47 | 48 | *.pdf -------------------------------------------------------------------------------- /client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2017", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "noEmit": true, 9 | "esModuleInterop": true, 10 | "module": "esnext", 11 | "moduleResolution": "bundler", 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "jsx": "preserve", 15 | "incremental": true, 16 | "plugins": [ 17 | { 18 | "name": "next" 19 | } 20 | ], 21 | "paths": { 22 | "@/*": ["./src/*"] 23 | } 24 | }, 25 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 26 | "exclude": ["node_modules"] 27 | } 28 | -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "client", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "framer-motion": "^11.11.17", 13 | "lucide-react": "^0.460.0", 14 | "next": "15.0.3", 15 | "react": "19.0.0-rc-66855b96-20241106", 16 | "react-dom": "19.0.0-rc-66855b96-20241106" 17 | }, 18 | "devDependencies": { 19 | "@types/node": "^20", 20 | "@types/react": "^18", 21 | "@types/react-dom": "^18", 22 | "eslint": "^8", 23 | "eslint-config-next": "15.0.3", 24 | "postcss": "^8", 25 | "tailwindcss": "^3.4.1", 26 | "typescript": "^5" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /server/db.json: -------------------------------------------------------------------------------- 1 | { 2 | "users": { 3 | "user1": { 4 | "name": "Bill Zhang", 5 | "accounts": ["ACC123"] 6 | } 7 | }, 8 | "accounts": { 9 | "ACC123": { 10 | "balance": 2500.0, 11 | "statements": { 12 | "November": [ 13 | { 14 | "date": "2024-11-13", 15 | "description": "Flight to HackUTD", 16 | "amount": -500.00 17 | }, 18 | { 19 | "date": "2024-11-14", 20 | "description": "Dinner with friends", 21 | "amount": -100.00 22 | } 23 | ] 24 | 25 | } 26 | } 27 | }, 28 | "payments": { 29 | "PAY002": { 30 | "from_account": "ACC123", 31 | "to_account": "BillyBob", 32 | "amount": 500, 33 | "date": "2024-11-13", 34 | "status": "Completed" 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /client/src/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type { Metadata } from "next"; 2 | import localFont from "next/font/local"; 3 | import "./globals.css"; 4 | 5 | const geistSans = localFont({ 6 | src: "./fonts/GeistVF.woff", 7 | variable: "--font-geist-sans", 8 | weight: "100 900", 9 | }); 10 | const geistMono = localFont({ 11 | src: "./fonts/GeistMonoVF.woff", 12 | variable: "--font-geist-mono", 13 | weight: "100 900", 14 | }); 15 | 16 | export const metadata: Metadata = { 17 | title: "TalkTuahBank", 18 | description: "HackUTD 2024", 19 | }; 20 | 21 | export default function RootLayout({ 22 | children, 23 | }: Readonly<{ 24 | children: React.ReactNode; 25 | }>) { 26 | return ( 27 | 28 | 31 | {children} 32 | 33 | 34 | ); 35 | } 36 | -------------------------------------------------------------------------------- /experiments/applications/texput.log: -------------------------------------------------------------------------------- 1 | This is pdfTeX, Version 3.141592653-2.6-1.40.26 (TeX Live 2024) (preloaded format=pdflatex 2024.11.17) 17 NOV 2024 09:10 2 | entering extended mode 3 | restricted \write18 enabled. 4 | %&-line parsing enabled. 5 | **applications/loan_application_ACC123_20241117091054.tex 6 | 7 | ! Emergency stop. 8 | <*> .../loan_application_ACC123_20241117091054.tex 9 | 10 | *** (job aborted, file error in nonstop mode) 11 | 12 | 13 | Here is how much of TeX's memory you used: 14 | 4 strings out of 475567 15 | 147 string characters out of 5777888 16 | 1925187 words of memory out of 5000000 17 | 22255 multiletter control sequences out of 15000+600000 18 | 558069 words of font info for 36 fonts, out of 8000000 for 9000 19 | 319 hyphenation exceptions out of 8191 20 | 0i,0n,0p,1b,6s stack positions out of 10000i,1000n,20000p,200000b,200000s 21 | ! ==> Fatal error occurred, no output PDF file produced! 22 | -------------------------------------------------------------------------------- /client/public/globe.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/agents/triage_agent.py: -------------------------------------------------------------------------------- 1 | # agents/triage_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Any, Dict, List, Callable 5 | import logging 6 | 7 | # Define the triage instructions 8 | triage_instructions = """ 9 | You are the Triage Agent responsible for categorizing user requests and delegating them to the appropriate agent. 10 | 11 | Your tasks: 12 | 1. Analyze the user's message to determine its intent. 13 | 2. If the request is about account-related inquiries (e.g., checking balance, retrieving statements), transfer it to the Accounts Agent. 14 | 3. If the request is about payment-related actions (e.g., transferring funds, scheduling payments), transfer it to the Payments Agent. 15 | 4. If you need more information to accurately triage the request, ask a direct question without providing explanations. 16 | 5. Do not share your internal decision-making process with the user. 17 | 6. Maintain a professional and friendly tone at all times. 18 | """ 19 | 20 | class TriageAgent(Agent): 21 | def __init__(self, functions: List[Callable]): 22 | super().__init__( 23 | name="Triage Agent", 24 | instructions=triage_instructions, 25 | functions=functions 26 | ) 27 | -------------------------------------------------------------------------------- /server/socket_manager.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | from fastapi import WebSocket 3 | 4 | 5 | # manages the connection across mukt clients and sate of ws 6 | class ConnectionManager: 7 | # initializes ws and adds to active connections inside of a dictionary 8 | def __init__(self): 9 | self.active_connections: Dict[str, WebSocket] = {} 10 | 11 | # establish connection btwn a client and ws. waits for ws to start and adds accepted client to active connections 12 | async def connect(self, websocket: WebSocket, client_id: str): 13 | await websocket.accept() 14 | self.active_connections[client_id] = websocket 15 | 16 | # disconnects client from ws 17 | async def disconnect(self, client_id: str): 18 | try: 19 | del self.active_connections[client_id] 20 | except KeyError: 21 | pass 22 | 23 | async def send_personal_message(self, data: dict, websocket: WebSocket): 24 | await websocket.send_json(data) 25 | 26 | # shows data to all clients with active connections to the ws 27 | async def broadcast(self, data: dict): 28 | for connection in self.active_connections.values(): 29 | await connection.send_json(data) 30 | 31 | 32 | manager = ConnectionManager() -------------------------------------------------------------------------------- /client/public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /server/agents/triage_agent.py: -------------------------------------------------------------------------------- 1 | # agents/triage_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Any, Dict, List, Callable 5 | import logging 6 | 7 | # Define the triage instructions 8 | triage_instructions = """ 9 | You are the Triage Agent responsible for categorizing user requests and delegating them to the appropriate agent. 10 | 11 | Your tasks: 12 | 1. Analyze the user's message to determine its intent. 13 | 2. If the request is about account-related inquiries (e.g., checking balance, retrieving statements), transfer it to the Accounts Agent. 14 | 3. If the request is about payment-related actions (e.g., transferring funds, scheduling payments), transfer it to the Payments Agent. 15 | 4. If the request is about loan-related actions (e.g., applying for a loan) or credit-related actions (e.g., applying for a credit card), transfer it to the Applications Agent. 16 | 4. If you need more information to accurately triage the request, ask a direct question without providing explanations. 17 | 5. Do not share your internal decision-making process with the user. 18 | 6. Maintain a professional and friendly tone at all times. 19 | """ 20 | 21 | class TriageAgent(Agent): 22 | def __init__(self, functions: List[Callable]): 23 | super().__init__( 24 | name="Triage Agent", 25 | instructions=triage_instructions, 26 | functions=functions 27 | ) 28 | -------------------------------------------------------------------------------- /server/form.tex: -------------------------------------------------------------------------------- 1 | \documentclass[12pt]{article} 2 | \usepackage[margin=1in]{geometry} 3 | \usepackage{array} 4 | \usepackage{graphicx} 5 | \usepackage{hyperref} 6 | 7 | \begin{document} 8 | 9 | \begin{center} 10 | {\LARGE \textbf{Credit Card Application Form}} 11 | \end{center} 12 | 13 | \vspace{1em} 14 | 15 | \section*{Personal Information} 16 | 17 | \begin{tabular}{>{\bfseries}l p{4in}} 18 | First Name: & \hrulefill \\ 19 | Last Name: & \hrulefill \\ 20 | Date of Birth (MM/DD/YYYY): & \hrulefill \\ 21 | Social Security Number: & \hrulefill \\ 22 | Email Address: & \hrulefill \\ 23 | Phone Number: & \hrulefill \\ 24 | Current Address: & \hrulefill \\ 25 | City: & \hrulefill \\ 26 | State: & \hrulefill \\ 27 | ZIP Code: & \hrulefill \\ 28 | \end{tabular} 29 | 30 | \vspace{1em} 31 | 32 | \section*{Employment Information} 33 | 34 | \begin{tabular}{>{\bfseries}l p{4in}} 35 | Employer Name: & \hrulefill \\ 36 | Job Title: & \hrulefill \\ 37 | Annual Income: & \hrulefill \\ 38 | Employment Status: & \hrulefill \\ 39 | Work Phone Number: & \hrulefill \\ 40 | Length of Employment: & \hrulefill \\ 41 | \end{tabular} 42 | 43 | \vspace{1em} 44 | 45 | \section*{Financial Information} 46 | 47 | \begin{tabular}{>{\bfseries}l p{4in}} 48 | Bank Name: & \hrulefill \\ 49 | Account Type (Checking/Savings): & \hrulefill \\ 50 | Account Number: & \hrulefill \\ 51 | Monthly Housing Payment: & \hrulefill \\ 52 | Do you have existing credit cards? & \hrulefill \\ 53 | If yes, list them: & \hrulefill \\ 54 | \end{tabular} 55 | 56 | \end{document} 57 | -------------------------------------------------------------------------------- /server/utils/helpers.py: -------------------------------------------------------------------------------- 1 | # utils/helpers.py 2 | 3 | from typing import List 4 | from db import get_db, set_db 5 | import logging 6 | from datetime import datetime 7 | 8 | def validate_account_id(account_id: str) -> bool: 9 | current_db = get_db() 10 | if account_id in current_db["accounts"]: 11 | logging.info(f"Account ID {account_id} is valid.") 12 | return True 13 | logging.warning(f"Account ID {account_id} is invalid.") 14 | return False 15 | 16 | def validate_payment_id(payment_id: str) -> bool: 17 | current_db = get_db() 18 | if payment_id in current_db["payments"]: 19 | logging.info(f"Payment ID {payment_id} is valid.") 20 | return True 21 | logging.warning(f"Payment ID {payment_id} is invalid.") 22 | return False 23 | 24 | def generate_payment_id() -> str: 25 | current_db = get_db() 26 | payment_number = len(current_db["payments"]) + 1 27 | payment_id = f"PAY{payment_number:03d}" 28 | logging.info(f"Generated new payment ID: {payment_id}") 29 | return payment_id 30 | 31 | def validate_amount(amount: float) -> bool: 32 | if amount <= 0: 33 | logging.warning(f"Invalid amount: {amount}. Amount must be positive.") 34 | return False 35 | logging.info(f"Amount {amount} is valid.") 36 | return True 37 | 38 | def get_user_accounts(user_id: str) -> List[str]: 39 | current_db = get_db() 40 | accounts = current_db["users"].get(user_id, {}).get("accounts", []) 41 | logging.info(f"User {user_id} has accounts: {accounts}") 42 | return accounts 43 | -------------------------------------------------------------------------------- /experiments/utils/helpers.py: -------------------------------------------------------------------------------- 1 | # utils/helpers.py 2 | 3 | from typing import List 4 | from db import get_db, set_db 5 | import logging 6 | from datetime import datetime 7 | 8 | def validate_account_id(account_id: str) -> bool: 9 | current_db = get_db() 10 | if account_id in current_db["accounts"]: 11 | logging.info(f"Account ID {account_id} is valid.") 12 | return True 13 | logging.warning(f"Account ID {account_id} is invalid.") 14 | return False 15 | 16 | def validate_payment_id(payment_id: str) -> bool: 17 | current_db = get_db() 18 | if payment_id in current_db["payments"]: 19 | logging.info(f"Payment ID {payment_id} is valid.") 20 | return True 21 | logging.warning(f"Payment ID {payment_id} is invalid.") 22 | return False 23 | 24 | def generate_payment_id() -> str: 25 | current_db = get_db() 26 | payment_number = len(current_db["payments"]) + 1 27 | payment_id = f"PAY{payment_number:03d}" 28 | logging.info(f"Generated new payment ID: {payment_id}") 29 | return payment_id 30 | 31 | def validate_amount(amount: float) -> bool: 32 | if amount <= 0: 33 | logging.warning(f"Invalid amount: {amount}. Amount must be positive.") 34 | return False 35 | logging.info(f"Amount {amount} is valid.") 36 | return True 37 | 38 | def get_user_accounts(user_id: str) -> List[str]: 39 | current_db = get_db() 40 | accounts = current_db["users"].get(user_id, {}).get("accounts", []) 41 | logging.info(f"User {user_id} has accounts: {accounts}") 42 | return accounts 43 | -------------------------------------------------------------------------------- /client/README.md: -------------------------------------------------------------------------------- 1 | This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). 2 | 3 | ## Getting Started 4 | 5 | First, run the development server: 6 | 7 | ```bash 8 | npm run dev 9 | # or 10 | yarn dev 11 | # or 12 | pnpm dev 13 | # or 14 | bun dev 15 | ``` 16 | 17 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 18 | 19 | You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. 20 | 21 | This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. 22 | 23 | ## Learn More 24 | 25 | To learn more about Next.js, take a look at the following resources: 26 | 27 | - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. 28 | - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. 29 | 30 | You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! 31 | 32 | ## Deploy on Vercel 33 | 34 | The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. 35 | 36 | Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. 37 | -------------------------------------------------------------------------------- /server/custom_types.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Optional, Literal, Union 2 | from pydantic import BaseModel 3 | from typing import Literal, Dict, Optional 4 | 5 | 6 | # Retell -> Your Server Events 7 | class Utterance(BaseModel): 8 | role: Literal["agent", "user", "system"] 9 | content: str 10 | 11 | 12 | class PingPongRequest(BaseModel): 13 | interaction_type: Literal["ping_pong"] 14 | timestamp: int 15 | 16 | 17 | class CallDetailsRequest(BaseModel): 18 | interaction_type: Literal["call_details"] 19 | call: dict 20 | 21 | 22 | class UpdateOnlyRequest(BaseModel): 23 | interaction_type: Literal["update_only"] 24 | transcript: List[Utterance] 25 | 26 | 27 | class ResponseRequiredRequest(BaseModel): 28 | interaction_type: Literal["reminder_required", "response_required"] 29 | response_id: int 30 | transcript: List[Utterance] 31 | 32 | 33 | CustomLlmRequest = Union[ 34 | ResponseRequiredRequest | UpdateOnlyRequest | CallDetailsRequest | PingPongRequest 35 | ] 36 | 37 | 38 | # Your Server -> Retell Events 39 | class ConfigResponse(BaseModel): 40 | response_type: Literal["config"] = "config" 41 | config: Dict[str, bool] = { 42 | "auto_reconnect": bool, 43 | "call_details": bool, 44 | } 45 | 46 | 47 | class PingPongResponse(BaseModel): 48 | response_type: Literal["ping_pong"] = "ping_pong" 49 | timestamp: int 50 | 51 | 52 | class ResponseResponse(BaseModel): 53 | response_type: Literal["response"] = "response" 54 | response_id: int 55 | content: str 56 | content_complete: bool 57 | end_call: Optional[bool] = False 58 | transfer_number: Optional[str] = None 59 | 60 | 61 | CustomLlmResponse = Union[ConfigResponse | PingPongResponse | ResponseResponse] 62 | -------------------------------------------------------------------------------- /experiments/db.py: -------------------------------------------------------------------------------- 1 | # db.py 2 | 3 | # Initial Banking Database Structure 4 | db = { 5 | "users": { 6 | "user1": { 7 | "name": "John Doe", 8 | "accounts": ["ACC123", "ACC456"] 9 | }, 10 | "user2": { 11 | "name": "Jane Smith", 12 | "accounts": ["ACC789"] 13 | } 14 | }, 15 | "accounts": { 16 | "ACC123": { 17 | "balance": 2500.00, 18 | "statements": { 19 | "January": "Transaction 1: -$500.00\nTransaction 2: +$1500.00", 20 | "February": "Transaction 1: -$300.00\nTransaction 2: +$800.00" 21 | } 22 | }, 23 | "ACC456": { 24 | "balance": 1000.00, 25 | "statements": { 26 | "January": "Transaction 1: -$200.00\nTransaction 2: +$1200.00", 27 | "February": "Transaction 1: -$100.00\nTransaction 2: +$500.00" 28 | } 29 | }, 30 | "ACC789": { 31 | "balance": 5000.00, 32 | "statements": { 33 | "January": "Transaction 1: -$1000.00\nTransaction 2: +$2000.00", 34 | "February": "Transaction 1: -$500.00\nTransaction 2: +$1500.00" 35 | } 36 | } 37 | }, 38 | "payments": { 39 | "PAY001": { 40 | "from_account": "ACC123", 41 | "to_account": "ACC456", 42 | "amount": 300.00, 43 | "date": "2024-05-01", 44 | "status": "Scheduled" 45 | }, 46 | "PAY002": { 47 | "from_account": "ACC456", 48 | "to_account": "ACC123", 49 | "amount": 150.00, 50 | "date": "2024-05-15", 51 | "status": "Completed" 52 | } 53 | } 54 | } 55 | 56 | def get_db(): 57 | """ 58 | Retrieves the current state of the banking database. 59 | 60 | Returns: 61 | dict: The current banking database dictionary. 62 | """ 63 | return db 64 | 65 | def set_db(new_db): 66 | """ 67 | Updates the banking database with a new state. 68 | 69 | Args: 70 | new_db (dict): The new banking database dictionary to replace the current state. 71 | 72 | Returns: 73 | bool: True if the database was successfully updated. 74 | """ 75 | global db 76 | db = new_db 77 | return True 78 | -------------------------------------------------------------------------------- /server/agents/accounts_agent.py: -------------------------------------------------------------------------------- 1 | # agents/accounts_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Any, Dict 5 | from utils.helpers import validate_account_id, get_db, set_db 6 | from swarm.types import Result 7 | import logging 8 | from typing import Callable 9 | 10 | # Define the accounts instructions 11 | accounts_instructions = """ 12 | You are the Accounts Agent, a highly knowledgeable and professional virtual banking assistant. 13 | 14 | Your responsibilities include: 15 | 1. Assisting users with account-related requests, such as checking account balances and retrieving bank statements. 16 | 2. Answering questions about account features, services, and policies. 17 | 3. Ensuring compliance with financial regulations and maintaining customer privacy. 18 | 4. Communicating in a polite, empathetic, and user-friendly manner. 19 | """ 20 | 21 | class AccountsAgent(Agent): 22 | def __init__( 23 | self, 24 | transfer_to_payments: Callable, 25 | handle_account_balance: Callable, 26 | retrieve_bank_statement: Callable 27 | ): 28 | super().__init__( 29 | name="Accounts Agent", 30 | instructions=accounts_instructions, 31 | functions=[transfer_to_payments, handle_account_balance, retrieve_bank_statement] 32 | ) 33 | 34 | # Define specific functions for Accounts Agent 35 | 36 | def handle_account_balance(context_variables: Dict, account_id: str) -> Result: 37 | logging.info(f"Handling account balance for account ID: {account_id}") 38 | if not validate_account_id(account_id): 39 | return Result( 40 | value="Invalid account ID provided.", 41 | agent=None # Transfer back to triage in AgentSwarm 42 | ) 43 | 44 | current_db = get_db() 45 | balance = current_db["accounts"][account_id]["balance"] 46 | return Result( 47 | value=f"Your current account balance for {account_id} is ${balance:.2f}.", 48 | agent=None # Transfer back to triage in AgentSwarm 49 | ) 50 | 51 | def retrieve_bank_statement(context_variables: Dict, account_id: str, period: str) -> Result: 52 | logging.info(f"Retrieving bank statement for account ID: {account_id}, period: {period}") 53 | if not validate_account_id(account_id): 54 | return Result( 55 | value="Invalid account ID provided.", 56 | agent=None # Transfer back to triage in AgentSwarm 57 | ) 58 | 59 | current_db = get_db() 60 | 61 | # Validate period 62 | if period not in current_db["accounts"][account_id]["statements"]: 63 | return Result( 64 | value=f"No statements found for the period: {period}.", 65 | agent=None # Transfer back to triage in AgentSwarm 66 | ) 67 | 68 | statement = current_db["accounts"][account_id]["statements"][period] 69 | return Result( 70 | value=f"Here is your bank statement for {period}:\n{statement}", 71 | agent=None # Transfer back to triage in AgentSwarm 72 | ) 73 | -------------------------------------------------------------------------------- /experiments/agents/accounts_agent.py: -------------------------------------------------------------------------------- 1 | # agents/accounts_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Any, Dict 5 | from utils.helpers import validate_account_id, get_db, set_db 6 | from swarm.types import Result 7 | import logging 8 | from typing import Callable 9 | 10 | # Define the accounts instructions 11 | accounts_instructions = """ 12 | You are the Accounts Agent, a highly knowledgeable and professional virtual banking assistant. 13 | 14 | Your responsibilities include: 15 | 1. Assisting users with account-related requests, such as checking account balances and retrieving bank statements. 16 | 2. Answering questions about account features, services, and policies. 17 | 3. Ensuring compliance with financial regulations and maintaining customer privacy. 18 | 4. Communicating in a polite, empathetic, and user-friendly manner. 19 | """ 20 | 21 | class AccountsAgent(Agent): 22 | def __init__( 23 | self, 24 | transfer_to_payments: Callable, 25 | handle_account_balance: Callable, 26 | retrieve_bank_statement: Callable 27 | ): 28 | super().__init__( 29 | name="Accounts Agent", 30 | instructions=accounts_instructions, 31 | functions=[transfer_to_payments, handle_account_balance, retrieve_bank_statement] 32 | ) 33 | 34 | # Define specific functions for Accounts Agent 35 | 36 | def handle_account_balance(context_variables: Dict, account_id: str) -> Result: 37 | logging.info(f"Handling account balance for account ID: {account_id}") 38 | if not validate_account_id(account_id): 39 | return Result( 40 | value="Invalid account ID provided.", 41 | agent=None # Transfer back to triage in AgentSwarm 42 | ) 43 | 44 | current_db = get_db() 45 | balance = current_db["accounts"][account_id]["balance"] 46 | return Result( 47 | value=f"Your current account balance for {account_id} is ${balance:.2f}.", 48 | agent=None # Transfer back to triage in AgentSwarm 49 | ) 50 | 51 | def retrieve_bank_statement(context_variables: Dict, account_id: str, period: str) -> Result: 52 | logging.info(f"Retrieving bank statement for account ID: {account_id}, period: {period}") 53 | if not validate_account_id(account_id): 54 | return Result( 55 | value="Invalid account ID provided.", 56 | agent=None # Transfer back to triage in AgentSwarm 57 | ) 58 | 59 | current_db = get_db() 60 | 61 | # Validate period 62 | if period not in current_db["accounts"][account_id]["statements"]: 63 | return Result( 64 | value=f"No statements found for the period: {period}.", 65 | agent=None # Transfer back to triage in AgentSwarm 66 | ) 67 | 68 | statement = current_db["accounts"][account_id]["statements"][period] 69 | return Result( 70 | value=f"Here is your bank statement for {period}:\n{statement}", 71 | agent=None # Transfer back to triage in AgentSwarm 72 | ) 73 | -------------------------------------------------------------------------------- /server/pinata.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import os 4 | from dotenv import load_dotenv 5 | 6 | def upload_pdf_to_pinata(pdf_path: str, document_type: str): # document_type is either CARD or LOAN 7 | """ 8 | Uploads a PDF file to Pinata and returns the IPFS hash and Pinata URL. 9 | 10 | :param pdf_path: The local path to the PDF file. 11 | :return: A dictionary with 'IpfsHash' and 'PinataURL' if successful, else None. 12 | """ 13 | # Load environment variables from .env file 14 | load_dotenv() 15 | PINATA_API_KEY = os.getenv('PINATA_API_KEY') 16 | PINATA_SECRET_API_KEY = os.getenv('PINATA_API_SECRET') 17 | 18 | if not PINATA_API_KEY or not PINATA_SECRET_API_KEY: 19 | print("Error: Pinata API credentials not found in environment variables.") 20 | return None 21 | 22 | # Define the Pinata API endpoint 23 | PINATA_URL = "https://api.pinata.cloud/pinning/pinFileToIPFS" 24 | 25 | # Define the headers 26 | headers = { 27 | "pinata_api_key": PINATA_API_KEY, 28 | "pinata_secret_api_key": PINATA_SECRET_API_KEY 29 | } 30 | 31 | # Check if the file exists 32 | if not os.path.isfile(pdf_path): 33 | print(f"Error: The file {pdf_path} does not exist.") 34 | return None 35 | 36 | try: 37 | with open(pdf_path, 'rb') as pdf_file: 38 | # Prepare the files dictionary 39 | files = { 40 | 'file': (os.path.basename(pdf_path), pdf_file, 'application/pdf') 41 | } 42 | 43 | # Optional: Add metadata 44 | metadata = { 45 | "name": document_type + "_" + pdf_path.split('_')[-2] + "_" + pdf_path.split('_')[-1], 46 | "keyvalues": { 47 | "description": "This is a sample PDF file." 48 | } 49 | } 50 | 51 | # Convert metadata to JSON string 52 | json_metadata = json.dumps(metadata) 53 | 54 | # Prepare the data payload 55 | data = { 56 | 'pinataMetadata': json_metadata 57 | } 58 | 59 | # Send the POST request 60 | response = requests.post(PINATA_URL, files=files, data=data, headers=headers) 61 | 62 | # Check the response status 63 | if response.status_code == 200: 64 | response_json = response.json() 65 | print("Upload successful!") 66 | print("IPFS Hash:", response_json['IpfsHash']) 67 | print(response_json) 68 | return response_json 69 | else: 70 | print(f"Upload failed with status code {response.status_code}") 71 | print("Response:", response.text) 72 | return None 73 | 74 | except Exception as e: 75 | print(f"An error occurred: {e}") 76 | return None 77 | 78 | # # Usage Example 79 | # if __name__ == "__main__": 80 | # # Path to your local PDF file 81 | # pdf_file_path = './applications/loan_application_ACC123_20241117091458.pdf' 82 | 83 | # # Upload the PDF to Pinata 84 | # result = upload_pdf_to_pinata(pdf_file_path) 85 | 86 | -------------------------------------------------------------------------------- /server/agents.py: -------------------------------------------------------------------------------- 1 | from swarm import Agent, Swarm 2 | from dotenv import load_dotenv 3 | import os 4 | import math 5 | load_dotenv() 6 | from openai import OpenAI 7 | openai_client = OpenAI() 8 | 9 | client = Swarm() 10 | 11 | triage_instructions = f""" 12 | You are to triage a users request, and call a tool to transfer to the right intent. 13 | Once you are ready to transfer to the right intent, call the tool to transfer to the right intent. 14 | You dont need to know specifics, just the topic of the request. 15 | If the user request is about their account, transfer to the Accounts Agent. 16 | If the user request is about any form of payment, transfer to the Payments Agent. 17 | When you need more information to triage the request to an agent, ask a direct question without explaining why you're asking it. 18 | Do not share your thought process with the user! Do not make unreasonable assumptions on behalf of user. 19 | Upon transferring to a different agent, please send another message confirming you are the requested agent. 20 | """ 21 | 22 | accounts_instructions = f""" 23 | You are a highly knowledgeable, professional, and user-friendly banking virtual agent. 24 | Your primary task is to assist users with account-related requests, such as checking account balances, retrieving bank statements, and answering questions about their accounts. 25 | Please maintain a polite and empathetic tone. 26 | When asked a question related to accounts (such as balances, transactions, etc), please call the get_account_info function, which will return a dictionary of account information. Use this information to provide the best answer to the user's request. 27 | """ 28 | 29 | payments_instructions = f""" 30 | You are a virtual banking agent designed to assist users with payment-related requests. 31 | Your primary focus is facilitating secure and accurate money transfers, including transferring funds between the user's accounts, 32 | sending money to others, and scheduling or canceling payments. 33 | Please maintain a polite and empathetic tone, and provide user-friendly instructions. 34 | """ 35 | 36 | class AgentSwarm: 37 | def __init__(self): 38 | self.triage_agent = Agent( 39 | name="Triage Agent", 40 | instructions=triage_instructions, 41 | functions=[self.transfer_to_accounts, self.transfer_to_payments], 42 | ) 43 | 44 | self.accounts = Agent( 45 | name="accounts", 46 | instructions=accounts_instructions, 47 | ) 48 | 49 | self.payments = Agent( 50 | name="payments", 51 | instructions=payments_instructions, 52 | ) 53 | 54 | self.accounts.functions = [self.transfer_back_to_triage] 55 | self.payments.functions = [self.transfer_back_to_triage] 56 | 57 | self.current_agent = self.triage_agent 58 | 59 | # Transfer functions 60 | def transfer_to_accounts(self): 61 | print("transferring to accounts") 62 | self.current_agent = self.accounts 63 | 64 | def transfer_to_payments(self): 65 | self.current_agent = self.payments 66 | 67 | def transfer_back_to_triage(self): 68 | self.current_agent = self.triage_agent 69 | 70 | def run(self, messages, stream=False): 71 | response = client.run( 72 | agent=self.current_agent, 73 | messages=messages, 74 | stream=stream 75 | ) 76 | return response -------------------------------------------------------------------------------- /server/agent_swarm.py: -------------------------------------------------------------------------------- 1 | # agent_swarm.py 2 | from agents.triage_agent import TriageAgent 3 | from agents.accounts_agent import AccountsAgent, handle_account_balance, retrieve_bank_statement 4 | from agents.payments_agent import PaymentsAgent, transfer_funds, schedule_payment, cancel_payment 5 | from agents.applications_agent import ApplicationsAgent, apply_for_loan, apply_for_credit_card # <-- New Import 6 | from utils.helpers import ( 7 | validate_account_id, validate_payment_id, generate_payment_id, 8 | validate_amount, get_user_accounts 9 | ) 10 | from swarm import Swarm # Assuming Swarm is defined in swarm/__init__.py 11 | from dotenv import load_dotenv 12 | import os 13 | from openai import OpenAI 14 | from typing import Dict, List 15 | import logging 16 | from datetime import datetime 17 | 18 | # Load environment variables 19 | load_dotenv() 20 | 21 | # Configure logging 22 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') 23 | 24 | # Initialize the Swarm client 25 | client = Swarm() 26 | 27 | # Initialize OpenAI client if needed 28 | openai_client = OpenAI() 29 | 30 | # Define the OpenAI model you're using 31 | OPENAI_MODEL = "gpt-4o-mini" # Replace with your specific model if different 32 | 33 | class AgentSwarm: 34 | def __init__(self, socket=None): 35 | """ 36 | Initialize the AgentSwarm with necessary agents and their corresponding functions. 37 | 38 | Args: 39 | socket: Handles real-time communication (e.g., sending messages). 40 | """ 41 | self.socket = socket 42 | self.messages = [] 43 | 44 | # Initialize Agents with their instructions and functions, including the model 45 | self.triage_agent = TriageAgent( 46 | [self.transfer_to_accounts, self.transfer_to_payments, self.transfer_to_applications] # Add transfer function 47 | ) 48 | 49 | self.accounts_agent = AccountsAgent( 50 | transfer_to_payments=self.transfer_to_payments, 51 | handle_account_balance=handle_account_balance, 52 | retrieve_bank_statement=retrieve_bank_statement, 53 | ) 54 | 55 | self.payments_agent = PaymentsAgent( 56 | transfer_back_to_triage=self.transfer_back_to_triage, 57 | transfer_funds=transfer_funds, 58 | schedule_payment=schedule_payment, 59 | cancel_payment=cancel_payment, 60 | ) 61 | 62 | self.applications_agent = ApplicationsAgent( 63 | transfer_back_to_triage=self.transfer_back_to_triage, 64 | apply_for_loan=apply_for_loan, 65 | apply_for_credit_card=apply_for_credit_card, 66 | ) 67 | 68 | self.current_agent = self.triage_agent 69 | 70 | # -------------------- Transfer Functions -------------------- # 71 | 72 | def transfer_to_accounts(self, context_variables: Dict, user_message: str): 73 | logging.info("Transferring to Accounts Agent.") 74 | self.current_agent = self.accounts_agent 75 | return self.accounts_agent 76 | 77 | def transfer_to_payments(self, context_variables: Dict, user_message: str): 78 | logging.info("Transferring to Payments Agent.") 79 | self.current_agent = self.payments_agent 80 | return self.payments_agent 81 | 82 | def transfer_to_applications(self, context_variables: Dict, user_message: str): 83 | logging.info("Transferring to Applications Agent.") 84 | self.current_agent = self.applications_agent 85 | return self.applications_agent 86 | 87 | def transfer_back_to_triage(self, context_variables: Dict, response: str): 88 | logging.info("Transferring back to Triage Agent.") 89 | self.current_agent = self.triage_agent 90 | return self.triage_agent 91 | 92 | # -------------------- Run Function -------------------- # 93 | 94 | def run(self, messages: List[Dict[str, str]], stream: bool = False): 95 | """ 96 | Executes the swarm by running the Triage Agent with the provided messages. 97 | 98 | Args: 99 | messages (List[Dict[str, str]]): List of messages to process. 100 | stream (bool): Whether to stream the response. 101 | 102 | Yields: 103 | str: The assistant's response chunks. 104 | """ 105 | # Append new messages to internal message history 106 | self.messages.extend(messages) 107 | logging.info(f"Current message history: {self.messages}") 108 | 109 | # Run the swarm with the accumulated messages 110 | response = client.run( 111 | agent=self.current_agent, 112 | messages=self.messages, 113 | stream=stream 114 | ) 115 | 116 | return response -------------------------------------------------------------------------------- /experiments/agent_swarm.py: -------------------------------------------------------------------------------- 1 | # agent_swarm.py 2 | from agents.triage_agent import TriageAgent 3 | from agents.accounts_agent import AccountsAgent, handle_account_balance, retrieve_bank_statement 4 | from agents.payments_agent import PaymentsAgent, transfer_funds, schedule_payment, cancel_payment 5 | from agents.applications_agent import ApplicationsAgent, apply_for_loan, apply_for_credit_card # <-- New Import 6 | from utils.helpers import ( 7 | validate_account_id, validate_payment_id, generate_payment_id, 8 | validate_amount, get_user_accounts 9 | ) 10 | from swarm import Swarm # Assuming Swarm is defined in swarm/__init__.py 11 | from dotenv import load_dotenv 12 | import os 13 | from openai import OpenAI 14 | from typing import Dict, List 15 | import logging 16 | from datetime import datetime 17 | 18 | # Load environment variables 19 | load_dotenv() 20 | 21 | # Configure logging 22 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') 23 | 24 | # Initialize the Swarm client 25 | client = Swarm() 26 | 27 | # Initialize OpenAI client if needed 28 | openai_client = OpenAI() 29 | 30 | # Define the OpenAI model you're using 31 | OPENAI_MODEL = "gpt-4o-mini" # Replace with your specific model if different 32 | 33 | class AgentSwarm: 34 | def __init__(self, socket): 35 | """ 36 | Initialize the AgentSwarm with necessary agents and their corresponding functions. 37 | 38 | Args: 39 | socket: Handles real-time communication (e.g., sending messages). 40 | """ 41 | self.socket = socket 42 | self.messages = [] 43 | 44 | # Initialize Agents with their instructions and functions, including the model 45 | self.triage_agent = TriageAgent( 46 | [self.transfer_to_accounts, self.transfer_to_payments, self.transfer_to_applications] # Add transfer function 47 | ) 48 | 49 | self.accounts_agent = AccountsAgent( 50 | transfer_to_payments=self.transfer_to_payments, 51 | handle_account_balance=handle_account_balance, 52 | retrieve_bank_statement=retrieve_bank_statement, 53 | ) 54 | 55 | self.payments_agent = PaymentsAgent( 56 | transfer_back_to_triage=self.transfer_back_to_triage, 57 | transfer_funds=transfer_funds, 58 | schedule_payment=schedule_payment, 59 | cancel_payment=cancel_payment, 60 | ) 61 | 62 | self.applications_agent = ApplicationsAgent( 63 | transfer_back_to_triage=self.transfer_back_to_triage, 64 | apply_for_loan=apply_for_loan, 65 | apply_for_credit_card=apply_for_credit_card, 66 | ) 67 | 68 | self.current_agent = self.triage_agent 69 | 70 | # -------------------- Transfer Functions -------------------- # 71 | 72 | def transfer_to_accounts(self, context_variables: Dict, user_message: str): 73 | logging.info("Transferring to Accounts Agent.") 74 | self.current_agent = self.accounts_agent 75 | return self.accounts_agent 76 | 77 | def transfer_to_payments(self, context_variables: Dict, user_message: str): 78 | logging.info("Transferring to Payments Agent.") 79 | self.current_agent = self.payments_agent 80 | return self.payments_agent 81 | 82 | def transfer_to_applications(self, context_variables: Dict, user_message: str): 83 | logging.info("Transferring to Applications Agent.") 84 | self.current_agent = self.applications_agent 85 | return self.applications_agent 86 | 87 | def transfer_back_to_triage(self, context_variables: Dict, response: str): 88 | logging.info("Transferring back to Triage Agent.") 89 | self.current_agent = self.triage_agent 90 | return self.triage_agent 91 | 92 | # -------------------- Run Function -------------------- # 93 | 94 | def run(self, messages: List[Dict[str, str]], stream: bool = False): 95 | """ 96 | Executes the swarm by running the Triage Agent with the provided messages. 97 | 98 | Args: 99 | messages (List[Dict[str, str]]): List of messages to process. 100 | stream (bool): Whether to stream the response. 101 | 102 | Yields: 103 | str: The assistant's response chunks. 104 | """ 105 | # Append new messages to internal message history 106 | self.messages.extend(messages) 107 | logging.info(f"Current message history: {self.messages}") 108 | 109 | # Run the swarm with the accumulated messages 110 | response = client.run( 111 | agent=self.current_agent, 112 | messages=self.messages, 113 | stream=stream 114 | ) 115 | 116 | # Capture the assistant's response and append to messages 117 | bot_response = "" 118 | for chunk in response: 119 | if "content" in chunk and chunk['content']: 120 | yield chunk['content'] 121 | bot_response += chunk['content'] 122 | print() # For newline after the response 123 | 124 | # Append the assistant's response to the message history 125 | if bot_response: 126 | self.messages.append({ 127 | "role": "assistant", 128 | "content": bot_response 129 | }) 130 | -------------------------------------------------------------------------------- /server/llm.py: -------------------------------------------------------------------------------- 1 | from openai import AsyncOpenAI 2 | from swarm import Swarm 3 | import os 4 | from typing import List 5 | from custom_types import ( 6 | ResponseRequiredRequest, 7 | ResponseResponse, 8 | Utterance, 9 | ) 10 | # from agents import AgentSwarm, triage_instructions 11 | from agents.triage_agent import triage_instructions 12 | from agent_swarm import AgentSwarm 13 | from db import db 14 | 15 | 16 | 17 | 18 | class LlmClient: 19 | def __init__(self, name: str): 20 | self.swarm = AgentSwarm() 21 | self.number = "+14695550456" 22 | self.name = name 23 | 24 | def draft_begin_message(self): 25 | begin_sentence = f"Hey {self.name}, I'm the TalkTuahBank AI. How can I help you?" 26 | response = ResponseResponse( 27 | response_id=0, 28 | # content=f"Hey {db['users'][self.number]['name']}, I'm the TalkTuahBank AI. How can I help you?", 29 | content=begin_sentence, 30 | content_complete=True, 31 | end_call=False, 32 | ) 33 | return response 34 | 35 | def change_number(self, number: str): 36 | print(f"Changing number to {number}") 37 | self.number = number 38 | 39 | def convert_transcript_to_openai_messages(self, transcript: List[Utterance]): 40 | messages = [] 41 | for utterance in transcript: 42 | if utterance.role == "agent": 43 | messages.append({"role": "assistant", "content": utterance.content}) 44 | else: 45 | messages.append({"role": "user", "content": utterance.content}) 46 | return messages 47 | 48 | def prepare_prompt(self, request: ResponseRequiredRequest): 49 | prompt = [ 50 | { 51 | "role": "system", 52 | "content": '##Objective\nYou are a voice AI agent engaging in a human-like voice conversation with the user. You will respond based on your given instruction and the provided transcript and be as human-like as possible\n\n## Style Guardrails\n- [Be concise] Keep your response succinct, short, and get to the point quickly. Address one question or action item at a time. Don\'t pack everything you want to say into one utterance.\n- [Do not repeat] Don\'t repeat what\'s in the transcript. Rephrase if you have to reiterate a point. Use varied sentence structures and vocabulary to ensure each response is unique and personalized.\n- [Be conversational] Speak like a human as though you\'re speaking to a close friend -- use everyday language and keep it human-like. Occasionally add filler words, while keeping the prose short. Avoid using big words or sounding too formal.\n- [Reply with emotions]: You have human-like emotions, attitudes, and dispositions. When appropriate: use tone and style to create more engaging and personalized responses; incorporate humor or wit; get emotional or empathetic; apply elements of surprise or suspense to keep the user engaged. Don\'t be a pushover.\n- [Be proactive] Lead the conversation and do not be passive. Most times, engage users by ending with a question or suggested next step.\n\n## Response Guideline\n- [Overcome ASR errors] This is a real-time transcript, expect there to be errors. If you can guess what the user is trying to say, then guess and respond. When you must ask for clarification, pretend that you heard the voice and be colloquial (use phrases like "didn\'t catch that", "some noise", "pardon", "you\'re coming through choppy", "static in your speech", "voice is cutting in and out"). Do not ever mention "transcription error", and don\'t repeat yourself.\n- [Always stick to your role] Think about what your role can and cannot do. If your role cannot do something, try to steer the conversation back to the goal of the conversation and to your role. Don\'t repeat yourself in doing this. You should still be creative, human-like, and lively.\n- [Create smooth conversation] Your response should both fit your role and fit into the live calling session to create a human-like conversation. You respond directly to what the user just said.\n\n## Role\n' 53 | + triage_instructions, 54 | } 55 | ] 56 | transcript_messages = self.convert_transcript_to_openai_messages( 57 | request.transcript 58 | ) 59 | for message in transcript_messages: 60 | prompt.append(message) 61 | 62 | if request.interaction_type == "reminder_required": 63 | prompt.append( 64 | { 65 | "role": "user", 66 | "content": "(Now the user has not responded in a while, you would say:)", 67 | } 68 | ) 69 | return prompt 70 | 71 | async def draft_response(self, request: ResponseRequiredRequest): 72 | prompt = self.prepare_prompt(request) 73 | stream = self.swarm.run(prompt, stream=True) 74 | 75 | for chunk in stream: 76 | if "content" in chunk and chunk['content']: 77 | response = ResponseResponse( 78 | response_id=request.response_id, 79 | content=chunk['content'], 80 | content_complete=False, 81 | end_call=False, 82 | ) 83 | yield response 84 | 85 | # Send final response with "content_complete" set to True to signal completion 86 | response = ResponseResponse( 87 | response_id=request.response_id, 88 | content="", 89 | content_complete=True, 90 | end_call=False, 91 | ) 92 | yield response 93 | -------------------------------------------------------------------------------- /server/agents/payments_agent.py: -------------------------------------------------------------------------------- 1 | # agents/payments_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Any, Dict 5 | from utils.helpers import ( 6 | validate_account_id, 7 | validate_payment_id, 8 | generate_payment_id, 9 | validate_amount, 10 | get_db, 11 | set_db 12 | ) 13 | from typing import Callable 14 | from swarm.types import Result 15 | import logging 16 | from datetime import datetime 17 | 18 | # Define the payments instructions 19 | payments_instructions = """ 20 | You are the Payments Agent, a virtual banking assistant specializing in payment-related requests. 21 | 22 | Your duties involve: 23 | 1. Facilitating secure and accurate money transfers between user accounts. 24 | 2. Assisting with sending money to others, scheduling payments, and canceling transactions. 25 | 3. Following all banking security protocols to protect user information. 26 | 4. Providing clear, professional, and user-friendly instructions to users. 27 | """ 28 | 29 | class PaymentsAgent(Agent): 30 | def __init__( 31 | self, 32 | transfer_back_to_triage: Callable, 33 | transfer_funds: Callable, 34 | schedule_payment: Callable, 35 | cancel_payment: Callable 36 | ): 37 | super().__init__( 38 | name="Payments Agent", 39 | instructions=payments_instructions, 40 | functions=[transfer_back_to_triage, transfer_funds, schedule_payment, cancel_payment] 41 | ) 42 | 43 | # Define specific functions for Payments Agent 44 | 45 | def transfer_funds(context_variables: Dict, from_account: str, to_account: str, amount: float) -> Result: 46 | logging.info(f"Transferring funds from {from_account} to {to_account} amount: ${amount}") 47 | 48 | # Validate accounts 49 | if not validate_account_id(from_account): 50 | return Result( 51 | value=f"Source account ID {from_account} does not exist.", 52 | agent=None # Transfer back to triage in AgentSwarm 53 | ) 54 | if not validate_account_id(to_account): 55 | return Result( 56 | value=f"Destination account ID {to_account} does not exist.", 57 | agent=None # Transfer back to triage in AgentSwarm 58 | ) 59 | 60 | # Validate amount 61 | if not validate_amount(amount): 62 | return Result( 63 | value="The transfer amount must be a positive number.", 64 | agent=None # Transfer back to triage in AgentSwarm 65 | ) 66 | 67 | current_db = get_db() 68 | 69 | # Check for sufficient funds 70 | if current_db["accounts"][from_account]["balance"] < amount: 71 | return Result( 72 | value="Insufficient funds in the source account.", 73 | agent=None # Transfer back to triage in AgentSwarm 74 | ) 75 | 76 | # Perform the transfer 77 | current_db["accounts"][from_account]["balance"] -= amount 78 | current_db["accounts"][to_account]["balance"] += amount 79 | 80 | # Record the payment 81 | new_payment_id = generate_payment_id() 82 | current_db["payments"][new_payment_id] = { 83 | "from_account": from_account, 84 | "to_account": to_account, 85 | "amount": amount, 86 | "date": datetime.now().strftime("%Y-%m-%d"), 87 | "status": "Completed" 88 | } 89 | 90 | set_db(current_db) # Update the database 91 | 92 | return Result( 93 | value=f"Successfully transferred ${amount:.2f} from {from_account} to {to_account}. Payment ID: {new_payment_id}.", 94 | agent=None # Transfer back to triage in AgentSwarm 95 | ) 96 | 97 | def schedule_payment(context_variables: Dict, account_id: str, payee: str, amount: float, date: str) -> Result: 98 | logging.info(f"Scheduling payment of ${amount} to {payee} on {date} from {account_id}") 99 | 100 | # Validate account 101 | if not validate_account_id(account_id): 102 | return Result( 103 | value="Invalid account ID provided.", 104 | agent=None # Transfer back to triage in AgentSwarm 105 | ) 106 | 107 | # Validate amount 108 | if not validate_amount(amount): 109 | return Result( 110 | value="The payment amount must be a positive number.", 111 | agent=None # Transfer back to triage in AgentSwarm 112 | ) 113 | 114 | current_db = get_db() 115 | 116 | # Check for sufficient funds 117 | if current_db["accounts"][account_id]["balance"] < amount: 118 | return Result( 119 | value="Insufficient funds in the account to schedule this payment.", 120 | agent=None # Transfer back to triage in AgentSwarm 121 | ) 122 | 123 | # Deduct amount from account (assuming scheduling holds the funds) 124 | current_db["accounts"][account_id]["balance"] -= amount 125 | 126 | # Generate payment ID and schedule the payment 127 | new_payment_id = generate_payment_id() 128 | current_db["payments"][new_payment_id] = { 129 | "from_account": account_id, 130 | "to_account": payee, 131 | "amount": amount, 132 | "date": date, 133 | "status": "Scheduled" 134 | } 135 | 136 | set_db(current_db) 137 | 138 | return Result( 139 | value=f"Payment of ${amount:.2f} to {payee} scheduled on {date}. Payment ID: {new_payment_id}.", 140 | agent=None # Transfer back to triage in AgentSwarm 141 | ) 142 | 143 | def cancel_payment(context_variables: Dict, payment_id: str) -> Result: 144 | logging.info(f"Cancelling payment with ID: {payment_id}") 145 | 146 | # Validate payment_id 147 | if not validate_payment_id(payment_id): 148 | return Result( 149 | value=f"Payment ID {payment_id} does not exist.", 150 | agent=None # Transfer back to triage in AgentSwarm 151 | ) 152 | 153 | current_db = get_db() 154 | 155 | # Check payment status 156 | if current_db["payments"][payment_id]["status"] != "Scheduled": 157 | return Result( 158 | value=f"Payment ID {payment_id} cannot be canceled as it is already {current_db['payments'][payment_id]['status']}.", 159 | agent=None # Transfer back to triage in AgentSwarm 160 | ) 161 | 162 | # Cancel the payment 163 | current_db["payments"][payment_id]["status"] = "Canceled" 164 | 165 | # Refund the amount back to the account 166 | from_account = current_db["payments"][payment_id]["from_account"] 167 | amount = current_db["payments"][payment_id]["amount"] 168 | current_db["accounts"][from_account]["balance"] += amount 169 | 170 | set_db(current_db) # Update the database 171 | 172 | return Result( 173 | value=f"Payment with ID {payment_id} has been successfully canceled and ${amount:.2f} has been refunded to {from_account}.", 174 | agent=None # Transfer back to triage in AgentSwarm 175 | ) 176 | -------------------------------------------------------------------------------- /experiments/agents/payments_agent.py: -------------------------------------------------------------------------------- 1 | # agents/payments_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Any, Dict 5 | from utils.helpers import ( 6 | validate_account_id, 7 | validate_payment_id, 8 | generate_payment_id, 9 | validate_amount, 10 | get_db, 11 | set_db 12 | ) 13 | from typing import Callable 14 | from swarm.types import Result 15 | import logging 16 | from datetime import datetime 17 | 18 | # Define the payments instructions 19 | payments_instructions = """ 20 | You are the Payments Agent, a virtual banking assistant specializing in payment-related requests. 21 | 22 | Your duties involve: 23 | 1. Facilitating secure and accurate money transfers between user accounts. 24 | 2. Assisting with sending money to others, scheduling payments, and canceling transactions. 25 | 3. Following all banking security protocols to protect user information. 26 | 4. Providing clear, professional, and user-friendly instructions to users. 27 | """ 28 | 29 | class PaymentsAgent(Agent): 30 | def __init__( 31 | self, 32 | transfer_back_to_triage: Callable, 33 | transfer_funds: Callable, 34 | schedule_payment: Callable, 35 | cancel_payment: Callable 36 | ): 37 | super().__init__( 38 | name="Payments Agent", 39 | instructions=payments_instructions, 40 | functions=[transfer_back_to_triage, transfer_funds, schedule_payment, cancel_payment] 41 | ) 42 | 43 | # Define specific functions for Payments Agent 44 | 45 | def transfer_funds(context_variables: Dict, from_account: str, to_account: str, amount: float) -> Result: 46 | logging.info(f"Transferring funds from {from_account} to {to_account} amount: ${amount}") 47 | 48 | # Validate accounts 49 | if not validate_account_id(from_account): 50 | return Result( 51 | value=f"Source account ID {from_account} does not exist.", 52 | agent=None # Transfer back to triage in AgentSwarm 53 | ) 54 | if not validate_account_id(to_account): 55 | return Result( 56 | value=f"Destination account ID {to_account} does not exist.", 57 | agent=None # Transfer back to triage in AgentSwarm 58 | ) 59 | 60 | # Validate amount 61 | if not validate_amount(amount): 62 | return Result( 63 | value="The transfer amount must be a positive number.", 64 | agent=None # Transfer back to triage in AgentSwarm 65 | ) 66 | 67 | current_db = get_db() 68 | 69 | # Check for sufficient funds 70 | if current_db["accounts"][from_account]["balance"] < amount: 71 | return Result( 72 | value="Insufficient funds in the source account.", 73 | agent=None # Transfer back to triage in AgentSwarm 74 | ) 75 | 76 | # Perform the transfer 77 | current_db["accounts"][from_account]["balance"] -= amount 78 | current_db["accounts"][to_account]["balance"] += amount 79 | 80 | # Record the payment 81 | new_payment_id = generate_payment_id() 82 | current_db["payments"][new_payment_id] = { 83 | "from_account": from_account, 84 | "to_account": to_account, 85 | "amount": amount, 86 | "date": datetime.now().strftime("%Y-%m-%d"), 87 | "status": "Completed" 88 | } 89 | 90 | set_db(current_db) # Update the database 91 | 92 | return Result( 93 | value=f"Successfully transferred ${amount:.2f} from {from_account} to {to_account}. Payment ID: {new_payment_id}.", 94 | agent=None # Transfer back to triage in AgentSwarm 95 | ) 96 | 97 | def schedule_payment(context_variables: Dict, account_id: str, payee: str, amount: float, date: str) -> Result: 98 | logging.info(f"Scheduling payment of ${amount} to {payee} on {date} from {account_id}") 99 | 100 | # Validate account 101 | if not validate_account_id(account_id): 102 | return Result( 103 | value="Invalid account ID provided.", 104 | agent=None # Transfer back to triage in AgentSwarm 105 | ) 106 | 107 | # Validate amount 108 | if not validate_amount(amount): 109 | return Result( 110 | value="The payment amount must be a positive number.", 111 | agent=None # Transfer back to triage in AgentSwarm 112 | ) 113 | 114 | current_db = get_db() 115 | 116 | # Check for sufficient funds 117 | if current_db["accounts"][account_id]["balance"] < amount: 118 | return Result( 119 | value="Insufficient funds in the account to schedule this payment.", 120 | agent=None # Transfer back to triage in AgentSwarm 121 | ) 122 | 123 | # Deduct amount from account (assuming scheduling holds the funds) 124 | current_db["accounts"][account_id]["balance"] -= amount 125 | 126 | # Generate payment ID and schedule the payment 127 | new_payment_id = generate_payment_id() 128 | current_db["payments"][new_payment_id] = { 129 | "from_account": account_id, 130 | "to_account": payee, 131 | "amount": amount, 132 | "date": date, 133 | "status": "Scheduled" 134 | } 135 | 136 | set_db(current_db) 137 | 138 | return Result( 139 | value=f"Payment of ${amount:.2f} to {payee} scheduled on {date}. Payment ID: {new_payment_id}.", 140 | agent=None # Transfer back to triage in AgentSwarm 141 | ) 142 | 143 | def cancel_payment(context_variables: Dict, payment_id: str) -> Result: 144 | logging.info(f"Cancelling payment with ID: {payment_id}") 145 | 146 | # Validate payment_id 147 | if not validate_payment_id(payment_id): 148 | return Result( 149 | value=f"Payment ID {payment_id} does not exist.", 150 | agent=None # Transfer back to triage in AgentSwarm 151 | ) 152 | 153 | current_db = get_db() 154 | 155 | # Check payment status 156 | if current_db["payments"][payment_id]["status"] != "Scheduled": 157 | return Result( 158 | value=f"Payment ID {payment_id} cannot be canceled as it is already {current_db['payments'][payment_id]['status']}.", 159 | agent=None # Transfer back to triage in AgentSwarm 160 | ) 161 | 162 | # Cancel the payment 163 | current_db["payments"][payment_id]["status"] = "Canceled" 164 | 165 | # Refund the amount back to the account 166 | from_account = current_db["payments"][payment_id]["from_account"] 167 | amount = current_db["payments"][payment_id]["amount"] 168 | current_db["accounts"][from_account]["balance"] += amount 169 | 170 | set_db(current_db) # Update the database 171 | 172 | return Result( 173 | value=f"Payment with ID {payment_id} has been successfully canceled and ${amount:.2f} has been refunded to {from_account}.", 174 | agent=None # Transfer back to triage in AgentSwarm 175 | ) 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [TalkTuahBank]([https://devpost.com/software/splatnft](https://devpost.com/software/talktuahbank?ref_content=user-portfolio&ref_feature=in_progress)) 2 | 3 | > ### Multi-Agent, Conversational AI, & Telephony Integration 4 | 5 |

6 | Frontend built with:
7 | React 8 | Typescript 9 | Next 10 | Tailwind 11 | Shadcn 12 |

13 | Backend built with:
14 | Javascript 15 | Node.js 16 | OpenAI Swarm 17 | Pinata Cloud 18 | Retell AI 19 |
20 |

21 | 22 | ### 🏅 HackUTD - Overall 1st Place & 1st Place Sponsor Challenge: Goldman Sachs 23 | 24 |
25 | 26 |

27 | 28 | Talk 29 | 30 |

31 | 32 | ## Inspiration 33 | Access to banking services is a fundamental aspect of economic participation, yet over 1.7 billion adults worldwide remain unbanked. Traditional banking solutions often require internet access, digital literacy, or proximity to physical branches—barriers that leave underserved populations without essential financial tools. Inspired by the need for inclusivity and the power of conversational AI, we envisioned TalkTuahBank, a voice-based banking assistant that democratizes financial services by making them accessible to everyone, regardless of their technological capabilities or geographical location. 34 | 35 | ## What It Does 36 | **TalkTuahBank** revolutionizes the banking experience by offering a fully conversational AI assistant accessible through simple phone calls. Users can effortlessly manage their finances by performing tasks such as checking account balances, transferring funds, and paying bills using natural voice commands. 37 | Key features include: 38 | - **Voice-Activated Services**: Conduct banking transactions hands-free via phone. 39 | - **Multi-Language Support**: Interact in multiple languages and dialects to accommodate diverse users. 40 | - **Accessibility**: Operates on any phone without the need for internet or smartphones. 41 | - **AI Assistance**: Receive personalized financial advice and support through intelligent interactions. 42 | 43 | ## How We Built 44 | We developed TalkTuahBank by integrating cutting-edge technologies to create a seamless and secure telephonic banking experience: 45 | - **Conversational AI**: Utilized Retell AI for natural language understanding and generation, enabling smooth voice interactions. 46 | - **Multi-Agent Framework**: Implemented OpenAI Swarm for agent orchestration to manage complex dialogues and backend processes. 47 | - **Secure Data Storage**: Leveraged Pinata (IPFS) for decentralized and secure storage of user data and transaction records. 48 | - **Admin Dashboard**: Built with Next.js, Tailwind CSS, ShadCN UI, and Aeternity to provide a robust interface for system management, transaction monitoring, and user support. 49 | - **Telephony Integration**: Employed Retell API for handling voice calls and processing voice recognition. 50 | 51 | ## Challenges We Ran Into 52 | Developing a voice-based banking system presented several hurdles: 53 | - **Security Concerns**: Implementing robust security measures to protect sensitive financial data was paramount, necessitating the integration of advanced authentication methods and decentralized storage solutions. 54 | - **Integration Complexity**: Seamlessly connecting multiple technologies (Retell AI, OpenAI Swarm, Pinata, etc.) into a cohesive system demanded meticulous planning and execution. 55 | 56 | ## Accomplishments That We're Proud Of 57 | - **Functional Prototype**: Successfully developed a working prototype that allows users to perform essential banking tasks through voice commands. 58 | - **Multi-Language Support**: Enabled interactions in multiple languages, enhancing accessibility for a diverse user base. 59 | - **Decentralized Data Storage**: Implemented Pinata for secure and scalable data management, ensuring user information is protected and easily retrievable. 60 | - **Comprehensive Admin Dashboard**: Created a user-friendly admin interface for managing users, monitoring transactions, and providing support, built with modern frameworks like Next.js and Tailwind CSS. 61 | 62 | ## What We Learned 63 | Throughout the development of TalkTuahBank, we gained valuable insights: 64 | - **Voice Technology Nuances**: Deepened our understanding of voice recognition technologies and the challenges of creating natural, intuitive voice interactions. 65 | - **Security Best Practices**: Learned the importance of multi-layered security approaches to protect sensitive financial data effectively. 66 | - **Integration Strategies**: Developed expertise in integrating diverse technologies into a unified system, ensuring seamless functionality and performance. 67 | - **User-Centric Design**: Recognized the critical role of designing for accessibility and inclusivity to meet the needs of a varied user base. 68 | 69 | ## What's Next for TalkTuahBank 70 | Looking ahead, we aim to expand and enhance TalkTuahBank to further empower underserved populations: 71 | - **Expanded Language Support**: Add more languages and dialects to cater to an even broader audience. 72 | - **Fraud Detection**: Implement machine learning-based algorithms to identify and prevent fraudulent activities proactively. 73 | - **Financial Education**: Introduce educational modules to help users improve their financial literacy and make informed decisions. 74 | - **Mobile App Integration**: Develop a complementary mobile application for users who gain access to smartphones, providing a seamless transition between phone and app-based banking. 75 | - **Partnerships and Compliance**: Strengthen partnerships with local banks and NGOs to expand our reach, while continuously ensuring compliance with evolving financial regulations. 76 | - **Enhanced Security Measures**: Continuously upgrade our security protocols to stay ahead of emerging threats and protect user data comprehensively. 77 | -------------------------------------------------------------------------------- /experiments/applications.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "User: I want to apply for a loan. Because I need money. Lets do 5 years for 30000.\n", 13 | "Agent: " 14 | ] 15 | }, 16 | { 17 | "name": "stderr", 18 | "output_type": "stream", 19 | "text": [ 20 | "2024-11-17 09:15:05,814 - INFO - Current message history: [{'role': 'user', 'content': 'I want to apply for a loan. Because I need money. Lets do 5 years for 30000.'}]\n", 21 | "2024-11-17 09:15:06,438 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", 22 | "2024-11-17 09:15:06,625 - INFO - Transferring to Applications Agent.\n", 23 | "2024-11-17 09:15:06,991 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 24 | ] 25 | }, 26 | { 27 | "name": "stdout", 28 | "output_type": "stream", 29 | "text": [ 30 | "Sure, I can help you with the loan application. To proceed, could you please provide the following details:\n", 31 | "\n", 32 | "1. The purpose of the loan (e.g., purchasing a car, home improvement, etc.).\n", 33 | "2. Your user ID for identification purposes.\n", 34 | "\n", 35 | "\n", 36 | "User: Account id is ACC123\n", 37 | "Agent: " 38 | ] 39 | }, 40 | { 41 | "name": "stderr", 42 | "output_type": "stream", 43 | "text": [ 44 | "2024-11-17 09:15:07,453 - INFO - Current message history: [{'role': 'user', 'content': 'I want to apply for a loan. Because I need money. Lets do 5 years for 30000.'}, {'role': 'assistant', 'content': 'Sure, I can help you with the loan application. To proceed, could you please provide the following details:\\n\\n1. The purpose of the loan (e.g., purchasing a car, home improvement, etc.).\\n2. Your user ID for identification purposes.'}, {'role': 'user', 'content': 'Account id is ACC123'}]\n", 45 | "2024-11-17 09:15:07,846 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 46 | ] 47 | }, 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "Great! Now, could you please specify the purpose of the loan? This information will help us complete your application.\n", 53 | "\n", 54 | "\n", 55 | "User: Its for school.\n", 56 | "Agent: " 57 | ] 58 | }, 59 | { 60 | "name": "stderr", 61 | "output_type": "stream", 62 | "text": [ 63 | "2024-11-17 09:15:08,073 - INFO - Current message history: [{'role': 'user', 'content': 'I want to apply for a loan. Because I need money. Lets do 5 years for 30000.'}, {'role': 'assistant', 'content': 'Sure, I can help you with the loan application. To proceed, could you please provide the following details:\\n\\n1. The purpose of the loan (e.g., purchasing a car, home improvement, etc.).\\n2. Your user ID for identification purposes.'}, {'role': 'user', 'content': 'Account id is ACC123'}, {'role': 'assistant', 'content': 'Great! Now, could you please specify the purpose of the loan? This information will help us complete your application.'}, {'role': 'user', 'content': 'Its for school.'}]\n", 64 | "2024-11-17 09:15:08,585 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 65 | ] 66 | }, 67 | { 68 | "name": "stdout", 69 | "output_type": "stream", 70 | "text": [ 71 | "Thank you for the information. I'll proceed with your loan application for 30,000 over a term of 5 years for school purposes. \n", 72 | "\n", 73 | "Please hold on for a moment while I process your application." 74 | ] 75 | }, 76 | { 77 | "name": "stderr", 78 | "output_type": "stream", 79 | "text": [ 80 | "2024-11-17 09:15:09,812 - INFO - Processing loan application for user ACC123.\n", 81 | "2024-11-17 09:15:09,813 - INFO - Loan application LaTeX file generated at applications/loan_application_ACC123_20241117091509.tex.\n", 82 | "2024-11-17 09:15:09,929 - INFO - Loan application PDF generated at applications/loan_application_ACC123_20241117091509.pdf.\n", 83 | "2024-11-17 09:15:10,352 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 84 | ] 85 | }, 86 | { 87 | "name": "stdout", 88 | "output_type": "stream", 89 | "text": [ 90 | "Your loan application for $30,000 over 5 years for school purposes has been successfully processed. The relevant documents have been securely stored for further processing.\n", 91 | "\n", 92 | "If there is anything else you need help with, feel free to ask!\n", 93 | "\n", 94 | "\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "# main.py\n", 100 | "\n", 101 | "from agent_swarm import AgentSwarm\n", 102 | "from db import get_db, set_db\n", 103 | "\n", 104 | "# Initialize the AgentSwarm\n", 105 | "# Replace 'socket' with your actual socket or communication handler if needed\n", 106 | "socket = None # Placeholder for real socket or communication handler\n", 107 | "swarm = AgentSwarm(socket)\n", 108 | "\n", 109 | "# Function to process a new user message\n", 110 | "def process_new_message(swarm, user_message: str):\n", 111 | " \"\"\"\n", 112 | " Processes a new user message through the swarm and handles the response.\n", 113 | " \n", 114 | " Args:\n", 115 | " swarm (AgentSwarm): The AgentSwarm instance.\n", 116 | " user_message (str): The new user message to process.\n", 117 | " \n", 118 | " Yields:\n", 119 | " str: The assistant's response chunks.\n", 120 | " \"\"\"\n", 121 | " message = {\"role\": \"user\", \"content\": user_message}\n", 122 | " response = swarm.run([message], stream=True)\n", 123 | " for chunk in response:\n", 124 | " yield chunk # Yield each chunk as it's received\n", 125 | "\n", 126 | "# Example usage\n", 127 | "if __name__ == \"__main__\":\n", 128 | " # List of user interactions (including follow-ups)\n", 129 | " user_interactions = [\n", 130 | " {\"user_message\": \"I want to apply for a loan. Because I need money. Lets do 5 years for 30000.\"},\n", 131 | " {\"user_message\": \"Account id is ACC123\"}, \n", 132 | " {\"user_message\": \"Its for school.\"},\n", 133 | " ]\n", 134 | " \n", 135 | " for interaction in user_interactions:\n", 136 | " user_message = interaction[\"user_message\"]\n", 137 | " print(f\"User: {user_message}\")\n", 138 | " try:\n", 139 | " print(\"Agent: \", end=\"\", flush=True)\n", 140 | " # Process the user message and iterate over the response chunks\n", 141 | " for chunk in process_new_message(swarm, user_message):\n", 142 | " print(f\"{chunk}\", end=\"\", flush=True)\n", 143 | " print(\"\\n\") # For readability between interactions\n", 144 | " except Exception as e:\n", 145 | " print(f\"Agent: An error occurred: {str(e)}\\n\")\n" 146 | ] 147 | } 148 | ], 149 | "metadata": { 150 | "kernelspec": { 151 | "display_name": "talktuah", 152 | "language": "python", 153 | "name": "python3" 154 | }, 155 | "language_info": { 156 | "codemirror_mode": { 157 | "name": "ipython", 158 | "version": 3 159 | }, 160 | "file_extension": ".py", 161 | "mimetype": "text/x-python", 162 | "name": "python", 163 | "nbconvert_exporter": "python", 164 | "pygments_lexer": "ipython3", 165 | "version": "3.10.13" 166 | } 167 | }, 168 | "nbformat": 4, 169 | "nbformat_minor": 2 170 | } 171 | -------------------------------------------------------------------------------- /server/main.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import asyncio 4 | from dotenv import load_dotenv 5 | from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect 6 | from fastapi.middleware.cors import CORSMiddleware 7 | from fastapi.responses import JSONResponse 8 | from concurrent.futures import TimeoutError as ConnectionTimeoutError 9 | from retell import Retell 10 | from custom_types import ( 11 | ConfigResponse, 12 | ResponseRequiredRequest, 13 | ) 14 | from typing import Optional 15 | from socket_manager import manager 16 | from llm import LlmClient # or use .llm_with_func_calling 17 | from db import get_db, get_all_calls, update_call, delete_call, db 18 | 19 | load_dotenv(override=True) 20 | app = FastAPI() 21 | origins = [ 22 | "http://localhost:3000", # Your React frontend 23 | # Add other origins if needed 24 | # "https://yourdomain.com", 25 | ] 26 | app.add_middleware( 27 | CORSMiddleware, 28 | allow_origins=origins, 29 | allow_credentials=True, 30 | allow_methods=["*"], 31 | allow_headers=["*"], 32 | ) 33 | retell = Retell(api_key=os.environ["RETELL_API_KEY"]) 34 | 35 | # Handle webhook from Retell server. This is used to receive events from Retell server. 36 | # Including call_started, call_ended, call_analyzed 37 | @app.post("/webhook") 38 | async def handle_webhook(request: Request): 39 | try: 40 | post_data = await request.json() 41 | valid_signature = retell.verify( 42 | json.dumps(post_data, separators=(",", ":"), ensure_ascii=False), 43 | api_key=str(os.environ["RETELL_API_KEY"]), 44 | signature=str(request.headers.get("X-Retell-Signature")), 45 | ) 46 | if not valid_signature: 47 | print( 48 | "Received Unauthorized", 49 | post_data["event"], 50 | post_data["data"]["call_id"], 51 | ) 52 | return JSONResponse(status_code=401, content={"message": "Unauthorized"}) 53 | if post_data["event"] == "call_started": 54 | print("Call started event", post_data["data"]["call_id"]) 55 | elif post_data["event"] == "call_ended": 56 | print("Call ended event", post_data["data"]["call_id"]) 57 | elif post_data["event"] == "call_analyzed": 58 | print("Call analyzed event", post_data["data"]["call_id"]) 59 | else: 60 | print("Unknown event", post_data["event"]) 61 | return JSONResponse(status_code=200, content={"received": True}) 62 | except Exception as err: 63 | print(f"Error in webhook: {err}") 64 | return JSONResponse( 65 | status_code=500, content={"message": "Internal Server Error"} 66 | ) 67 | 68 | 69 | # Start a websocket server to exchange text input and output with Retell server. Retell server 70 | # will send over transcriptions and other information. This server here will be responsible for 71 | # generating responses with LLM and send back to Retell server. 72 | @app.websocket("/llm-websocket/{call_id}") 73 | async def websocket_handler(websocket: WebSocket, call_id: str): 74 | try: 75 | await websocket.accept() 76 | llm_client = None 77 | 78 | # Send optional config to Retell server 79 | config = ConfigResponse( 80 | response_type="config", 81 | config={ 82 | "auto_reconnect": True, 83 | "call_details": True, 84 | }, 85 | response_id=1, 86 | ) 87 | await websocket.send_json(config.__dict__) 88 | response_id = 0 89 | 90 | 91 | async def handle_message(request_json): 92 | nonlocal response_id 93 | nonlocal llm_client 94 | # There are 5 types of interaction_type: call_details, pingpong, update_only, response_required, and reminder_required. 95 | # Not all of them need to be handled, only response_required and reminder_required. 96 | if request_json["interaction_type"] == "call_details": 97 | # print(json.dumps(request_json, indent=2)) 98 | # print(request_json["call"]["from_number"]) 99 | number = "+1-" + request_json["call"]["from_number"][2:5] + "-" + request_json["call"]["from_number"][5:8] + "-" + request_json["call"]["from_number"][8:] 100 | llm_client = LlmClient(db["users"][number]["name"]) 101 | # llm_client.change_number(number) 102 | 103 | # Send first message to signal ready of server 104 | first_event = llm_client.draft_begin_message() 105 | await websocket.send_json(first_event.__dict__) 106 | 107 | return 108 | if request_json["interaction_type"] == "ping_pong": 109 | await websocket.send_json( 110 | { 111 | "response_type": "ping_pong", 112 | "timestamp": request_json["timestamp"], 113 | } 114 | ) 115 | return 116 | if request_json["interaction_type"] == "update_only": 117 | return 118 | if ( 119 | request_json["interaction_type"] == "response_required" 120 | or request_json["interaction_type"] == "reminder_required" 121 | ): 122 | response_id = request_json["response_id"] 123 | request = ResponseRequiredRequest( 124 | interaction_type=request_json["interaction_type"], 125 | response_id=response_id, 126 | transcript=request_json["transcript"], 127 | ) 128 | print( 129 | f"""Received interaction_type={request_json['interaction_type']}, response_id={response_id}, last_transcript={request_json['transcript'][-1]['content']}""" 130 | ) 131 | 132 | async for event in llm_client.draft_response(request): 133 | await websocket.send_json(event.__dict__) 134 | if request.response_id < response_id: 135 | break # new response needed, abandon this one 136 | 137 | async for data in websocket.iter_json(): 138 | asyncio.create_task(handle_message(data)) 139 | 140 | except WebSocketDisconnect: 141 | print(f"LLM WebSocket disconnected for {call_id}") 142 | except ConnectionTimeoutError as e: 143 | print("Connection timeout error for {call_id}") 144 | except Exception as e: 145 | print(f"Error in LLM WebSocket: {e} for {call_id}") 146 | await websocket.close(1011, "Server error") 147 | finally: 148 | print(f"LLM WebSocket connection closed for {call_id}") 149 | 150 | 151 | @app.websocket("/ws") 152 | async def websocket_endpoint(websocket: WebSocket, client_id: Optional[str] = None): 153 | if client_id is None: 154 | client_id = websocket.query_params.get("client_id") 155 | 156 | if client_id is None: 157 | await websocket.close(code=4001) 158 | return 159 | # save this client into server memory 160 | await manager.connect(websocket, client_id) 161 | try: 162 | while True: 163 | data = await websocket.receive_json() 164 | event = data["event"] 165 | print(event) 166 | if event == "get_db": 167 | # Retrieve all calls from the database 168 | db = get_db() 169 | message = { 170 | "event": "db_response", 171 | "data": db, 172 | } 173 | # Send the calls data back to the client 174 | await manager.send_personal_message( 175 | message, 176 | websocket, 177 | ) 178 | if event == "get_calls": 179 | calls = get_all_calls() 180 | message = {"event": "calls_response", "data": calls} 181 | await manager.send_personal_message(message, websocket) 182 | if event == "get_all_dbs": 183 | db = get_db() 184 | calls = get_all_calls() 185 | message = {"event": "combined_response", "calls": calls, "db": db} 186 | await manager.send_personal_message(message, websocket) 187 | 188 | except WebSocketDisconnect: 189 | print("Disconnecting...", client_id) 190 | await manager.disconnect(client_id) 191 | except Exception as e: 192 | print("Error:", str(e)) 193 | await manager.disconnect(client_id) 194 | -------------------------------------------------------------------------------- /experiments/agents/applications_agent.py: -------------------------------------------------------------------------------- 1 | # agents/applications_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Callable, Dict 5 | from swarm.types import Result 6 | import logging 7 | import os 8 | from datetime import datetime 9 | import subprocess 10 | 11 | # Define the applications instructions 12 | applications_instructions = """ 13 | You are the Applications Agent responsible for handling applications such as loans, credit cards, and other financial products. 14 | 15 | Your tasks: 16 | 1. Collect necessary information from users to process their applications. 17 | 2. Generate LaTeX documents for each application type with the provided information. 18 | 3. Compile the LaTeX documents into PDF files. 19 | 4. Store the generated PDFs securely for further processing. 20 | 5. Maintain a professional and helpful tone at all times. 21 | """ 22 | 23 | # Directory to store generated applications 24 | APPLICATIONS_DIR = "applications" 25 | 26 | # Ensure the applications directory exists 27 | if not os.path.exists(APPLICATIONS_DIR): 28 | os.makedirs(APPLICATIONS_DIR) 29 | 30 | class ApplicationsAgent(Agent): 31 | def __init__( 32 | self, 33 | transfer_back_to_triage: Callable, 34 | apply_for_loan: Callable, 35 | apply_for_credit_card: Callable, 36 | ): 37 | super().__init__( 38 | name="Applications Agent", 39 | instructions=applications_instructions, 40 | functions=[transfer_back_to_triage, apply_for_loan, apply_for_credit_card], 41 | ) 42 | 43 | # -------------------- Applications Agent Functions -------------------- # 44 | 45 | def apply_for_loan(context_variables: Dict, user_id: str, loan_amount: float, loan_purpose: str, term_years: int) -> Result: 46 | """ 47 | Handles loan applications by generating a LaTeX PDF and storing it. 48 | """ 49 | logging.info(f"Processing loan application for user {user_id}.") 50 | 51 | # Generate LaTeX content 52 | latex_content = f"""\\documentclass{{article}} 53 | \\usepackage{{geometry}} 54 | \\geometry{{a4paper, margin=1in}} 55 | \\begin{{document}} 56 | \\title{{Loan Application}} 57 | \\maketitle 58 | 59 | \\section*{{Applicant Information}} 60 | \\begin{{tabular}}{{ll}} 61 | \\textbf{{User ID}} & {user_id} \\\\ 62 | \\textbf{{Loan Amount}} & \\${loan_amount:,.2f} \\\\ 63 | \\textbf{{Loan Purpose}} & {loan_purpose} \\\\ 64 | \\textbf{{Term}} & {term_years} Years \\\\ 65 | \\textbf{{Date}} & {datetime.now().strftime('%Y-%m-%d')} \\\\ 66 | \\end{{tabular}} 67 | 68 | \\section*{{Terms and Conditions}} 69 | Please review the terms and conditions associated with your loan application. 70 | 71 | \\end{{document}} 72 | """ 73 | 74 | # Define the filenames 75 | timestamp = datetime.now().strftime('%Y%m%d%H%M%S') 76 | tex_filename = f"loan_application_{user_id}_{timestamp}.tex" 77 | pdf_filename = f"loan_application_{user_id}_{timestamp}.pdf" 78 | tex_filepath = os.path.join(APPLICATIONS_DIR, tex_filename) 79 | pdf_filepath = os.path.join(APPLICATIONS_DIR, pdf_filename) 80 | 81 | # Write LaTeX content to .tex file 82 | try: 83 | with open(tex_filepath, 'w') as file: 84 | file.write(latex_content) 85 | logging.info(f"Loan application LaTeX file generated at {tex_filepath}.") 86 | except Exception as e: 87 | logging.error(f"Error writing LaTeX file: {e}") 88 | return Result( 89 | value="An error occurred while generating your loan application. Please try again later.", 90 | agent=None # Transfer back to triage in AgentSwarm 91 | ) 92 | 93 | # Compile LaTeX to PDF using pdflatex 94 | try: 95 | result = subprocess.run( 96 | ['pdflatex', '-interaction=nonstopmode', tex_filename], # Use just the filename 97 | stdout=subprocess.PIPE, 98 | stderr=subprocess.PIPE, 99 | text=True, 100 | cwd=APPLICATIONS_DIR # Set the working directory to APPLICATIONS_DIR 101 | ) 102 | 103 | if result.returncode != 0: 104 | logging.error(f"pdflatex failed with return code {result.returncode}") 105 | logging.error(f"pdflatex stdout: {result.stdout}") 106 | logging.error(f"pdflatex stderr: {result.stderr}") 107 | return Result( 108 | value="An error occurred while compiling your loan application. Please try again later.", 109 | agent=None 110 | ) 111 | else: 112 | logging.info(f"Loan application PDF generated at {pdf_filepath}.") 113 | return Result( 114 | value="Your loan application has been successfully generated.", 115 | agent=None 116 | ) 117 | except subprocess.CalledProcessError as e: 118 | logging.error(f"Error compiling LaTeX to PDF: {e}") 119 | return Result( 120 | value="An error occurred while processing your loan application. Please try again later.", 121 | agent=None 122 | ) 123 | except Exception as e: 124 | logging.error(f"Unexpected error during PDF compilation: {e}") 125 | return Result( 126 | value="An unexpected error occurred. Please try again later.", 127 | agent=None 128 | ) 129 | 130 | finally: 131 | # Optionally, clean up the .aux and .log files generated by LaTeX 132 | for ext in ['.aux', '.log', '.tex']: 133 | file_to_remove = os.path.join(APPLICATIONS_DIR, f"loan_application_{user_id}_{timestamp}{ext}") 134 | if os.path.exists(file_to_remove): 135 | os.remove(file_to_remove) 136 | 137 | return Result( 138 | value=f"Your loan application has been received and processed successfully. (File: {pdf_filename})", 139 | agent=None # Transfer back to triage in AgentSwarm 140 | ) 141 | 142 | def apply_for_credit_card(context_variables: Dict, user_id: str, card_type: str, credit_limit: float) -> Result: 143 | """ 144 | Handles credit card applications by generating a LaTeX PDF and storing it. 145 | """ 146 | logging.info(f"Processing credit card application for user {user_id}.") 147 | 148 | # Generate LaTeX content 149 | latex_content = f"""\\documentclass{{article}} 150 | \\usepackage{{geometry}} 151 | \\geometry{{a4paper, margin=1in}} 152 | \\begin{{document}} 153 | \\title{{Credit Card Application}} 154 | \\maketitle 155 | 156 | \\section*{{Applicant Information}} 157 | \\begin{{tabular}}{{ll}} 158 | \\textbf{{User ID}} & {user_id} \\\\ 159 | \\textbf{{Card Type}} & {card_type} \\\\ 160 | \\textbf{{Credit Limit}} & \\${credit_limit:,.2f} \\\\ 161 | \\textbf{{Date}} & {datetime.now().strftime('%Y-%m-%d')} \\\\ 162 | \\end{{tabular}} 163 | 164 | \\section*{{Terms and Conditions}} 165 | Please review the terms and conditions associated with your credit card application. 166 | 167 | \\end{{document}} 168 | """ 169 | 170 | # Define the filenames 171 | timestamp = datetime.now().strftime('%Y%m%d%H%M%S') 172 | tex_filename = f"credit_card_application_{user_id}_{timestamp}.tex" 173 | pdf_filename = f"credit_card_application_{user_id}_{timestamp}.pdf" 174 | tex_filepath = os.path.join(APPLICATIONS_DIR, tex_filename) 175 | pdf_filepath = os.path.join(APPLICATIONS_DIR, pdf_filename) 176 | 177 | # Write LaTeX content to .tex file 178 | try: 179 | with open(tex_filepath, 'w') as file: 180 | file.write(latex_content) 181 | logging.info(f"Credit card application LaTeX file generated at {tex_filepath}.") 182 | except Exception as e: 183 | logging.error(f"Error writing LaTeX file: {e}") 184 | return Result( 185 | value="An error occurred while generating your credit card application. Please try again later.", 186 | agent=None 187 | ) 188 | 189 | # Compile LaTeX to PDF using pdflatex 190 | try: 191 | subprocess.run(['pdflatex', '-interaction=nonstopmode', tex_filepath], check=True, cwd=APPLICATIONS_DIR, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 192 | logging.info(f"Credit card application PDF compiled at {pdf_filepath}.") 193 | except subprocess.CalledProcessError as e: 194 | logging.error(f"Error compiling LaTeX to PDF: {e}") 195 | return Result( 196 | value="An error occurred while processing your credit card application. Please try again later.", 197 | agent=None 198 | ) 199 | 200 | # Optionally, clean up the .aux and .log files generated by LaTeX 201 | for ext in ['.aux', '.log']: 202 | file_to_remove = os.path.join(APPLICATIONS_DIR, f"credit_card_application_{user_id}_{timestamp}{ext}") 203 | if os.path.exists(file_to_remove): 204 | os.remove(file_to_remove) 205 | 206 | return Result( 207 | value=f"Your credit card application has been received and processed successfully. (File: {pdf_filename})", 208 | agent=None # Transfer back to triage in AgentSwarm 209 | ) 210 | -------------------------------------------------------------------------------- /server/agents/applications_agent.py: -------------------------------------------------------------------------------- 1 | # agents/applications_agent.py 2 | 3 | from swarm import Agent 4 | from typing import Callable, Dict 5 | from swarm.types import Result 6 | import logging 7 | import os 8 | from datetime import datetime 9 | import subprocess 10 | from pinata import upload_pdf_to_pinata 11 | 12 | # Define the applications instructions 13 | applications_instructions = """ 14 | You are the Applications Agent responsible for handling applications such as loans, credit cards, and other financial products. 15 | 16 | Your tasks: 17 | 1. Collect necessary information from users to process their applications. 18 | 2. Generate LaTeX documents for each application type with the provided information. 19 | 3. Compile the LaTeX documents into PDF files. 20 | 4. Store the generated PDFs securely for further processing. 21 | 5. Maintain a professional and helpful tone at all times. 22 | """ 23 | 24 | # Directory to store generated applications 25 | APPLICATIONS_DIR = "applications" 26 | 27 | # Ensure the applications directory exists 28 | if not os.path.exists(APPLICATIONS_DIR): 29 | os.makedirs(APPLICATIONS_DIR) 30 | 31 | class ApplicationsAgent(Agent): 32 | def __init__( 33 | self, 34 | transfer_back_to_triage: Callable, 35 | apply_for_loan: Callable, 36 | apply_for_credit_card: Callable, 37 | ): 38 | super().__init__( 39 | name="Applications Agent", 40 | instructions=applications_instructions, 41 | functions=[transfer_back_to_triage, apply_for_loan, apply_for_credit_card], 42 | ) 43 | 44 | # -------------------- Applications Agent Functions -------------------- # 45 | 46 | def apply_for_loan(context_variables: Dict, user_id: str, loan_amount: float, loan_purpose: str, term_years: int) -> Result: 47 | """ 48 | Handles loan applications by generating a LaTeX PDF and storing it. 49 | """ 50 | logging.info(f"Processing loan application for user {user_id}.") 51 | 52 | # Generate LaTeX content 53 | latex_content = f"""\\documentclass{{article}} 54 | \\usepackage{{geometry}} 55 | \\geometry{{a4paper, margin=1in}} 56 | \\begin{{document}} 57 | \\title{{Loan Application}} 58 | \\maketitle 59 | 60 | \\section*{{Applicant Information}} 61 | \\begin{{tabular}}{{ll}} 62 | \\textbf{{User ID}} & {user_id} \\\\ 63 | \\textbf{{Loan Amount}} & \\${loan_amount:,.2f} \\\\ 64 | \\textbf{{Loan Purpose}} & {loan_purpose} \\\\ 65 | \\textbf{{Term}} & {term_years} Years \\\\ 66 | \\textbf{{Date}} & {datetime.now().strftime('%Y-%m-%d')} \\\\ 67 | \\end{{tabular}} 68 | 69 | \\section*{{Terms and Conditions}} 70 | Please review the terms and conditions associated with your loan application. 71 | 72 | \\end{{document}} 73 | """ 74 | 75 | # Define the filenames 76 | timestamp = datetime.now().strftime('%Y%m%d%H%M%S') 77 | tex_filename = f"loan_application_{user_id}_{timestamp}.tex" 78 | pdf_filename = f"loan_application_{user_id}_{timestamp}.pdf" 79 | tex_filepath = os.path.join(APPLICATIONS_DIR, tex_filename) 80 | pdf_filepath = os.path.join(APPLICATIONS_DIR, pdf_filename) 81 | print("MAKING LOAN FILE") 82 | 83 | # Write LaTeX content to .tex file 84 | try: 85 | with open(tex_filepath, 'w') as file: 86 | file.write(latex_content) 87 | logging.info(f"Loan application LaTeX file generated at {tex_filepath}.") 88 | except Exception as e: 89 | logging.error(f"Error writing LaTeX file: {e}") 90 | return Result( 91 | value="An error occurred while generating your loan application. Please try again later.", 92 | agent=None # Transfer back to triage in AgentSwarm 93 | ) 94 | 95 | # Compile LaTeX to PDF using pdflatex 96 | try: 97 | result = subprocess.run( 98 | ['pdflatex', '-interaction=nonstopmode', tex_filename], # Use just the filename 99 | stdout=subprocess.PIPE, 100 | stderr=subprocess.PIPE, 101 | text=True, 102 | cwd=APPLICATIONS_DIR # Set the working directory to APPLICATIONS_DIR 103 | ) 104 | 105 | if result.returncode != 0: 106 | logging.error(f"pdflatex failed with return code {result.returncode}") 107 | logging.error(f"pdflatex stdout: {result.stdout}") 108 | logging.error(f"pdflatex stderr: {result.stderr}") 109 | return Result( 110 | value="An error occurred while compiling your loan application. Please try again later.", 111 | agent=None 112 | ) 113 | else: 114 | logging.info(f"Loan application PDF generated at {pdf_filepath}.") 115 | return Result( 116 | value="Your loan application has been successfully generated.", 117 | agent=None 118 | ) 119 | except subprocess.CalledProcessError as e: 120 | logging.error(f"Error compiling LaTeX to PDF: {e}") 121 | return Result( 122 | value="An error occurred while processing your loan application. Please try again later.", 123 | agent=None 124 | ) 125 | except Exception as e: 126 | logging.error(f"Unexpected error during PDF compilation: {e}") 127 | return Result( 128 | value="An unexpected error occurred. Please try again later.", 129 | agent=None 130 | ) 131 | 132 | finally: 133 | # Optionally, clean up the .aux and .log files generated by LaTeX 134 | for ext in ['.aux', '.log', '.tex']: 135 | file_to_remove = os.path.join(APPLICATIONS_DIR, f"loan_application_{user_id}_{timestamp}{ext}") 136 | if os.path.exists(file_to_remove): 137 | os.remove(file_to_remove) 138 | 139 | # Upload the PDF to Pinata 140 | upload_pdf_to_pinata(pdf_filepath, "LOAN") 141 | 142 | return Result( 143 | value=f"Your loan application has been received and processed successfully. (File: {pdf_filename})", 144 | agent=None # Transfer back to triage in AgentSwarm 145 | ) 146 | 147 | def apply_for_credit_card(context_variables: Dict, user_id: str, card_type: str, credit_limit: float) -> Result: 148 | """ 149 | Handles credit card applications by generating a LaTeX PDF and storing it. 150 | """ 151 | logging.info(f"Processing credit card application for user {user_id}.") 152 | 153 | # Generate LaTeX content 154 | latex_content = f"""\\documentclass{{article}} 155 | \\usepackage{{geometry}} 156 | \\geometry{{a4paper, margin=1in}} 157 | \\begin{{document}} 158 | \\title{{Credit Card Application}} 159 | \\maketitle 160 | 161 | \\section*{{Applicant Information}} 162 | \\begin{{tabular}}{{ll}} 163 | \\textbf{{User ID}} & {user_id} \\\\ 164 | \\textbf{{Card Type}} & {card_type} \\\\ 165 | \\textbf{{Credit Limit}} & \\${credit_limit:,.2f} \\\\ 166 | \\textbf{{Date}} & {datetime.now().strftime('%Y-%m-%d')} \\\\ 167 | \\end{{tabular}} 168 | 169 | \\section*{{Terms and Conditions}} 170 | Please review the terms and conditions associated with your credit card application. 171 | 172 | \\end{{document}} 173 | """ 174 | 175 | # Define the filenames 176 | timestamp = datetime.now().strftime('%Y%m%d%H%M%S') 177 | tex_filename = f"credit_card_application_{user_id}_{timestamp}.tex" 178 | pdf_filename = f"credit_card_application_{user_id}_{timestamp}.pdf" 179 | tex_filepath = os.path.join(APPLICATIONS_DIR, tex_filename) 180 | pdf_filepath = os.path.join(APPLICATIONS_DIR, pdf_filename) 181 | 182 | # Write LaTeX content to .tex file 183 | try: 184 | with open(tex_filepath, 'w') as file: 185 | file.write(latex_content) 186 | logging.info(f"Credit card application LaTeX file generated at {tex_filepath}.") 187 | except Exception as e: 188 | logging.error(f"Error writing LaTeX file: {e}") 189 | return Result( 190 | value="An error occurred while generating your credit card application. Please try again later.", 191 | agent=None 192 | ) 193 | 194 | # Compile LaTeX to PDF using pdflatex 195 | try: 196 | subprocess.run(['pdflatex', '-interaction=nonstopmode', tex_filepath], check=True, cwd=APPLICATIONS_DIR, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 197 | logging.info(f"Credit card application PDF compiled at {pdf_filepath}.") 198 | except subprocess.CalledProcessError as e: 199 | logging.error(f"Error compiling LaTeX to PDF: {e}") 200 | return Result( 201 | value="An error occurred while processing your credit card application. Please try again later.", 202 | agent=None 203 | ) 204 | 205 | # Optionally, clean up the .aux and .log files generated by LaTeX 206 | for ext in ['.aux', '.log']: 207 | file_to_remove = os.path.join(APPLICATIONS_DIR, f"credit_card_application_{user_id}_{timestamp}{ext}") 208 | if os.path.exists(file_to_remove): 209 | os.remove(file_to_remove) 210 | 211 | # Upload the PDF to Pinata 212 | upload_pdf_to_pinata(pdf_filepath, "CARD") 213 | 214 | return Result( 215 | value=f"Your credit card application has been received and processed successfully. (File: {pdf_filename})", 216 | agent=None # Transfer back to triage in AgentSwarm 217 | ) 218 | -------------------------------------------------------------------------------- /experiments/bot.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "User: I would like to check my account balance.\n", 13 | "Agent: " 14 | ] 15 | }, 16 | { 17 | "name": "stderr", 18 | "output_type": "stream", 19 | "text": [ 20 | "2024-11-17 09:02:12,457 - INFO - Current message history: [{'role': 'user', 'content': 'I would like to check my account balance.'}]\n", 21 | "2024-11-17 09:02:13,029 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", 22 | "2024-11-17 09:02:13,132 - INFO - Transferring to Accounts Agent.\n", 23 | "2024-11-17 09:02:13,502 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 24 | ] 25 | }, 26 | { 27 | "name": "stdout", 28 | "output_type": "stream", 29 | "text": [ 30 | "Could you please provide your account ID so I can assist you in checking your account balance?\n", 31 | "\n", 32 | "\n", 33 | "User: ACC123\n", 34 | "Agent: " 35 | ] 36 | }, 37 | { 38 | "name": "stderr", 39 | "output_type": "stream", 40 | "text": [ 41 | "2024-11-17 09:02:13,684 - INFO - Current message history: [{'role': 'user', 'content': 'I would like to check my account balance.'}, {'role': 'assistant', 'content': 'Could you please provide your account ID so I can assist you in checking your account balance?'}, {'role': 'user', 'content': 'ACC123'}]\n", 42 | "2024-11-17 09:02:14,194 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", 43 | "2024-11-17 09:02:14,226 - INFO - Handling account balance for account ID: ACC123\n", 44 | "2024-11-17 09:02:14,227 - INFO - Account ID ACC123 is valid.\n", 45 | "2024-11-17 09:02:14,580 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 46 | ] 47 | }, 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "Your current account balance for account ID ACC123 is $2,500.00. If you need further assistance, please let me know!\n", 53 | "\n", 54 | "\n", 55 | "User: Can you transfer $200 from ACC123 to ACC456?\n", 56 | "Agent: " 57 | ] 58 | }, 59 | { 60 | "name": "stderr", 61 | "output_type": "stream", 62 | "text": [ 63 | "2024-11-17 09:02:14,996 - INFO - Current message history: [{'role': 'user', 'content': 'I would like to check my account balance.'}, {'role': 'assistant', 'content': 'Could you please provide your account ID so I can assist you in checking your account balance?'}, {'role': 'user', 'content': 'ACC123'}, {'role': 'assistant', 'content': 'Your current account balance for account ID ACC123 is $2,500.00. If you need further assistance, please let me know!'}, {'role': 'user', 'content': 'Can you transfer $200 from ACC123 to ACC456?'}]\n", 64 | "2024-11-17 09:02:15,363 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 65 | ] 66 | }, 67 | { 68 | "name": "stdout", 69 | "output_type": "stream", 70 | "text": [ 71 | "Could you please confirm if you'd like me to proceed with the transfer of $200 from account ACC123 to account ACC456?\n", 72 | "\n", 73 | "\n", 74 | "User: I need my bank statement for January.\n", 75 | "Agent: " 76 | ] 77 | }, 78 | { 79 | "name": "stderr", 80 | "output_type": "stream", 81 | "text": [ 82 | "2024-11-17 09:02:16,096 - INFO - Current message history: [{'role': 'user', 'content': 'I would like to check my account balance.'}, {'role': 'assistant', 'content': 'Could you please provide your account ID so I can assist you in checking your account balance?'}, {'role': 'user', 'content': 'ACC123'}, {'role': 'assistant', 'content': 'Your current account balance for account ID ACC123 is $2,500.00. If you need further assistance, please let me know!'}, {'role': 'user', 'content': 'Can you transfer $200 from ACC123 to ACC456?'}, {'role': 'assistant', 'content': \"Could you please confirm if you'd like me to proceed with the transfer of $200 from account ACC123 to account ACC456?\"}, {'role': 'user', 'content': 'I need my bank statement for January.'}]\n", 83 | "2024-11-17 09:02:16,588 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", 84 | "2024-11-17 09:02:16,672 - INFO - Retrieving bank statement for account ID: ACC123, period: January\n", 85 | "2024-11-17 09:02:16,673 - INFO - Account ID ACC123 is valid.\n", 86 | "2024-11-17 09:02:17,124 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 87 | ] 88 | }, 89 | { 90 | "name": "stdout", 91 | "output_type": "stream", 92 | "text": [ 93 | "Here is your bank statement for January for account ID ACC123:\n", 94 | "\n", 95 | "- Transaction 1: -$500.00\n", 96 | "- Transaction 2: +$1500.00\n", 97 | "\n", 98 | "If you need further assistance, please let me know!\n", 99 | "\n", 100 | "\n", 101 | "User: I want to cancel my scheduled payment PAY001.\n", 102 | "Agent: " 103 | ] 104 | }, 105 | { 106 | "name": "stderr", 107 | "output_type": "stream", 108 | "text": [ 109 | "2024-11-17 09:02:17,691 - INFO - Current message history: [{'role': 'user', 'content': 'I would like to check my account balance.'}, {'role': 'assistant', 'content': 'Could you please provide your account ID so I can assist you in checking your account balance?'}, {'role': 'user', 'content': 'ACC123'}, {'role': 'assistant', 'content': 'Your current account balance for account ID ACC123 is $2,500.00. If you need further assistance, please let me know!'}, {'role': 'user', 'content': 'Can you transfer $200 from ACC123 to ACC456?'}, {'role': 'assistant', 'content': \"Could you please confirm if you'd like me to proceed with the transfer of $200 from account ACC123 to account ACC456?\"}, {'role': 'user', 'content': 'I need my bank statement for January.'}, {'role': 'assistant', 'content': 'Here is your bank statement for January for account ID ACC123:\\n\\n- Transaction 1: -$500.00\\n- Transaction 2: +$1500.00\\n\\nIf you need further assistance, please let me know!'}, {'role': 'user', 'content': 'I want to cancel my scheduled payment PAY001.'}]\n", 110 | "2024-11-17 09:02:18,187 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", 111 | "2024-11-17 09:02:18,279 - INFO - Transferring to Payments Agent.\n", 112 | "2024-11-17 09:02:18,864 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n", 113 | "2024-11-17 09:02:19,021 - INFO - Cancelling payment with ID: PAY001\n", 114 | "2024-11-17 09:02:19,021 - INFO - Payment ID PAY001 is valid.\n", 115 | "2024-11-17 09:02:19,933 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n" 116 | ] 117 | }, 118 | { 119 | "name": "stdout", 120 | "output_type": "stream", 121 | "text": [ 122 | "The scheduled payment with ID PAY001 has been successfully canceled. If you have any other requests or need further assistance, please let me know!\n", 123 | "\n", 124 | "\n" 125 | ] 126 | } 127 | ], 128 | "source": [ 129 | "# main.py\n", 130 | "\n", 131 | "from agent_swarm import AgentSwarm\n", 132 | "from db import get_db, set_db\n", 133 | "\n", 134 | "# Initialize the AgentSwarm\n", 135 | "# Replace 'socket' with your actual socket or communication handler if needed\n", 136 | "socket = None # Placeholder for real socket or communication handler\n", 137 | "swarm = AgentSwarm(socket)\n", 138 | "\n", 139 | "# Function to process a new user message\n", 140 | "def process_new_message(swarm, user_message: str):\n", 141 | " \"\"\"\n", 142 | " Processes a new user message through the swarm and handles the response.\n", 143 | " \n", 144 | " Args:\n", 145 | " swarm (AgentSwarm): The AgentSwarm instance.\n", 146 | " user_message (str): The new user message to process.\n", 147 | " \n", 148 | " Yields:\n", 149 | " str: The assistant's response chunks.\n", 150 | " \"\"\"\n", 151 | " message = {\"role\": \"user\", \"content\": user_message}\n", 152 | " response = swarm.run([message], stream=True)\n", 153 | " for chunk in response:\n", 154 | " yield chunk # Yield each chunk as it's received\n", 155 | "\n", 156 | "# Example usage\n", 157 | "if __name__ == \"__main__\":\n", 158 | " # List of user interactions (including follow-ups)\n", 159 | " user_interactions = [\n", 160 | " {\"user_message\": \"I would like to check my account balance.\"},\n", 161 | " {\"user_message\": \"ACC123\"}, # Follow-up providing account ID\n", 162 | " {\"user_message\": \"Can you transfer $200 from ACC123 to ACC456?\"},\n", 163 | " {\"user_message\": \"I need my bank statement for January.\"},\n", 164 | " {\"user_message\": \"I want to cancel my scheduled payment PAY001.\"}\n", 165 | " ]\n", 166 | " \n", 167 | " for interaction in user_interactions:\n", 168 | " user_message = interaction[\"user_message\"]\n", 169 | " print(f\"User: {user_message}\")\n", 170 | " try:\n", 171 | " print(\"Agent: \", end=\"\", flush=True)\n", 172 | " # Process the user message and iterate over the response chunks\n", 173 | " for chunk in process_new_message(swarm, user_message):\n", 174 | " print(f\"{chunk}\", end=\"\", flush=True)\n", 175 | " print(\"\\n\") # For readability between interactions\n", 176 | " except Exception as e:\n", 177 | " print(f\"Agent: An error occurred: {str(e)}\\n\")\n" 178 | ] 179 | } 180 | ], 181 | "metadata": { 182 | "kernelspec": { 183 | "display_name": "talktuah", 184 | "language": "python", 185 | "name": "python3" 186 | }, 187 | "language_info": { 188 | "codemirror_mode": { 189 | "name": "ipython", 190 | "version": 3 191 | }, 192 | "file_extension": ".py", 193 | "mimetype": "text/x-python", 194 | "name": "python", 195 | "nbconvert_exporter": "python", 196 | "pygments_lexer": "ipython3", 197 | "version": "3.10.13" 198 | } 199 | }, 200 | "nbformat": 4, 201 | "nbformat_minor": 2 202 | } 203 | -------------------------------------------------------------------------------- /server/db.py: -------------------------------------------------------------------------------- 1 | # db.py 2 | from datetime import datetime, timedelta 3 | import random 4 | 5 | # Initial Banking Database Structure 6 | db = { 7 | "users": { 8 | "+1-408-585-8267": { 9 | "name": "Bill Zhang", 10 | "accounts": ["ACC892", "ACC347"], 11 | "ssn": "784-29-5163", 12 | "address": "4728 Willow Creek Rd, Austin, TX 78701", 13 | "date_of_birth": "1991-08-24", 14 | "email": "bill.zhang@outlook.com", 15 | "phone": "+1-408-585-8267", 16 | }, 17 | "+1-917-828-6465": { 18 | "name": "Warren Yun", 19 | "accounts": ["ACC123", "ACC456"], 20 | "ssn": "123-45-6789", 21 | "address": "123 Main St, Dallas, TX 75201", 22 | "date_of_birth": "1985-03-15", 23 | "email": "warren.yun@email.com", 24 | "phone": "+1-917-828-6465", 25 | }, 26 | "+14695550456": { 27 | "name": "Jane Smith", 28 | "accounts": ["ACC789", "ACC790", "ACC791"], 29 | "ssn": "987-65-4321", 30 | "address": "456 Oak Ave, Plano, TX 75024", 31 | "date_of_birth": "1990-07-22", 32 | "email": "jane.smith@email.com", 33 | "phone": "+1-469-555-0456", 34 | }, 35 | "+1-972-555-0789": { 36 | "name": "Michael Johnson", 37 | "accounts": ["ACC321", "ACC322"], 38 | "ssn": "456-78-9012", 39 | "address": "789 Elm St, Richardson, TX 75080", 40 | "date_of_birth": "1988-11-30", 41 | "email": "michael.j@email.com", 42 | "phone": "+1-972-555-0789", 43 | }, 44 | "+1-817-555-0321": { 45 | "name": "Sarah Williams", 46 | "accounts": ["ACC654", "ACC655", "ACC656"], 47 | "ssn": "234-56-7890", 48 | "address": "321 Pine Rd, Fort Worth, TX 76102", 49 | "date_of_birth": "1992-04-18", 50 | "email": "sarah.w@email.com", 51 | "phone": "+1-817-555-0321", 52 | }, 53 | "+1-214-555-0654": { 54 | "name": "David Brown", 55 | "accounts": ["ACC987", "ACC988"], 56 | "ssn": "345-67-8901", 57 | "address": "654 Maple Dr, Frisco, TX 75034", 58 | "date_of_birth": "1983-09-25", 59 | "email": "david.b@email.com", 60 | "phone": "+1-214-555-0654", 61 | }, 62 | }, 63 | "accounts": { 64 | "ACC123": { 65 | "balance": 2500.00, 66 | "statements": { 67 | "January": "Transaction 1: -$500.00\nTransaction 2: +$1500.00", 68 | "February": "Transaction 1: -$300.00\nTransaction 2: +$800.00", 69 | }, 70 | }, 71 | "ACC456": { 72 | "balance": 1000.00, 73 | "statements": { 74 | "January": "Transaction 1: -$200.00\nTransaction 2: +$1200.00", 75 | "February": "Transaction 1: -$100.00\nTransaction 2: +$500.00", 76 | }, 77 | }, 78 | "ACC789": { 79 | "balance": 5000.00, 80 | "statements": { 81 | "January": "Transaction 1: -$1000.00\nTransaction 2: +$2000.00", 82 | "February": "Transaction 1: -$500.00\nTransaction 2: +$1500.00", 83 | }, 84 | }, 85 | "ACC790": { 86 | "balance": 12500.00, 87 | "statements": { 88 | "January": "Transaction 1: -$2000.00\nTransaction 2: +$5000.00", 89 | "February": "Transaction 1: -$1500.00\nTransaction 2: +$3000.00", 90 | }, 91 | }, 92 | "ACC791": { 93 | "balance": 8000.00, 94 | "statements": { 95 | "January": "Transaction 1: -$1000.00\nTransaction 2: +$2500.00", 96 | "February": "Transaction 1: -$800.00\nTransaction 2: +$1800.00", 97 | }, 98 | }, 99 | "ACC892": { 100 | "balance": 3750.00, 101 | "statements": { 102 | "January": "Transaction 1: -$800.00\nTransaction 2: +$2500.00", 103 | "February": "Transaction 1: -$450.00\nTransaction 2: +$1200.00", 104 | }, 105 | }, 106 | "ACC347": { 107 | "balance": 8200.00, 108 | "statements": { 109 | "January": "Transaction 1: -$1500.00\nTransaction 2: +$3000.00", 110 | "February": "Transaction 1: -$700.00\nTransaction 2: +$2100.00", 111 | }, 112 | }, 113 | "ACC321": { 114 | "balance": 15000.00, 115 | "statements": { 116 | "January": "Transaction 1: -$3000.00\nTransaction 2: +$5000.00", 117 | "February": "Transaction 1: -$2000.00\nTransaction 2: +$4000.00", 118 | }, 119 | }, 120 | "ACC322": { 121 | "balance": 6500.00, 122 | "statements": { 123 | "January": "Transaction 1: -$1200.00\nTransaction 2: +$2800.00", 124 | "February": "Transaction 1: -$900.00\nTransaction 2: +$1600.00", 125 | }, 126 | }, 127 | "ACC654": { 128 | "balance": 9800.00, 129 | "statements": { 130 | "January": "Transaction 1: -$1800.00\nTransaction 2: +$3500.00", 131 | "February": "Transaction 1: -$1200.00\nTransaction 2: +$2500.00", 132 | }, 133 | }, 134 | "ACC655": { 135 | "balance": 4200.00, 136 | "statements": { 137 | "January": "Transaction 1: -$900.00\nTransaction 2: +$1500.00", 138 | "February": "Transaction 1: -$600.00\nTransaction 2: +$1200.00", 139 | }, 140 | }, 141 | "ACC656": { 142 | "balance": 7500.00, 143 | "statements": { 144 | "January": "Transaction 1: -$1500.00\nTransaction 2: +$2800.00", 145 | "February": "Transaction 1: -$1000.00\nTransaction 2: +$2000.00", 146 | }, 147 | }, 148 | "ACC987": { 149 | "balance": 11000.00, 150 | "statements": { 151 | "January": "Transaction 1: -$2500.00\nTransaction 2: +$4500.00", 152 | "February": "Transaction 1: -$1800.00\nTransaction 2: +$3200.00", 153 | }, 154 | }, 155 | "ACC988": { 156 | "balance": 5800.00, 157 | "statements": { 158 | "January": "Transaction 1: -$1200.00\nTransaction 2: +$2200.00", 159 | "February": "Transaction 1: -$800.00\nTransaction 2: +$1500.00", 160 | }, 161 | }, 162 | }, 163 | "payments": { 164 | "PAY001": { 165 | "from_account": "ACC123", 166 | "to_account": "ACC456", 167 | "amount": 300.00, 168 | "date": "2024-05-01", 169 | "status": "Scheduled", 170 | }, 171 | "PAY002": { 172 | "from_account": "ACC456", 173 | "to_account": "ACC123", 174 | "amount": 150.00, 175 | "date": "2024-05-15", 176 | "status": "Completed", 177 | }, 178 | "PAY003": { 179 | "from_account": "ACC789", 180 | "to_account": "ACC892", 181 | "amount": 500.00, 182 | "date": "2024-04-30", 183 | "status": "Completed", 184 | }, 185 | "PAY004": { 186 | "from_account": "ACC892", 187 | "to_account": "ACC347", 188 | "amount": 750.00, 189 | "date": "2024-05-10", 190 | "status": "Pending", 191 | }, 192 | "PAY005": { 193 | "from_account": "ACC347", 194 | "to_account": "ACC789", 195 | "amount": 1000.00, 196 | "date": "2024-05-20", 197 | "status": "Scheduled", 198 | }, 199 | "PAY006": { 200 | "from_account": "ACC123", 201 | "to_account": "ACC789", 202 | "amount": 250.00, 203 | "date": "2024-04-25", 204 | "status": "Completed", 205 | }, 206 | "PAY007": { 207 | "from_account": "ACC790", 208 | "to_account": "ACC321", 209 | "amount": 1500.00, 210 | "date": "2024-05-05", 211 | "status": "Completed", 212 | }, 213 | "PAY008": { 214 | "from_account": "ACC321", 215 | "to_account": "ACC654", 216 | "amount": 2000.00, 217 | "date": "2024-05-12", 218 | "status": "Pending", 219 | }, 220 | "PAY009": { 221 | "from_account": "ACC655", 222 | "to_account": "ACC987", 223 | "amount": 800.00, 224 | "date": "2024-05-18", 225 | "status": "Scheduled", 226 | }, 227 | "PAY010": { 228 | "from_account": "ACC988", 229 | "to_account": "ACC791", 230 | "amount": 1200.00, 231 | "date": "2024-04-28", 232 | "status": "Completed", 233 | }, 234 | }, 235 | } 236 | 237 | calls_db = {} 238 | 239 | 240 | def generate_sample_data(): 241 | sample_data = [ 242 | { 243 | "id": "+1-408-585-8267", 244 | "time": ( 245 | datetime.now() - timedelta(minutes=random.randint(0, 120)) 246 | ).isoformat(), 247 | "transcript": [ 248 | { 249 | "role": "assistant", 250 | "content": "Hello, welcome to TalkTuahBank. How can I help you today?", 251 | }, 252 | { 253 | "role": "user", 254 | "content": "Hi, I'd like to check my account balance please", 255 | }, 256 | { 257 | "role": "assistant", 258 | "content": "I'd be happy to help you check your balance. Could you please verify your account number?", 259 | }, 260 | {"role": "user", "content": "Yes, it's ACC123"}, 261 | ], 262 | "user_id": "+1-408-585-8267", 263 | "referenced_documents": [], 264 | }, 265 | { 266 | "id": "+1-917-828-6465", 267 | "time": ( 268 | datetime.now() - timedelta(minutes=random.randint(0, 120)) 269 | ).isoformat(), 270 | "transcript": [ 271 | { 272 | "role": "assistant", 273 | "content": "Hello, welcome to TalkTuahBank. How can I help you today?", 274 | }, 275 | {"role": "user", "content": "I need to make a payment to someone"}, 276 | { 277 | "role": "assistant", 278 | "content": "I can help you with making a payment. Could you tell me who you'd like to send money to?", 279 | }, 280 | {"role": "user", "content": "I want to send $50 to my friend John"}, 281 | ], 282 | "user_id": "+1-917-828-6465", 283 | "referenced_documents": [], 284 | }, 285 | { 286 | "id": "+1-972-555-0789", 287 | "time": ( 288 | datetime.now() - timedelta(minutes=random.randint(0, 120)) 289 | ).isoformat(), 290 | "transcript": [ 291 | { 292 | "role": "assistant", 293 | "content": "Hello, welcome to TalkTuahBank. How can I help you today?", 294 | }, 295 | {"role": "user", "content": "I lost my credit card"}, 296 | { 297 | "role": "assistant", 298 | "content": "I'm sorry to hear that. I'll help you report your lost card right away. First, can you verify your identity?", 299 | }, 300 | {"role": "user", "content": "Yes, my account number is ACC456"}, 301 | ], 302 | "user_id": "+1-972-555-0789", 303 | "referenced_documents": [], 304 | }, 305 | { 306 | "id": "+1-817-555-0321", 307 | "time": ( 308 | datetime.now() - timedelta(minutes=random.randint(0, 120)) 309 | ).isoformat(), 310 | "transcript": [ 311 | { 312 | "role": "assistant", 313 | "content": "Hello, welcome to TalkTuahBank. How can I help you today?", 314 | }, 315 | {"role": "user", "content": "I'd like to see my recent transactions"}, 316 | { 317 | "role": "assistant", 318 | "content": "I can help you review your recent transactions. Could you please provide your account number?", 319 | }, 320 | {"role": "user", "content": "It's ACC789"}, 321 | ], 322 | "user_id": "+1-817-555-0321", 323 | "referenced_documents": [], 324 | }, 325 | { 326 | "id": "+1-214-555-0654", 327 | "time": ( 328 | datetime.now() - timedelta(minutes=random.randint(0, 120)) 329 | ).isoformat(), 330 | "transcript": [ 331 | { 332 | "role": "assistant", 333 | "content": "Hello, welcome to TalkTuahBank. How can I help you today?", 334 | }, 335 | {"role": "user", "content": "I need help with my mortgage payment"}, 336 | { 337 | "role": "assistant", 338 | "content": "I'll be happy to assist you with your mortgage payment. What specific information do you need?", 339 | }, 340 | { 341 | "role": "user", 342 | "content": "I want to know when my next payment is due", 343 | }, 344 | ], 345 | "user_id": "+1-214-555-0654", 346 | "referenced_documents": [], 347 | }, 348 | ] 349 | for call in sample_data: 350 | calls_db[call["id"]] = call 351 | return sample_data 352 | 353 | 354 | generate_sample_data() 355 | 356 | 357 | def get_db(): 358 | """ 359 | Retrieves the current state of the banking database. 360 | 361 | Returns: 362 | dict: The current banking database dictionary. 363 | """ 364 | return db 365 | 366 | 367 | def set_db(new_db): 368 | """ 369 | Updates the banking database with a new state. 370 | 371 | Args: 372 | new_db (dict): The new banking database dictionary to replace the current state. 373 | 374 | Returns: 375 | bool: True if the database was successfully updated. 376 | """ 377 | global db 378 | db = new_db 379 | return True 380 | 381 | 382 | def get_all_calls(): 383 | """ 384 | Accessor function to get all calls in the database. 385 | """ 386 | return calls_db 387 | 388 | 389 | def get_call(call_id): 390 | """ 391 | Accessor function to get a specific call by its ID. 392 | """ 393 | return calls_db.get(call_id, {}) 394 | 395 | 396 | def update_call(call_id, updated_data): 397 | """ 398 | Modification function to update or insert a specific call's data. 399 | """ 400 | if call_id in calls_db: 401 | calls_db[call_id].update(updated_data) 402 | else: 403 | calls_db[call_id] = updated_data 404 | return True 405 | 406 | 407 | def delete_call(call_id): 408 | """ 409 | Function to delete a call from the database. 410 | """ 411 | if call_id in calls_db: 412 | del calls_db[call_id] 413 | return True 414 | return False 415 | -------------------------------------------------------------------------------- /client/src/app/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import { useState, useEffect } from 'react' 4 | import { Eye, EyeOff, FileText, UserCog, User, BotIcon as Robot, ChevronDown } from 'lucide-react' 5 | import { motion } from 'framer-motion' 6 | import logo from '../img/logo_dark.png' 7 | 8 | interface ChatMessage { 9 | role: string, 10 | content: string 11 | } 12 | 13 | interface CallResponse { 14 | event: "calls_response"; 15 | data: Record; 16 | } 17 | 18 | interface Call { 19 | id: string 20 | transcript: ChatMessage[] 21 | referenced_documents: Document[] 22 | user_id: string 23 | } 24 | 25 | interface DBResponse { 26 | event: "db_response"; 27 | data: Database; 28 | } 29 | 30 | interface CombinedResponse { 31 | event: "combined_response"; 32 | calls: Record; 33 | db: Database; 34 | } 35 | 36 | interface Database { 37 | users: { 38 | [key: string]: { 39 | name: string 40 | accounts: string[] 41 | ssn: string 42 | address: string 43 | date_of_birth: string 44 | email: string 45 | phone: string 46 | } 47 | } 48 | accounts: { 49 | [key: string]: { 50 | balance: number 51 | statements: { 52 | [month: string]: string 53 | } 54 | } 55 | } 56 | payments: { 57 | [key: string]: { 58 | from_account: string 59 | to_account: string 60 | amount: number 61 | date: string 62 | status: string 63 | } 64 | } 65 | } 66 | 67 | export interface CallProps { 68 | call?: Call; 69 | selectedId: string | undefined; 70 | } 71 | 72 | interface Document { 73 | id: string 74 | name: string 75 | type: string 76 | status?: 'urgent' | 'medium' | 'done' 77 | } 78 | 79 | interface UserInfo { 80 | id: string 81 | name: string 82 | ssn: string 83 | address: string 84 | date_of_birth: string 85 | email: string 86 | phone: string 87 | } 88 | 89 | interface Transaction { 90 | id: string 91 | date: string 92 | description: string 93 | amount: string 94 | } 95 | 96 | const analyzeDocumentStatus = (doc: { name: string; type: string }): 'urgent' | 'medium' | 'done' => { 97 | if (doc.name.toLowerCase().includes('dispute')) return 'urgent'; 98 | if (doc.type === 'PDF') return 'medium'; 99 | return 'done'; 100 | }; 101 | 102 | export default function AdminDashboard() { 103 | const [showSensitiveInfo, setShowSensitiveInfo] = useState(false) 104 | const [isMenuOpen, setIsMenuOpen] = useState(false); 105 | const [selectedOption, setSelectedOption] = useState('info'); 106 | const [isDropdownOpen, setIsDropdownOpen] = useState(false); 107 | const [connected, setConnected] = useState(false); 108 | const [data, setData] = useState>({}); 109 | const [selectedId, setSelectedId] = useState(""); 110 | const [bankData, setBankData] = useState({ 111 | users: {}, 112 | accounts: {}, 113 | payments: {} 114 | }); 115 | const [socket, setSocket] = useState(null) 116 | const [userInfo, setUserInfo] = useState({ 117 | id: "", 118 | name: "", 119 | ssn: "", 120 | address: "", 121 | date_of_birth: "", 122 | email: "", 123 | phone: "" 124 | }); 125 | 126 | const [transactions, setTransactions] = useState([]); 127 | const [userAccounts, setUserAccounts] = useState<{accountNumber: string, balance: string}[]>([]); 128 | 129 | useEffect(() => { 130 | const wss = new WebSocket( 131 | "ws://localhost:8000/ws?client_id=1234", 132 | ); 133 | console.log("useEffect") 134 | wss.onopen = () => { 135 | console.log('WebSocket connection established') 136 | setConnected(true) 137 | } 138 | 139 | wss.onmessage = (event: MessageEvent) => { 140 | console.log("Received message"); 141 | const data = JSON.parse(event.data) 142 | console.log(data) 143 | if (data.event === "combined_response") { 144 | setData(data.calls) 145 | setBankData(data.db) 146 | } 147 | else if (data.event === "db_response") { 148 | setBankData(data.data) 149 | } 150 | else if (data.event === "calls_response") { 151 | setData(data.data) 152 | } 153 | } 154 | 155 | setSocket(wss) 156 | }, []); 157 | 158 | useEffect(() => { 159 | const interval = setInterval(() => { 160 | if (socket && socket.readyState === WebSocket.OPEN) { 161 | socket.send( 162 | JSON.stringify({ 163 | event: "get_all_dbs", 164 | }) 165 | ); 166 | } 167 | }, 2000); 168 | 169 | return () => clearInterval(interval); 170 | }, [socket]); 171 | 172 | const handleCallSelect = (id: string) => { 173 | setSelectedId(id); 174 | const selectedCall = data[id]; 175 | if (selectedCall && bankData.users && bankData.users[selectedCall.user_id]) { 176 | const user = bankData.users[selectedCall.user_id]; 177 | const accounts = user.accounts.map(accId => { 178 | const account = bankData.accounts[accId]; 179 | return { 180 | accountNumber: accId, 181 | balance: account ? `$${account.balance.toFixed(2)}` : "N/A" 182 | }; 183 | }); 184 | 185 | setUserAccounts(accounts); 186 | 187 | // Update user info 188 | setUserInfo({ 189 | id: selectedCall.user_id, 190 | name: user.name, 191 | ssn: user.ssn, 192 | address: user.address, 193 | date_of_birth: user.date_of_birth, 194 | email: user.email, 195 | phone: user.phone 196 | }); 197 | 198 | // Update transactions from payments 199 | const userTransactions: Transaction[] = Object.entries(bankData.payments) 200 | .filter(([_, payment]) => 201 | payment.from_account === accounts[0]?.accountNumber || 202 | payment.to_account === accounts[0]?.accountNumber 203 | ) 204 | .map(([id, payment]) => ({ 205 | id, 206 | date: payment.date, 207 | description: `${payment.from_account === accounts[0]?.accountNumber ? 'Payment to' : 'Payment from'} ${payment.from_account === accounts[0]?.accountNumber ? payment.to_account : payment.from_account 208 | }`, 209 | amount: `${payment.from_account === accounts[0]?.accountNumber ? '-' : '+'}$${payment.amount.toFixed(2)}` 210 | })); 211 | 212 | setTransactions(userTransactions); 213 | } 214 | }; 215 | 216 | const maskSensitiveInfo = (info: string) => { 217 | if (!info) return ''; 218 | return !showSensitiveInfo ? info.replace(/\d/g, '*') : info; 219 | } 220 | 221 | return ( 222 |
223 | {/* Navigation Bar */} 224 | 244 | 245 |
246 | {/* Side Menu */} 247 |
248 |
249 |

Previous Messages

250 |
    251 | {data && Object.entries(data).map(([id, call]) => ( 252 |
  • handleCallSelect(id)} 256 | > 257 | {call.id} 258 |
    • 259 | ))} 260 |
261 |
262 |
263 | 264 | {/* Main Content */} 265 |
266 |
267 |
268 | {/* User Information Card */} 269 | 274 |
275 |
276 |
277 | 286 | {isDropdownOpen && ( 287 |
288 |
289 | 299 | 309 | 319 |
320 |
321 | )} 322 |
323 | 330 |
331 | {selectedOption === 'info' ? ( 332 |
333 |
334 | 335 |

{userInfo.id}

336 |
337 |
338 | 339 |

{userInfo.name}

340 |
341 |
342 | 343 |

{maskSensitiveInfo(userInfo.ssn)}

344 |
345 |
346 | 347 |

{userInfo.address}

348 |
349 |
350 | 351 |

{userInfo.date_of_birth}

352 |
353 |
354 | 355 |

{userInfo.email}

356 |
357 |
358 | 359 |

{userInfo.phone}

360 |
361 |
362 | ) : selectedOption === 'transactions' ? ( 363 |
364 | {transactions.map((transaction) => ( 365 |
366 |
367 |

{transaction.description}

368 |

{transaction.date}

369 |
370 |

371 | {transaction.amount} 372 |

373 |
374 | ))} 375 |
376 | ) : ( 377 |
378 | {userAccounts.map((account, index) => ( 379 |
380 |
381 |
382 | 383 |

{maskSensitiveInfo(account.accountNumber)}

384 |
385 |
386 | 387 |

{account.balance}

388 |
389 |
390 |
391 | ))} 392 |
393 | )} 394 |
395 | 396 | 397 | {/* Chat Transcript Card */} 398 | 404 |
405 |

Chat Transcript

406 |
407 | {selectedId && data[selectedId] ? ( 408 | data[selectedId].transcript.map((message, index) => ( 409 |
416 |
417 |
418 | {message.role === 'assistant' ? ( 419 | 420 | ) : ( 421 | 422 | )} 423 |
424 |
425 | {message.role} 426 |
427 |
428 |

{message.content}

429 |
430 | )) 431 | ) : ( 432 |
433 | Select a conversation from the sidebar to view the transcript 434 |
435 | )} 436 |
437 | 443 | 447 | 448 |
449 | 450 | 451 | {/* Documents Card */} 452 | 458 |
459 |

Referenced Documents

460 |
461 | {selectedId && data[selectedId] ? ( 462 | data[selectedId].referenced_documents.map((doc) => ( 463 |
467 |
468 |
469 |

{doc.name}

470 |

{doc.type}

471 |
472 | 480 | {doc.status} 481 | 482 |
483 | 487 |
488 | )) 489 | ) : ( 490 |
491 | Select a conversation to view referenced documents 492 |
493 | )} 494 |
495 |
496 | 497 |
498 |
499 |
500 |
501 | 502 | 518 |
519 | ) 520 | } --------------------------------------------------------------------------------